From 446ef5e1b7756cc6bc812fe8d7850a6cecc3c7ea Mon Sep 17 00:00:00 2001 From: VasiliyF <5789590+vfofanov@users.noreply.github.com> Date: Mon, 5 Oct 2026 04:33:12 +0200 Subject: [PATCH 01/10] Add the DragoAntObserverLocal sibling-checkout switch - DragoAntObserverLocal=true swaps the DragoAnt.Observer.Core package for a ProjectReference into ../Observer (DragoAntObserverRoot) - CI checks out DragoAnt/Observer (same branch, else main) and builds with the switch on until Core 1.0.0 is published --- .github/workflows/ci.yml | 110 ++++++++++++++++++++++++++++++-- Directory.PackageAsProj.targets | 21 +++++- 2 files changed, 125 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e6fe783..30751ca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,10 +14,110 @@ concurrency: group: ci-${{ github.ref }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} +# Until DragoAnt.Observer.Core 1.0.0 is on nuget.org this job builds it from source: it checks out +# DragoAnt/Observer next to this repository (the branch of the same name when it exists, else main) +# and builds with DragoAntObserverLocal=true. Switch back to the shared dotnet-build.yml caller once +# the package is published. jobs: build: - name: Build and test - uses: DragoAnt/.github/.github/workflows/dotnet-build.yml@c9f20c11ebd297193a910d965d33bdf37c0d6c42 # main - with: - version: 0.0.0-ci.${{ github.run_number }} - coverage-threshold: 70 + name: Build and test (Observer from source) + runs-on: ubuntu-24.04 + timeout-minutes: 30 + env: + DOTNET_NOLOGO: true + DOTNET_CLI_TELEMETRY_OPTOUT: true + DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true + NUGET_PACKAGES: ${{ github.workspace }}/.nuget/packages + CONFIGURATION: Release + SOLUTION: DragoAnt.System.Text.Json.slnx + RESULTS_DIR: ${{ github.workspace }}/TestResults + COVERAGE_DIR: ${{ github.workspace }}/coverage + COVERAGE_THRESHOLD: 70 + BUILD_PROPS: -p:ContinuousIntegrationBuild=true -p:Version=0.0.0-ci.${{ github.run_number }} -p:DragoAntObserverLocal=true + defaults: + run: + working-directory: Extensions.System.Text.Json + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: Extensions.System.Text.Json + persist-credentials: false + + - name: Pick the DragoAnt/Observer ref + id: observer + working-directory: ${{ github.workspace }} + env: + BRANCH: ${{ github.head_ref || github.ref_name }} + run: | + if git ls-remote --exit-code --heads https://github.com/DragoAnt/Observer.git "$BRANCH" > /dev/null; then + ref="$BRANCH" + else + ref=main + fi + echo "ref=$ref" >> "$GITHUB_OUTPUT" + echo "Building DragoAnt.Observer.Core from DragoAnt/Observer@$ref" >> "$GITHUB_STEP_SUMMARY" + + - name: Checkout DragoAnt/Observer + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: DragoAnt/Observer + ref: ${{ steps.observer.outputs.ref }} + path: Observer + persist-credentials: false + + - name: Install .NET + uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0 + with: + dotnet-version: | + 8.0.x + global-json-file: Extensions.System.Text.Json/global.json + + - name: NuGet cache + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ env.NUGET_PACKAGES }} + key: nuget-${{ runner.os }}-${{ hashFiles('**/global.json', '**/Directory.Packages.props', '**/*.csproj') }} + restore-keys: | + nuget-${{ runner.os }}- + + - name: Restore + run: | + # shellcheck disable=SC2086 + dotnet restore "$SOLUTION" $BUILD_PROPS + + - name: Build + run: | + # shellcheck disable=SC2086 + dotnet build "$SOLUTION" -c "$CONFIGURATION" --no-restore $BUILD_PROPS + + - name: Test + run: | + dotnet test --solution "$SOLUTION" -c "$CONFIGURATION" --no-build --no-progress \ + --results-directory "$RESULTS_DIR" --report-xunit-trx --report-xunit-junit --coverage --coverage-output-format cobertura + + - name: Coverage + if: ${{ !cancelled() && hashFiles('TestResults/**/*.cobertura.xml') != '' }} + run: | + dotnet tool install dotnet-reportgenerator-globaltool --version 5.5.11 --tool-path "$RUNNER_TEMP/reportgenerator" + "$RUNNER_TEMP/reportgenerator/reportgenerator" \ + "-reports:$RESULTS_DIR/**/*.cobertura.xml" \ + "-targetdir:$COVERAGE_DIR" \ + "-reporttypes:MarkdownSummaryGithub;Cobertura" \ + "-assemblyfilters:-*.Tests;-*.Tests.*;-*.Benchmarks;-DragoAnt.Observer.*" + cat "$COVERAGE_DIR/SummaryGithub.md" >> "$GITHUB_STEP_SUMMARY" + rate=$(grep -oE ']*line-rate="[0-9.]+"' "$COVERAGE_DIR/Cobertura.xml" | grep -oE 'line-rate="[0-9.]+"' | grep -oE '[0-9.]+') + awk -v rate="$rate" -v min="$COVERAGE_THRESHOLD" 'BEGIN { + pct = rate * 100 + printf "Line coverage %.1f%% (minimum %s%%)\n", pct, min + if (pct + 1e-9 < min) { printf "::error::line coverage %.1f%% is below the %s%% threshold\n", pct, min; exit 1 } + }' + + - name: Upload test results + if: ${{ !cancelled() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: test-results + path: ${{ env.RESULTS_DIR }} + if-no-files-found: ignore + retention-days: 14 diff --git a/Directory.PackageAsProj.targets b/Directory.PackageAsProj.targets index c0e8fec..772f97e 100644 --- a/Directory.PackageAsProj.targets +++ b/Directory.PackageAsProj.targets @@ -10,4 +10,23 @@ - \ No newline at end of file + + + $(MSBuildThisFileDirectory)../Observer/ + $([MSBuild]::EnsureTrailingSlash('$(DragoAntObserverRoot)')) + <_DragoAntObserverCoreProject>$(DragoAntObserverRoot)DragoAnt.Observer.Core/DragoAnt.Observer.Core.csproj + <_DragoAntObserverAsProj>false + <_DragoAntObserverAsProj Condition="'$(DragoAntObserverLocal)'=='true'">true + + + + + + + + + + + From 683e203bc53bfd7d799d9545189545405fe3b7e5 Mon Sep 17 00:00:00 2001 From: VasiliyF <5789590+vfofanov@users.noreply.github.com> Date: Mon, 5 Oct 2026 09:49:13 +0200 Subject: [PATCH 02/10] Rebuild STJ.Observer on DragoAnt.Observer.Core - moved types come from Core under their final names: MaskTag, MaskKind (Omit is Null), MaskResult, MaskStatus (NotJson is Unrecognized), PathExplanation, PathOutcome, UnknownMemberPolicy, NoContext, DataPath, NameMatch and Names - JsonObserverOptions derives from ObserverOptions; JsonShapeOptions and JsonBodyLoggingContext have init properties; JsonShapeOptions.UnknownTag - strategies are ValueMaskStrategy writing to a MaskValueWriter - one custom rule delegate, JsonValueRule, and JsonValuePolicy accepting ValuePolicy for any context; LegacyAllowList removed - one masking verb Mask(tag | replacement | strategy) with MaskNulls; Relative is AnyDepth, multi-name Match is Path - Explain takes a ValueKind; FindMember and the case flag are internal - the package imports DragoAnt.Observer (DragoAntObserverImplicitUsing) - net8.0 and net10.0 only --- Directory.Build.props | 2 +- Directory.Build.targets | 4 + Directory.Packages.props | 1 + .../Baselines.cs | 14 +- .../ExtractBenchmarks.cs | 6 +- .../Harness/Probes.cs | 26 +- .../DocSnippetTests.cs | 2 +- .../JsonBodyLoggingTests.cs | 4 +- .../ReadmeSampleTests.cs | 2 +- .../JsonBodyLogging.cs | 4 +- .../JsonBodyLoggingContext.cs | 21 +- .../JsonBodyLoggingHandler.cs | 4 +- .../package.readme.md | 4 +- .../SkillSnippetTests.cs | 2 +- .../AllocationTests.cs | 25 +- .../ApiSurfaceTests.cs | 84 ++--- .../BytesApiTests.cs | 23 +- .../CaseSensitivityTests.cs | 32 +- .../DefaultPolicyTests.cs | 7 +- .../ExplainTests.cs | 128 +++---- .../JsonMaskingTests.cs | 61 +-- .../JsonReadTests.cs | 18 +- .../JsonShapeMetadataTests.cs | 2 +- .../JsonShapeTests.cs | 20 +- .../JsonWriterSpanTests.cs | 24 +- .../LeakTests.cs | 74 ++-- .../LimitsTests.cs | 25 +- .../MaskAndExtractTests.cs | 2 +- .../MaskTagTests.cs | 92 ++--- .../NameMatchingTests.cs | 33 +- .../NestedArrayRuleTests.cs | 17 +- .../NestingTests.cs | 21 +- .../PassThroughTests.cs | 5 +- .../PropertyPathTests.cs | 43 ++- .../RelativeNullTests.cs | 21 +- .../RobustnessTests.cs | 13 +- .../RuleCoverageTests.cs | 40 +- .../SensitiveRuleTests.cs | 17 +- .../SequenceInputTests.cs | 43 +-- .../ShapeCoverageTests.cs | 4 +- .../StringApiTests.cs | 25 +- .../WholeValueMaskingTests.cs | 11 +- .../BoundedJsonWriter.cs | 2 +- .../Builders/JsonAnyDepthBuilder.cs | 316 ++++++++++++++++ .../Builders/JsonArrayBuilder.cs | 114 +++--- .../Builders/JsonObjBuilder.cs | 217 +++++------ .../Builders/JsonValuePolicyBuilder.cs | 356 ------------------ .../Builders/RuleText.cs | 15 +- .../DragoAnt.System.Text.Json.Observer.csproj | 6 + .../GlobalUsings.cs | 3 +- .../JsonObserver.cs | 133 +++---- .../JsonObserverDelegate.cs | 38 +- .../JsonObserverItem.cs | 134 ++++--- .../JsonObserverOptions.cs | 52 +-- .../JsonObserverValuePolicies.cs | 212 ----------- .../JsonObserveringEmptyContext.cs | 16 - .../JsonPathExplanation.cs | 48 --- .../JsonShape.cs | 5 +- .../JsonShapeOptions.cs | 54 ++- .../JsonValueContext.cs | 94 +++++ .../JsonValuePolicy.cs | 234 ++++++++++++ .../JsonWalk.cs | 61 +++ .../JsonWriter.cs | 4 +- .../MaskResult.cs | 42 --- ...{PropertyPathMatch.cs => NamePathMatch.cs} | 30 +- .../PathExplainer.cs | 47 ++- .../PropertyPath.cs | 309 --------------- .../RelativeValuePolicy.cs | 18 +- .../RuleExplainer.cs | 75 ++-- DragoAnt.System.Text.Json.Observer/RuleSet.cs | 13 +- .../ShapeWalker.cs | 63 ++-- .../Strategies/MaskTag.cs | 103 ----- .../Strategies/NameMatcher.cs | 179 --------- .../Strategies/PropMatches.cs | 41 -- .../Strategies/PropMatchingStrategy.cs | 55 --- .../Strategies/Utf8MaskContext.cs | 54 --- .../Strategies/Utf8MaskStrategy.cs | 123 ------ .../{Strategies => }/StringMaskingStrategy.cs | 8 +- .../TagMasking.cs | 197 ++++++++-- .../DragoAnt.System.Text.Json.Observer.props | 7 + .../package.readme.md | 25 +- README.md | 114 +++--- skills/json-observer-http-logging/SKILL.md | 11 +- skills/json-observer-http-logging/examples.md | 19 +- skills/json-observer-http-logging/pitfalls.md | 5 +- skills/json-observer-http-logging/recipes.md | 5 +- skills/json-observer-masking/SKILL.md | 30 +- skills/json-observer-masking/examples.md | 114 +++--- .../migrating-from-1x.md | 27 +- skills/json-observer-masking/pitfalls.md | 45 +-- skills/json-observer-masking/recipes.md | 74 ++-- skills/json-observer-testing/SKILL.md | 2 +- skills/json-observer-testing/examples.md | 59 +-- skills/json-observer-testing/pitfalls.md | 26 +- skills/json-observer-testing/recipes.md | 23 +- 95 files changed, 2203 insertions(+), 2860 deletions(-) create mode 100644 DragoAnt.System.Text.Json.Observer/Builders/JsonAnyDepthBuilder.cs delete mode 100644 DragoAnt.System.Text.Json.Observer/Builders/JsonValuePolicyBuilder.cs delete mode 100644 DragoAnt.System.Text.Json.Observer/JsonObserverValuePolicies.cs delete mode 100644 DragoAnt.System.Text.Json.Observer/JsonObserveringEmptyContext.cs delete mode 100644 DragoAnt.System.Text.Json.Observer/JsonPathExplanation.cs create mode 100644 DragoAnt.System.Text.Json.Observer/JsonValueContext.cs create mode 100644 DragoAnt.System.Text.Json.Observer/JsonValuePolicy.cs create mode 100644 DragoAnt.System.Text.Json.Observer/JsonWalk.cs delete mode 100644 DragoAnt.System.Text.Json.Observer/MaskResult.cs rename DragoAnt.System.Text.Json.Observer/{PropertyPathMatch.cs => NamePathMatch.cs} (52%) delete mode 100644 DragoAnt.System.Text.Json.Observer/PropertyPath.cs delete mode 100644 DragoAnt.System.Text.Json.Observer/Strategies/MaskTag.cs delete mode 100644 DragoAnt.System.Text.Json.Observer/Strategies/NameMatcher.cs delete mode 100644 DragoAnt.System.Text.Json.Observer/Strategies/PropMatches.cs delete mode 100644 DragoAnt.System.Text.Json.Observer/Strategies/PropMatchingStrategy.cs delete mode 100644 DragoAnt.System.Text.Json.Observer/Strategies/Utf8MaskContext.cs delete mode 100644 DragoAnt.System.Text.Json.Observer/Strategies/Utf8MaskStrategy.cs rename DragoAnt.System.Text.Json.Observer/{Strategies => }/StringMaskingStrategy.cs (91%) create mode 100644 DragoAnt.System.Text.Json.Observer/buildTransitive/DragoAnt.System.Text.Json.Observer.props diff --git a/Directory.Build.props b/Directory.Build.props index 5e33a71..fa6bc0c 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -1,6 +1,6 @@ - net8.0;net9.0;net10.0 + net8.0;net10.0 Copyright (c) 2025 DragoAnt diff --git a/Directory.Build.targets b/Directory.Build.targets index 80622f2..9acf6c5 100644 --- a/Directory.Build.targets +++ b/Directory.Build.targets @@ -2,6 +2,10 @@ + + + diff --git a/Directory.Packages.props b/Directory.Packages.props index 3ab3882..a338385 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -36,6 +36,7 @@ + diff --git a/DragoAnt.System.Text.Json.Observer.Benchmarks/Baselines.cs b/DragoAnt.System.Text.Json.Observer.Benchmarks/Baselines.cs index 2bb46e7..554c23b 100644 --- a/DragoAnt.System.Text.Json.Observer.Benchmarks/Baselines.cs +++ b/DragoAnt.System.Text.Json.Observer.Benchmarks/Baselines.cs @@ -16,29 +16,29 @@ public static JsonObserver BuildObserver(int ruleCount = 8, string[]? names = nu { names ??= Payloads.SensitiveNames; var count = Math.Min(ruleCount, names.Length); - var policy = JsonObserverValuePolicies.Relative(b => + var policy = JsonValuePolicy.AnyDepth(b => { for (var i = 0; i < count; i++) { - b.Match(names[i]).MaskStr((_, _) => Mask); + b.Match(names[i]).Mask((_, _) => Mask, MaskNulls.Mask); } }, - JsonObserverValuePolicies.BlockList); + ValuePolicy.BlockList); return JsonObserver.Any(o => { }, a => { }, policy); } - public static JsonObserver BuildObserver(Action> extra) + public static JsonObserver BuildObserver(Action> extra) { - var policy = JsonObserverValuePolicies.Relative(b => + var policy = JsonValuePolicy.AnyDepth(b => { foreach (var name in Payloads.SensitiveNames) { - b.Match(name).MaskStr((_, _) => Mask); + b.Match(name).Mask((_, _) => Mask, MaskNulls.Mask); } extra(b); }, - JsonObserverValuePolicies.BlockList); + ValuePolicy.BlockList); return JsonObserver.Any(o => { }, a => { }, policy); } diff --git a/DragoAnt.System.Text.Json.Observer.Benchmarks/ExtractBenchmarks.cs b/DragoAnt.System.Text.Json.Observer.Benchmarks/ExtractBenchmarks.cs index 6c4179b..e85dd73 100644 --- a/DragoAnt.System.Text.Json.Observer.Benchmarks/ExtractBenchmarks.cs +++ b/DragoAnt.System.Text.Json.Observer.Benchmarks/ExtractBenchmarks.cs @@ -33,14 +33,14 @@ public static JsonObserver BuildExtractor() => public static JsonObserver BuildMaskAndExtract() { - var masking = JsonObserverValuePolicies.Relative(b => + var masking = JsonValuePolicy.AnyDepth(b => { foreach (var name in Payloads.SensitiveNames) { - b.Match(name).MaskStr((_, _) => Baselines.Mask); + b.Match(name).Mask((_, _) => Baselines.Mask, MaskNulls.Mask); } }, - JsonObserverValuePolicies.BlockList); + ValuePolicy.BlockList); return JsonObserver.Obj(b => b .Match("id").ReadStr((v, c) => c.Id = v) diff --git a/DragoAnt.System.Text.Json.Observer.Benchmarks/Harness/Probes.cs b/DragoAnt.System.Text.Json.Observer.Benchmarks/Harness/Probes.cs index 05becfd..aeee049 100644 --- a/DragoAnt.System.Text.Json.Observer.Benchmarks/Harness/Probes.cs +++ b/DragoAnt.System.Text.Json.Observer.Benchmarks/Harness/Probes.cs @@ -19,7 +19,7 @@ public static void Run(string? outputPath) ("MaskRawValue", Wrap(BuildObserver(raw: true))), ("MaskAny", Wrap(BuildMaskAnyObserver())), ("Bytes API + MaskAny", MaskBytes(BuildMaskAnyObserver())), - ("Bytes API + MaskAny, 4 KB output / 256 B value caps", MaskBytes(BuildMaskAnyObserver(), new JsonObserverOptions(MaxOutputBytes: 4096, MaxValueBytes: 256))), + ("Bytes API + MaskAny, 4 KB output / 256 B value caps", MaskBytes(BuildMaskAnyObserver(), new JsonObserverOptions { MaxOutputBytes = 4096, MaxValueBytes = 256 })), ("Read-mode mask+extract", MaskAndExtract), }; @@ -56,17 +56,17 @@ public static void Run(string? outputPath) private static Func Wrap(JsonObserver observer) => (json, options) => observer.Mask(json, ToOptions(options)); - private static JsonObserverOptions ToOptions(JsonReaderOptions options) => new(MaxDepth: options.MaxDepth == 0 ? 64 : options.MaxDepth); + private static JsonObserverOptions ToOptions(JsonReaderOptions options) => new() { MaxDepth = options.MaxDepth == 0 ? 64 : options.MaxDepth }; private static string? MaskAndExtract(string json, JsonReaderOptions options) { var observer = JsonObserver.Obj(b => b .Match("id").ReadInt((v, c) => c.Id = v) .Match("active").ReadBool((v, c) => c.Active = v), - JsonObserverValuePolicies.Relative(b => b - .Match("password").MaskStr((_, _) => "***") - .Match("pin").MaskStr((_, _) => "***"), - JsonObserverValuePolicies.BlockList)); + JsonValuePolicy.AnyDepth(b => b + .Match("password").Mask((_, _) => "***", MaskNulls.Mask) + .Match("pin").Mask((_, _) => "***", MaskNulls.Mask), + ValuePolicy.BlockList)); return observer.Mask(json, new ProbeContext(), ToOptions(options)); } @@ -84,27 +84,27 @@ private sealed class ProbeContext }; private static JsonObserver BuildMaskAnyObserver() => - JsonObserver.Any(_ => { }, _ => { }, JsonObserverValuePolicies.Relative( - b => b.Match(Strategies.PropMatches.OneOf("password", "pin")).MaskAny("***"), - JsonObserverValuePolicies.BlockList)); + JsonObserver.Any(_ => { }, _ => { }, JsonValuePolicy.AnyDepth( + b => b.Match(Names.OneOf("password", "pin")).Mask("***"), + ValuePolicy.BlockList)); private static JsonObserver BuildObserver(bool raw) { - var policy = JsonObserverValuePolicies.Relative(b => + var policy = JsonValuePolicy.AnyDepth(b => { foreach (var name in new[] { "password", "pin" }) { if (raw) { - b.Match(name).MaskRawValue((_, _) => "***"); + b.Match(name).Mask(MaskTag.Full); } else { - b.Match(name).MaskStr((_, _) => "***"); + b.Match(name).Mask((_, _) => "***", MaskNulls.Mask); } } }, - JsonObserverValuePolicies.BlockList); + ValuePolicy.BlockList); return JsonObserver.Any(o => { }, a => { }, policy); } diff --git a/DragoAnt.System.Text.Json.Observer.Docs.Tests/DocSnippetTests.cs b/DragoAnt.System.Text.Json.Observer.Docs.Tests/DocSnippetTests.cs index 598f5a6..2b1f0d1 100644 --- a/DragoAnt.System.Text.Json.Observer.Docs.Tests/DocSnippetTests.cs +++ b/DragoAnt.System.Text.Json.Observer.Docs.Tests/DocSnippetTests.cs @@ -139,7 +139,7 @@ private static string Run(Assembly assembly) return NewLines().Replace(printed.ToString(), "\n"); } - private const string GlobalUsings = "global using System;\nglobal using System.Collections.Generic;\nglobal using System.Linq;\nglobal using System.Threading.Tasks;\n"; + private const string GlobalUsings = "global using System;\nglobal using System.Collections.Generic;\nglobal using System.Linq;\nglobal using System.Threading.Tasks;\nglobal using DragoAnt.Observer;\n"; private static MetadataReference[] LoadReferences() { diff --git a/DragoAnt.System.Text.Json.Observer.Http.Tests/JsonBodyLoggingTests.cs b/DragoAnt.System.Text.Json.Observer.Http.Tests/JsonBodyLoggingTests.cs index 163f488..f816168 100644 --- a/DragoAnt.System.Text.Json.Observer.Http.Tests/JsonBodyLoggingTests.cs +++ b/DragoAnt.System.Text.Json.Observer.Http.Tests/JsonBodyLoggingTests.cs @@ -54,7 +54,7 @@ public async Task OnFailure_Success_LogsNothing() public async Task OnFailure_500_LogsMaskedBoth() { var sink = new TestSink(); - var masker = JsonObserver.Obj(rules => rules.Match("secret").MaskStr("*****")); + var masker = JsonObserver.Obj(rules => rules.Match("secret").Mask("*****", MaskNulls.Mask)); var provider = new TestMaskerProvider { GetMaskerFunc = (_, _) => masker }; var handler = CreateHandler( @@ -264,7 +264,7 @@ public async Task NoContext_UsesClientDefault() GetMaskerFunc = (type, _) => { passedTypes.Add(type); - return JsonObserver.Obj(rules => rules.Match("id").MaskStr("***")); + return JsonObserver.Obj(rules => rules.Match("id").Mask("***", MaskNulls.Mask)); } }; diff --git a/DragoAnt.System.Text.Json.Observer.Http.Tests/ReadmeSampleTests.cs b/DragoAnt.System.Text.Json.Observer.Http.Tests/ReadmeSampleTests.cs index 8e7a8dd..6d07ee4 100644 --- a/DragoAnt.System.Text.Json.Observer.Http.Tests/ReadmeSampleTests.cs +++ b/DragoAnt.System.Text.Json.Observer.Http.Tests/ReadmeSampleTests.cs @@ -50,7 +50,7 @@ public sealed record ChargeResponse(string Id); // README sample: masker provider public sealed class PaymentMaskers : IJsonBodyMaskerProvider { - private static readonly JsonObserver Charge = JsonObserver.Obj(rules => rules.Match("cardNumber").MaskStr("****")); + private static readonly JsonObserver Charge = JsonObserver.Obj(rules => rules.Match("cardNumber").Mask("****", MaskNulls.Mask)); public JsonObserver? GetMasker(Type? modelType, string clientName) => modelType == typeof(ChargeRequest) ? Charge : null; diff --git a/DragoAnt.System.Text.Json.Observer.Http/JsonBodyLogging.cs b/DragoAnt.System.Text.Json.Observer.Http/JsonBodyLogging.cs index 814e60c..7e98329 100644 --- a/DragoAnt.System.Text.Json.Observer.Http/JsonBodyLogging.cs +++ b/DragoAnt.System.Text.Json.Observer.Http/JsonBodyLogging.cs @@ -24,7 +24,7 @@ public static HttpRequestMessage WithBodyLogging( string? operation = null) { ArgumentNullException.ThrowIfNull(request); - request.Options.Set(Key, new JsonBodyLoggingContext(typeof(TRequest), typeof(TResponse), operation)); + request.Options.Set(Key, new JsonBodyLoggingContext { RequestType = typeof(TRequest), ResponseType = typeof(TResponse), Operation = operation }); return request; } @@ -43,7 +43,7 @@ public static HttpRequestMessage WithBodyLogging( string? operation = null) { ArgumentNullException.ThrowIfNull(request); - request.Options.Set(Key, new JsonBodyLoggingContext(requestType, responseType, operation)); + request.Options.Set(Key, new JsonBodyLoggingContext { RequestType = requestType, ResponseType = responseType, Operation = operation }); return request; } diff --git a/DragoAnt.System.Text.Json.Observer.Http/JsonBodyLoggingContext.cs b/DragoAnt.System.Text.Json.Observer.Http/JsonBodyLoggingContext.cs index e337b18..8d07f3c 100644 --- a/DragoAnt.System.Text.Json.Observer.Http/JsonBodyLoggingContext.cs +++ b/DragoAnt.System.Text.Json.Observer.Http/JsonBodyLoggingContext.cs @@ -4,7 +4,20 @@ namespace DragoAnt.System.Text.Json.Observer.Http; /// Per-request logging details, attached with and stored under /// . /// -/// Model type of the request body, passed to . -/// Model type of the response body, passed to . -/// Operation name written to . -public sealed record JsonBodyLoggingContext(Type? RequestType, Type? ResponseType, string? Operation = null); +public sealed record JsonBodyLoggingContext +{ + /// + /// Model type of the request body, passed to . + /// + public Type? RequestType { get; init; } + + /// + /// Model type of the response body, passed to . + /// + public Type? ResponseType { get; init; } + + /// + /// Operation name written to . + /// + public string? Operation { get; init; } +} diff --git a/DragoAnt.System.Text.Json.Observer.Http/JsonBodyLoggingHandler.cs b/DragoAnt.System.Text.Json.Observer.Http/JsonBodyLoggingHandler.cs index b66f259..ee3294e 100644 --- a/DragoAnt.System.Text.Json.Observer.Http/JsonBodyLoggingHandler.cs +++ b/DragoAnt.System.Text.Json.Observer.Http/JsonBodyLoggingHandler.cs @@ -293,7 +293,7 @@ private Body Render(ReadOnlySpan utf8, bool truncated, Type? modelType, Js return new Body("[body withheld]", JsonBodyStatus.Withheld, truncated); } - result = masker.Mask(utf8, output, new JsonObserverOptions(MaxOutputBytes: Math.Max(limit, 1))); + result = masker.Mask(utf8, output, new JsonObserverOptions { MaxOutputBytes = Math.Max(limit, 1) }); } catch (Exception ex) { @@ -307,7 +307,7 @@ private Body Render(ReadOnlySpan utf8, bool truncated, Type? modelType, Js { MaskStatus.Masked => truncated ? JsonBodyStatus.Truncated : JsonBodyStatus.Masked, MaskStatus.Truncated => JsonBodyStatus.Truncated, - MaskStatus.NotJson => JsonBodyStatus.NotJson, + MaskStatus.Unrecognized => JsonBodyStatus.NotJson, _ => JsonBodyStatus.Invalid, }; diff --git a/DragoAnt.System.Text.Json.Observer.Http/package.readme.md b/DragoAnt.System.Text.Json.Observer.Http/package.readme.md index 05720b5..d8cafa3 100644 --- a/DragoAnt.System.Text.Json.Observer.Http/package.readme.md +++ b/DragoAnt.System.Text.Json.Observer.Http/package.readme.md @@ -34,7 +34,7 @@ using var response = await httpClient.SendAsync(request, cancellationToken); public sealed class PaymentMaskers : IJsonBodyMaskerProvider { - private static readonly JsonObserver Charge = JsonObserver.Obj(rules => rules.Match("cardNumber").MaskStr("****")); + private static readonly JsonObserver Charge = JsonObserver.Obj(rules => rules.Match("cardNumber").Mask("****", MaskNulls.Mask)); // null logs the body as "[body withheld]". public JsonObserver? GetMasker(Type? modelType, string clientName) => @@ -47,7 +47,7 @@ Without an `IJsonBodyMaskerProvider` every value of an object or array body is m ## Behaviour - **Which calls** — `When` is `Never`, `OnFailure` (the default: a non-success status code, an exception or a cancellation) or `Always`. A canceled caller token and `HttpClient.Timeout` reach the handler as one token, so both are logged with the outcome `Canceled`. -- **Bodies** — at most `MaxBodyBytes` bytes of each body are read and masked (default 4096; 0 turns body logging off). `RequestBodyStatus` and `ResponseBodyStatus` say what was logged: `Masked`, `Truncated`, `Invalid`, `NotJson`, `Incomplete`, `Withheld`, `Skipped`, `NotBuffered`, `Raw` or `Failed`. When no JSON could be written, the body is a marker such as `[body not JSON]` or `[body not logged: text/plain]`. +- **Bodies** — at most `MaxBodyBytes` bytes of each body are read and masked (default 4096; 0 turns body logging off). `RequestBodyStatus` and `ResponseBodyStatus` say what was logged: `Masked`, `Truncated`, `Invalid`, `Unrecognized`, `Incomplete`, `Withheld`, `Skipped`, `NotBuffered`, `Raw` or `Failed`. When no JSON could be written, the body is a marker such as `[body not JSON]` or `[body not logged: text/plain]`. - **Skipped bodies** — a non-JSON media type, a charset other than UTF-8, or a `Content-Encoding` such as gzip. A request `StreamContent` whose stream cannot be read twice is logged as `[body not buffered]`. - **The caller is unaffected** — the response is returned as soon as its headers arrive, so `HttpCompletionOption.ResponseHeadersRead` keeps streaming; the body stays readable in full; the original exception of a failed call is rethrown with its stack trace; and a failure inside logging (a throwing sink, masker provider or logger) is reported through the handler's logger and never reaches the caller. - **When the entry is written** — the response body is captured in the background, so the entry is written once `MaxBodyBytes` bytes were captured, the body ended, the read failed or the caller disposed the response. diff --git a/DragoAnt.System.Text.Json.Observer.Skills.Tests/SkillSnippetTests.cs b/DragoAnt.System.Text.Json.Observer.Skills.Tests/SkillSnippetTests.cs index 8be4646..176231c 100644 --- a/DragoAnt.System.Text.Json.Observer.Skills.Tests/SkillSnippetTests.cs +++ b/DragoAnt.System.Text.Json.Observer.Skills.Tests/SkillSnippetTests.cs @@ -229,7 +229,7 @@ private static async Task Await(object? result) private const string ImplicitUsings = "global using System;\nglobal using System.Collections.Generic;\nglobal using System.IO;\nglobal using System.Linq;\n" + - "global using System.Net.Http;\nglobal using System.Threading;\nglobal using System.Threading.Tasks;\n"; + "global using System.Net.Http;\nglobal using System.Threading;\nglobal using System.Threading.Tasks;\nglobal using DragoAnt.Observer;\n"; private static MetadataReference[] LoadReferences() { diff --git a/DragoAnt.System.Text.Json.Observer.Tests.Shared/AllocationTests.cs b/DragoAnt.System.Text.Json.Observer.Tests.Shared/AllocationTests.cs index aff8ad3..3c40640 100644 --- a/DragoAnt.System.Text.Json.Observer.Tests.Shared/AllocationTests.cs +++ b/DragoAnt.System.Text.Json.Observer.Tests.Shared/AllocationTests.cs @@ -1,6 +1,7 @@ using System.Buffers; using System.Text; -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; namespace DragoAnt.System.Text.Json.Observer.Tests.Shared; @@ -9,16 +10,16 @@ public abstract class AllocationTests private static readonly JsonObserver Observer = JsonObserver.Any( _ => { }, _ => { }, - Relative(b => b.Match("password").MaskAny("***").Match("card", "number").MaskStr("***"), BlockList)); + AnyDepth(b => b.Match("password").Mask("***").Path("card", "number").Mask("***", MaskNulls.Mask), BlockList)); - private static readonly JsonObserver Reader = JsonObserver.Any( + private static readonly JsonObserver Reader = JsonObserver.Any( _ => { }, _ => { }, - JsonObserverValuePolicies.Relative( - b => b.Match("password").MaskAny("***"), - JsonObserverValuePolicies.BlockList)); + JsonValuePolicy.AnyDepth( + b => b.Match("password").Mask("***"), + ValuePolicy.BlockList)); - private static readonly JsonObserverOptions IgnoreNulls = new(IgnoreNulls: true); + private static readonly JsonObserverOptions IgnoreNulls = new() { IgnoreNulls = true }; public static TheoryData Budgets() { @@ -53,7 +54,7 @@ void Call() "span" => Observer.Mask(utf8, output), "sequence" => Observer.Mask(sequence, output), "ignore-nulls" => Observer.Mask(utf8, output, IgnoreNulls), - _ => Reader.Read(utf8, JsonObserveringEmptyContext.Instance), + _ => Reader.Read(utf8, NoContext.Instance), }; } @@ -82,8 +83,8 @@ void Call() [Fact] public void NestedCallOnSameThread_GetsItsOwnWriter() { - var inner = JsonObserver.Obj(Relative(b => b.Match("pin").MaskAny("#"), BlockList)); - var outer = JsonObserver.Obj(b => b.Match("payload").MaskStr((v, _) => inner.Mask(v)), BlockList); + var inner = JsonObserver.Obj(AnyDepth(b => b.Match("pin").Mask("#"), BlockList)); + var outer = JsonObserver.Obj(b => b.Match("payload").Mask((v, _) => inner.Mask(v), MaskNulls.Mask), BlockList); outer.Mask("""{"payload":"{\"pin\":1,\"x\":2}","y":3}""") .Should().Be("""{"payload":"{\"pin\":\"#\",\"x\":2}","y":3}"""); @@ -95,8 +96,8 @@ public void WriterSettingsChange_BetweenCalls_Respected() const string json = """{"a":"é<","password":"x"}"""; Observer.Mask(json).Should().Be("""{"a":"é<","password":"***"}"""); - Observer.Mask(json, new JsonObserverOptions(RelaxedEscaping: false)).Should().NotContain("é").And.Contain((char)92 + "u003C").And.EndWith(",\"password\":\"***\"}"); - Observer.Mask(json, new JsonObserverOptions(Indented: true)).Should().Contain(Environment.NewLine); + Observer.Mask(json, new JsonObserverOptions { RelaxedEscaping = false }).Should().NotContain("é").And.Contain((char)92 + "u003C").And.EndWith(",\"password\":\"***\"}"); + Observer.Mask(json, new JsonObserverOptions { Indented = true }).Should().Contain(Environment.NewLine); Observer.Mask(json).Should().Be("""{"a":"é<","password":"***"}"""); } diff --git a/DragoAnt.System.Text.Json.Observer.Tests.Shared/ApiSurfaceTests.cs b/DragoAnt.System.Text.Json.Observer.Tests.Shared/ApiSurfaceTests.cs index 31060f1..f2e9a75 100644 --- a/DragoAnt.System.Text.Json.Observer.Tests.Shared/ApiSurfaceTests.cs +++ b/DragoAnt.System.Text.Json.Observer.Tests.Shared/ApiSurfaceTests.cs @@ -1,5 +1,6 @@ -using DragoAnt.System.Text.Json.Observer.Strategies; -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; +using DragoAnt.Observer; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; namespace DragoAnt.System.Text.Json.Observer.Tests.Shared; @@ -10,19 +11,19 @@ public void ArrayBuilder_EveryRuleKind() { var context = new Values(); string Mask(Action> init, string json) - => JsonObserver.Array(init, JsonObserverValuePolicies.BlockList).Mask(json, context)!; + => JsonObserver.Array(init, ValuePolicy.BlockList).Mask(json, context)!; - Mask(a => a.MaskStr("s"), """[1,"x",{"a":1}]""").Should().Be("""["s","s","s"]"""); - Mask(a => a.MaskStr((v, _) => v + "!"), """["x"]""").Should().Be("""["x!"]"""); + Mask(a => a.Mask("s", MaskNulls.Mask), """[1,"x",{"a":1}]""").Should().Be("""["s","s","s"]"""); + Mask(a => a.Mask((v, _) => v + "!", MaskNulls.Mask), """["x"]""").Should().Be("""["x!"]"""); Mask(a => a.MaskInt((v, _) => $"{v}"), """[1,"x"]""").Should().Be("""["1",""]"""); Mask(a => a.MaskLong((v, _) => $"{v}"), """[2]""").Should().Be("""["2"]"""); Mask(a => a.MaskDecimal((v, _) => $"{v}"), """[3]""").Should().Be("""["3"]"""); Mask(a => a.MaskBool((v, _) => $"{v}"), """[true]""").Should().Be("""["True"]"""); - Mask(a => a.MaskAny((v, _) => v), """["a",null]""").Should().Be("""["a",null]"""); - Mask(a => a.MaskAny("*"), """[1]""").Should().Be("""["*"]"""); - Mask(a => a.MaskAny(MaskTag.Full), """[1]""").Should().Be("""["***"]"""); - Mask(a => a.MaskRawValue((v, _) => v), """["a\"b"]""").Should().Be("""["a\\\"b"]"""); - Mask(a => a.Array(i => i.MaskStr("n")), """[[1],2]""").Should().Be("""[["n"],2]"""); + Mask(a => a.Mask((v, _) => v), """["a",null]""").Should().Be("""["a",null]"""); + Mask(a => a.Mask("*"), """[1]""").Should().Be("""["*"]"""); + Mask(a => a.Mask(MaskTag.Full), """[1]""").Should().Be("""["***"]"""); + Mask(a => a.Mask((v, _) => v, MaskNulls.Mask), """["a\"b"]""").Should().Be("""["a\"b"]"""); + Mask(a => a.Array(i => i.Mask("n", MaskNulls.Mask)), """[[1],2]""").Should().Be("""[["n"],2]"""); Mask(a => a.Unmasked(), """[1,"x"]""").Should().Be("""[1,"x"]"""); Mask(a => a.ReadStr((v, c) => c.Str = v), """["s"]""").Should().Be("""["s"]"""); @@ -39,7 +40,7 @@ string Mask(Action> init, string json) public void RelativeBuilder_EveryReadRule_KeepsValues() { var context = new Values(); - var observer = JsonObserver.Obj(JsonObserverValuePolicies.Relative(b => b + var observer = JsonObserver.Obj(JsonValuePolicy.AnyDepth(b => b .Match("s").ReadStr((v, c) => c.Str = v).Unmasked() .Match("i").ReadInt((v, c) => c.Int = v).Unmasked() .Match("l").ReadLong((v, c) => c.Long = v).Unmasked() @@ -50,9 +51,9 @@ public void RelativeBuilder_EveryReadRule_KeepsValues() .Match("n").MaskLong((_, _) => "l") .Match("o").MaskDecimal((_, _) => "d") .Match("p").MaskBool((_, _) => "b") - .Match("q").MaskRawValue((_, _) => "r") + .Match("q").Mask((_, _) => "r", MaskNulls.Mask) .Match("u").Unmasked(), - JsonObserverValuePolicies.NullList)); + ValuePolicy.NullList)); observer.Mask("""{"x":{"s":"a","i":1,"l":2,"d":3.5,"b":true,"r":"z","m":1,"n":2,"o":3,"p":true,"q":"q","u":7,"v":8}}""", context) .Should().Be("""{"x":{"s":"a","i":1,"l":2,"d":3.5,"b":true,"r":"z","m":"i","n":"l","o":"d","p":"b","q":"r","u":7,"v":null}}"""); @@ -61,7 +62,7 @@ public void RelativeBuilder_EveryReadRule_KeepsValues() [Fact] public void ObjBuilder_RawAndAnyFunction() - => JsonObserver.Obj(b => b.Match("a").MaskRawValue((v, _) => "<" + v + ">").Match("b").MaskAny((v, _) => v + "?"), BlockList) + => JsonObserver.Obj(b => b.Match("a").Mask((v, _) => "<" + v + ">", MaskNulls.Mask).Match("b").Mask((v, _) => v + "?"), BlockList) .Mask("""{"a":1.0,"b":false}""").Should().Be("""{"a":"<1.0>","b":"false?"}"""); [Fact] @@ -69,15 +70,15 @@ public void Read_WithMaskingRules_WritesNothingReadsValues() { var context = new Values(); var observer = JsonObserver.Obj(b => b - .Match("p").MaskStr("***") - .Match("t").MaskAny(MaskTag.Last4) + .Match("p").Mask("***", MaskNulls.Mask) + .Match("t").Mask(MaskTag.Last4) .Match("n").MaskInt((_, _) => "*") .Match("s").ReadStr((v, c) => c.Str = v) - .Match("o").Obj(o => o.Match("i").ReadInt((v, c) => c.Int = v)), JsonObserverValuePolicies.BlockList); + .Match("o").Obj(o => o.Match("i").ReadInt((v, c) => c.Int = v)), ValuePolicy.BlockList); var result = observer.Read("""{"p":"secret","t":{"a":1},"n":5,"x":true,"y":null,"s":"v","o":{"i":3},"z":[1,"a",false,1.5]}""", context); - result.Should().Be(new MaskResult(MaskStatus.Masked, 0, -1)); + result.Should().Be(new MaskResult { Status = MaskStatus.Masked, BytesWritten = 0, FailedAtByte = -1 }); context.Str.Should().Be("v"); context.Int.Should().Be(3); } @@ -85,8 +86,7 @@ public void Read_WithMaskingRules_WritesNothingReadsValues() [Fact] public void CustomRule_EveryWriterMethod() { - var observer = JsonObserver.Obj(b => b.Match("a").MaskValue((ref Utf8JsonReader _, JsonWriter writer, JsonObserveringEmptyContext _, ref PropertyPath _) => - { + var observer = JsonObserver.Obj(b => b.Match("a").MaskValue((ref JsonValueContext __c) => { var writer = __c.Writer; writer.WriteStartArray(); writer.WriteNumberValue(1L); writer.WriteNumberValue(2.5m); @@ -104,13 +104,13 @@ public void CustomRule_EveryWriterMethod() }), BlockList); observer.Mask("""{"a":0}""").Should().Be("""{"a":[1,2.5,true,null,null,3e1,{"k":"v","j":"w"}]}"""); - observer.Mask("""{"a":0}""", new JsonObserverOptions(IgnoreNulls: true)).Should().Be("""{"a":[1,2.5,true,3e1,{"k":"v","j":"w"}]}"""); + observer.Mask("""{"a":0}""", new JsonObserverOptions { IgnoreNulls = true }).Should().Be("""{"a":[1,2.5,true,3e1,{"k":"v","j":"w"}]}"""); } [Fact] public void IgnoreNulls_EveryValueType() - => JsonObserver.Obj(Relative(b => b.Match("m").MaskStr((_, _) => null), BlockList)) - .Mask("""{"s":"x","n":1.5,"b":false,"z":null,"m":"gone","o":{"a":[true,null,{"q":null}],"e":"é"}}""", new JsonObserverOptions(IgnoreNulls: true)) + => JsonObserver.Obj(AnyDepth(b => b.Match("m").Mask((_, _) => null, MaskNulls.Mask), BlockList)) + .Mask("""{"s":"x","n":1.5,"b":false,"z":null,"m":"gone","o":{"a":[true,null,{"q":null}],"e":"é"}}""", new JsonObserverOptions { IgnoreNulls = true }) .Should().Be("""{"s":"x","n":1.5,"b":false,"o":{"a":[true],"e":"é"}}"""); [Fact] @@ -120,36 +120,30 @@ public void IgnoreNulls_DeepNesting_GrowsBuffers() var name = new string('n', 300); var json = string.Concat(Enumerable.Repeat($$"""{"{{name}}":""", depth)) + "1" + new string('}', depth); - JsonObserver.Obj(BlockList).Mask(json, new JsonObserverOptions(IgnoreNulls: true)).Should().Be(json); + JsonObserver.Obj(BlockList).Mask(json, new JsonObserverOptions { IgnoreNulls = true }).Should().Be(json); } -#pragma warning disable CS0618 [Fact] - public void LegacyAllowList_EveryValueType() - => JsonObserver.Obj(LegacyAllowList).Mask("""{"s":"x","n":1,"t":true,"f":false,"z":null}""") - .Should().Be("""{"s":"#str#*****","n":"#number#*****","t":true,"f":false,"z":null}"""); -#pragma warning restore CS0618 - - [Fact] - public void PropMatchingStrategy_ConvertsToFunction() + public void NameMatch_TestsDecodedNames() { - Func exact = (PropMatchingStrategy)"Pin"; - Func custom = (PropMatchingStrategy)(Func)(n => n == "x"); - - exact("PIN").Should().BeTrue(); - exact("pins").Should().BeFalse(); - custom("x").Should().BeTrue(); - ((Func)default(PropMatchingStrategy))("a").Should().BeFalse(); + NameMatch exact = "Pin"; + var custom = new NameMatch(n => n == "x"); + + exact.IsMatch("PIN", StringComparison.OrdinalIgnoreCase).Should().BeTrue(); + exact.IsMatch("PIN", StringComparison.Ordinal).Should().BeFalse(); + exact.IsMatch("pins", StringComparison.OrdinalIgnoreCase).Should().BeFalse(); + custom.IsMatch("x", StringComparison.Ordinal).Should().BeTrue(); + default(NameMatch).IsMatch("a", StringComparison.Ordinal).Should().BeFalse(); } [Fact] public void NonAsciiPatterns_FallBackToStringComparison() { - var observer = JsonObserver.Obj(Relative(b => b - .Match(PropMatches.StartsWith("пар")).MaskAny("1") - .Match(PropMatches.EndsWith("оль")).MaskAny("2") - .Match(PropMatches.Contains("ём")).MaskAny("3") - .Match("ключ").MaskAny("4"), + var observer = JsonObserver.Obj(AnyDepth(b => b + .Match(Names.StartsWith("пар")).Mask("1") + .Match(Names.EndsWith("оль")).Mask("2") + .Match(Names.Contains("ём")).Mask("3") + .Match("ключ").Mask("4"), BlockList)); observer.Mask("""{"пароль":"a","кроль":"b","объём":"c","КЛЮЧ":"d","other":"e"}""") @@ -159,7 +153,7 @@ public void NonAsciiPatterns_FallBackToStringComparison() [Fact] public void Match_WithoutNames_Throws() { - var build = () => JsonObserver.Obj(b => b.Match().MaskAny("*")); + var build = () => JsonObserver.Obj(b => b.Path().Mask("*")); build.Should().Throw(); } diff --git a/DragoAnt.System.Text.Json.Observer.Tests.Shared/BytesApiTests.cs b/DragoAnt.System.Text.Json.Observer.Tests.Shared/BytesApiTests.cs index 19af04c..a7f6cbe 100644 --- a/DragoAnt.System.Text.Json.Observer.Tests.Shared/BytesApiTests.cs +++ b/DragoAnt.System.Text.Json.Observer.Tests.Shared/BytesApiTests.cs @@ -1,6 +1,7 @@ using System.Buffers; using System.Text; -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; namespace DragoAnt.System.Text.Json.Observer.Tests.Shared; @@ -11,7 +12,7 @@ public abstract class BytesApiTests internal static readonly JsonObserver Observer = JsonObserver.Any( _ => { }, _ => { }, - Relative(b => b.Match("password").MaskAny("***").Match("pin").MaskAny("***"), BlockList)); + AnyDepth(b => b.Match("password").Mask("***").Match("pin").Mask("***"), BlockList)); internal static readonly string[] Payloads = [ @@ -65,7 +66,7 @@ public void UncompletedJson_MasksSensitiveAndSynthesizesClosingBraces() var observer = JsonObserver.Any( _ => { }, _ => { }, - Relative(b => b.Match("DriverLicense").MaskAny("***"), BlockList)); + AnyDepth(b => b.Match("DriverLicense").Mask("***"), BlockList)); var uncompletedJson = """ { @@ -89,7 +90,7 @@ public void UncompletedJson_IncompleteFieldName_RollsBackAndClosesContainers() var observer = JsonObserver.Any( _ => { }, _ => { }, - Relative(b => b.Match("DriverLicense").MaskAny("***"), BlockList)); + AnyDepth(b => b.Match("DriverLicense").Mask("***"), BlockList)); // Cut off mid-field name: "dr var uncompletedJson = """ @@ -114,7 +115,7 @@ public void UncompletedJson_IncompleteFieldValue_NeverLeaksAndClosesContainers() var observer = JsonObserver.Any( _ => { }, _ => { }, - Relative(b => b.Match("DriverLicense").MaskAny("***"), BlockList)); + AnyDepth(b => b.Match("DriverLicense").Mask("***"), BlockList)); // Cut off mid-field value: "vvv without closing quote var uncompletedJson = """ @@ -154,14 +155,14 @@ public void RootPrimitive_NotJson(string json) { var (result, output) = Mask(Observer, json); - result.Status.Should().Be(MaskStatus.NotJson); + result.Status.Should().Be(MaskStatus.Unrecognized); output.Should().BeEmpty(); } [Theory] [InlineData("")] [InlineData(" ")] - public void Empty_NotJson(string json) => Mask(Observer, json).Result.Status.Should().Be(MaskStatus.NotJson); + public void Empty_NotJson(string json) => Mask(Observer, json).Result.Status.Should().Be(MaskStatus.Unrecognized); [Fact] public void CommentWithoutReaderOption_AutoAllowed() @@ -184,7 +185,7 @@ public void CommentWithoutReaderOption_AutoAllowed() [InlineData(21, "{\"a\":{\"b\":[1,2,3,4]}}")] public void MaxOutputBytes_ClosesContainers(int max, string expected) { - var (result, output) = Mask(Observer, """{"a":{"b":[1,2,3,4,5,6,7,8,9]},"c":1}""", new JsonObserverOptions(MaxOutputBytes: max)); + var (result, output) = Mask(Observer, """{"a":{"b":[1,2,3,4,5,6,7,8,9]},"c":1}""", new JsonObserverOptions { MaxOutputBytes = max }); result.Status.Should().Be(MaskStatus.Truncated); output.Should().Be(expected); @@ -194,7 +195,7 @@ public void MaxOutputBytes_ClosesContainers(int max, string expected) [Fact] public void MaxValueBytes_TruncatesLongString() { - var (result, output) = Mask(Observer, """{"note":"abcdefghij","ru":"ИванИван"}""", new JsonObserverOptions(MaxValueBytes: 5)); + var (result, output) = Mask(Observer, """{"note":"abcdefghij","ru":"ИванИван"}""", new JsonObserverOptions { MaxValueBytes = 5 }); result.Status.Should().Be(MaskStatus.Truncated); var root = JsonDocument.Parse(output).RootElement; @@ -206,7 +207,7 @@ public void MaxValueBytes_TruncatesLongString() public void RelaxedEscaping_KeepsTextReadable() { Mask(Observer, """{"name":"Иван &"}""").Output.Should().Be("""{"name":"Иван &"}"""); - Mask(Observer, """{"name":"Иван"}""", new JsonObserverOptions(RelaxedEscaping: false)).Output.Should().NotContain("Иван"); + Mask(Observer, """{"name":"Иван"}""", new JsonObserverOptions { RelaxedEscaping = false }).Output.Should().NotContain("Иван"); } [Fact] @@ -219,7 +220,7 @@ public void NeverThrows_Fuzz() var payload = payloads[i % payloads.Length]; var cut = payload.AsSpan(0, random.Next(payload.Length + 1)); - var (result, output) = Mask(Observer, cut, new JsonObserverOptions(MaxOutputBytes: random.Next(2) == 0 ? int.MaxValue : random.Next(256))); + var (result, output) = Mask(Observer, cut, new JsonObserverOptions { MaxOutputBytes = random.Next(2) == 0 ? int.MaxValue : random.Next(256) }); output.Should().NotContain(Secret); if (result.BytesWritten > 0) diff --git a/DragoAnt.System.Text.Json.Observer.Tests.Shared/CaseSensitivityTests.cs b/DragoAnt.System.Text.Json.Observer.Tests.Shared/CaseSensitivityTests.cs index 3f0f490..d99b7ac 100644 --- a/DragoAnt.System.Text.Json.Observer.Tests.Shared/CaseSensitivityTests.cs +++ b/DragoAnt.System.Text.Json.Observer.Tests.Shared/CaseSensitivityTests.cs @@ -1,22 +1,23 @@ using System.Text; using System.Text.Json.Serialization.Metadata; -using DragoAnt.System.Text.Json.Observer.Strategies; -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; +using DragoAnt.Observer; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; namespace DragoAnt.System.Text.Json.Observer.Tests.Shared; public abstract class CaseSensitivityTests { - private static readonly JsonObserverOptions Exact = new(PropertyNameCaseInsensitive: false); + private static readonly JsonObserverOptions Exact = new() { NameCaseInsensitive = false }; - private static readonly JsonObserver Rules = JsonObserver.Obj(Relative(b => b - .Match("password").MaskAny("1") - .Match(PropMatches.StartsWith("tok")).MaskAny("2") - .Match(PropMatches.EndsWith("Card")).MaskAny("3") - .Match(PropMatches.Contains("mail")).MaskAny("4") - .Match(PropMatches.OneOf("pin", "cvv")).MaskAny("5") - .Match("ключ").MaskAny("6") - .Match(PropMatches.StartsWith("пар")).MaskAny("7"), + private static readonly JsonObserver Rules = JsonObserver.Obj(AnyDepth(b => b + .Match("password").Mask("1") + .Match(Names.StartsWith("tok")).Mask("2") + .Match(Names.EndsWith("Card")).Mask("3") + .Match(Names.Contains("mail")).Mask("4") + .Match(Names.OneOf("pin", "cvv")).Mask("5") + .Match("ключ").Mask("6") + .Match(Names.StartsWith("пар")).Mask("7"), BlockList)); private const string Payload = """{"password":"a","Password":"b","token":"c","Token":"d","myCard":"e","mycard":"f","email":"g","eMail":"h","pin":"i","PIN":"j","ключ":"k","КЛЮЧ":"l","пароль":"m","Пароль":"n"}"""; @@ -41,9 +42,8 @@ public void AbsoluteRules_CaseSensitive_UnderAllowList_MaskUnmatchedCase() => public void CustomRule_SeesTheCallsMatchingMode() { var modes = new List(); - var observer = JsonObserver.Obj(b => b.Match("a").MaskValue((ref Utf8JsonReader _, JsonWriter writer, JsonObserveringEmptyContext _, ref PropertyPath path) => - { - modes.Add(path.PropertyNameCaseInsensitive); + var observer = JsonObserver.Obj(b => b.Match("a").MaskValue((ref JsonValueContext __c) => { var writer = __c.Writer; var path = __c.Path; + modes.Add(__c.Options.NameCaseInsensitive); writer.WriteNullValue(); }), BlockList); @@ -64,9 +64,9 @@ public void Shape_FollowsSerializerOptions() JsonObserver.FromShape(shape, JsonShapeOptions.FromSerializerOptions(general)).Mask(json) .Should().Be("""{"Name":"a","name":"***","Secret":"***","secret":"***"}"""); JsonObserver.FromShape(shape).Mask(json, Exact).Should().Be("""{"Name":"a","name":"***","Secret":"***","secret":"***"}"""); - JsonObserver.FromShape(shape, new JsonShapeOptions(PropertyNameCaseInsensitive: true)).Mask(json, Exact) + JsonObserver.FromShape(shape, new JsonShapeOptions { NameCaseInsensitive = true }).Mask(json, Exact) .Should().Be("""{"Name":"a","name":"b","Secret":"***","secret":"***"}"""); - JsonShapeOptions.FromSerializerOptions(new JsonSerializerOptions(JsonSerializerDefaults.Web)).PropertyNameCaseInsensitive.Should().BeTrue(); + JsonShapeOptions.FromSerializerOptions(new JsonSerializerOptions(JsonSerializerDefaults.Web)).NameCaseInsensitive.Should().BeTrue(); } [Fact] diff --git a/DragoAnt.System.Text.Json.Observer.Tests.Shared/DefaultPolicyTests.cs b/DragoAnt.System.Text.Json.Observer.Tests.Shared/DefaultPolicyTests.cs index c5712ef..1c81391 100644 --- a/DragoAnt.System.Text.Json.Observer.Tests.Shared/DefaultPolicyTests.cs +++ b/DragoAnt.System.Text.Json.Observer.Tests.Shared/DefaultPolicyTests.cs @@ -1,4 +1,5 @@ -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; namespace DragoAnt.System.Text.Json.Observer.Tests.Shared; @@ -9,7 +10,7 @@ public abstract class DefaultPolicyTests [Fact] public void SharedNestedRule_TwoParentsDifferentDefaults_EachUsesOwn() { - var shared = JsonObserverItem.Obj(b => b.Match("pin").MaskStr((_, _) => "***"), null).Delegate; + Action> shared = b => b.Match("pin").Mask((_, _) => "***", MaskNulls.Mask); var blockList = JsonObserver.Obj(b => b.Match("a").Obj(shared), BlockList); var nullList = JsonObserver.Obj(b => b.Match("a").Obj(shared), NullList); @@ -27,7 +28,7 @@ public void ConcurrentFirstCalls_SameOutput() for (var round = 0; round < 20; round++) { var observer = JsonObserver.Obj( - b => b.Match("a").Obj(a => a.Match("pin").MaskStr((_, _) => "***")), + b => b.Match("a").Obj(a => a.Match("pin").Mask((_, _) => "***", MaskNulls.Mask)), BlockList); using var start = new ManualResetEventSlim(); var results = new string?[8]; diff --git a/DragoAnt.System.Text.Json.Observer.Tests.Shared/ExplainTests.cs b/DragoAnt.System.Text.Json.Observer.Tests.Shared/ExplainTests.cs index 91d3127..39f3beb 100644 --- a/DragoAnt.System.Text.Json.Observer.Tests.Shared/ExplainTests.cs +++ b/DragoAnt.System.Text.Json.Observer.Tests.Shared/ExplainTests.cs @@ -1,5 +1,6 @@ -using DragoAnt.System.Text.Json.Observer.Strategies; -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; +using DragoAnt.Observer; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; namespace DragoAnt.System.Text.Json.Observer.Tests.Shared; @@ -7,77 +8,76 @@ public abstract class ExplainTests { private static readonly JsonObserver Lines = JsonObserver.Obj( root => root - .Match("lines").Array(l => l.Obj(x => x.Match("qty").MaskAny("***").Match("note").ReadStr((_, _) => { }))) + .Match("lines").Array(l => l.Obj(x => x.Match("qty").Mask("***").Match("note").ReadStr((_, _) => { }))) .Match("id").Unmasked() - .Match("custom").MaskValue((ref Utf8JsonReader _, JsonWriter w, JsonObserveringEmptyContext _, ref PropertyPath _) => w.WriteNullValue()), + .Match("custom").MaskValue((ref JsonValueContext __c) => { var w = __c.Writer; w.WriteNullValue(); }), BlockList); [Fact] public void AbsoluteNestedRule_NamesTheChain() { - var explanation = Lines.Explain("lines[0].qty", JsonTokenType.Number); + var explanation = Lines.Explain("lines[0].qty", ValueKind.Number); - explanation.Should().BeEquivalentTo(new JsonPathExplanation( - "lines[0].qty", - JsonPathOutcome.Masked, - """Match("lines") > object item > Match("qty")""", - """MaskAny("***")""", + explanation.Should().BeEquivalentTo(new PathExplanation + { + Path = "lines[0].qty", + Outcome = PathOutcome.Masked, + Rule = """Match("lines") > object item > Match("qty")""", + Action = """Mask("***")""", + Steps = [ """lines: Match("lines") → Array(...)""", "lines[0]: object item → Obj(...)", - """lines[0].qty: Match("qty") → MaskAny("***")""", - ])); - explanation.ToString().Should().Be("""lines[0].qty: Masked by Match("lines") > object item > Match("qty") → MaskAny("***")"""); + """lines[0].qty: Match("qty") → Mask("***")""", + ], + }); + explanation.ToString().Should().Be("""lines[0].qty: Masked by Match("lines") > object item > Match("qty") → Mask("***")"""); } [Fact] public void RuleKinds_Outcomes() { - Lines.Explain("lines[3].sku").Should().Match(e => e.Outcome == JsonPathOutcome.Unchanged && e.Rule == "default policy BlockList"); - Lines.Explain("lines[0].note").Outcome.Should().Be(JsonPathOutcome.Read); - Lines.Explain("id", JsonTokenType.Number).Outcome.Should().Be(JsonPathOutcome.Unchanged); - Lines.Explain("custom").Outcome.Should().Be(JsonPathOutcome.Custom); - Lines.Explain("lines[0].qty", JsonTokenType.Null).Should().Match(e => e.Outcome == JsonPathOutcome.Unchanged && e.Action.EndsWith("keeps null")); - Lines.Explain("other.deep", JsonTokenType.StartObject).Should().Match(e => e.Outcome == JsonPathOutcome.Unchanged && e.Action.Contains("descended")); - Lines.Explain("[0].id").Outcome.Should().Be(JsonPathOutcome.Invalid); + Lines.Explain("lines[3].sku").Should().Match(e => e.Outcome == PathOutcome.Unchanged && e.Rule == "default policy BlockList"); + Lines.Explain("lines[0].note").Outcome.Should().Be(PathOutcome.Read); + Lines.Explain("id", ValueKind.Number).Outcome.Should().Be(PathOutcome.Unchanged); + Lines.Explain("custom").Outcome.Should().Be(PathOutcome.Custom); + Lines.Explain("lines[0].qty", ValueKind.Null).Should().Match(e => e.Outcome == PathOutcome.Unchanged && e.Action.EndsWith("keeps null")); + Lines.Explain("other.deep", ValueKind.Object).Should().Match(e => e.Outcome == PathOutcome.Unchanged && e.Action.Contains("descended")); + Lines.Explain("[0].id").Outcome.Should().Be(PathOutcome.Invalid); Lines.Explain("$").Rule.Should().Be("root"); } [Fact] public void RelativeRules_AndDefaults() { - var observer = JsonObserver.Obj(Relative(rules => rules - .Match(PropMatches.EndsWith("card"), "saved", "id").MaskStr("***") - .Match("card").MaskAny(MaskTag.Last4) - .Match(PropMatches.Contains("email")).MaskStr((v, _) => v), + var observer = JsonObserver.Obj(AnyDepth(rules => rules + .Path(Names.EndsWith("card"), "saved", "id").Mask("***", MaskNulls.Mask) + .Match("card").Mask(MaskTag.Last4) + .Match(Names.Contains("email")).Mask((v, _) => v, MaskNulls.Mask), AllowList)); - observer.Explain("s.MY_card.saved.id").Should().Match(e => - e.Outcome == JsonPathOutcome.Masked && e.Rule == """relative Match(EndsWith("card"), "saved", "id")""" && e.Action == """MaskStr("***")"""); - observer.Explain("a.card.number").Should().Match(e => - e.Rule == """relative Match("card")""" && e.Action == "MaskAny(MaskTag.Last4) on the whole object"); - observer.Explain("c.workEmail").Action.Should().Be("MaskStr(function)"); - observer.Explain("c.tier").Should().Match(e => e.Rule == "default policy AllowList" && e.Action == "writes \"***\""); - observer.Explain("c.tier", JsonTokenType.Null).Action.Should().Be("keeps null"); - observer.Explain("c.Tier", options: new JsonObserverOptions(PropertyNameCaseInsensitive: false)).Rule.Should().Be("default policy AllowList"); - observer.Explain("c.WORKEMAIL", options: new JsonObserverOptions(PropertyNameCaseInsensitive: false)).Rule.Should().Be("default policy AllowList"); - observer.Explain("c.WORKEMAIL").Rule.Should().StartWith("relative"); + observer.Explain("s.MY_card.saved.id").Should().Match(e => + e.Outcome == PathOutcome.Masked && e.Rule == """AnyDepth Path(EndsWith("card"), "saved", "id")""" && e.Action == """Mask("***", MaskNulls.Mask)"""); + observer.Explain("a.card.number").Should().Match(e => + e.Rule == """AnyDepth Match("card")""" && e.Action == "Mask(MaskTag.Last4) on the whole object"); + observer.Explain("c.workEmail").Action.Should().Be("Mask(function, MaskNulls.Mask)"); + observer.Explain("c.tier").Should().Match(e => e.Rule == "default policy AllowList" && e.Action == "writes \"***\""); + observer.Explain("c.tier", ValueKind.Null).Action.Should().Be("keeps null"); + observer.Explain("c.Tier", options: new JsonObserverOptions { NameCaseInsensitive = false }).Rule.Should().Be("default policy AllowList"); + observer.Explain("c.WORKEMAIL", options: new JsonObserverOptions { NameCaseInsensitive = false }).Rule.Should().Be("default policy AllowList"); + observer.Explain("c.WORKEMAIL").Rule.Should().StartWith("AnyDepth"); } [Fact] public void DefaultPolicies_Named() { -#pragma warning disable CS0618 - JsonObserver.Obj(LegacyAllowList).Explain("b", JsonTokenType.True).Outcome.Should().Be(JsonPathOutcome.Unchanged); - JsonObserver.Obj(LegacyAllowList).Explain("s").Action.Should().Be("writes \"#str#*****\""); - JsonObserver.Obj(LegacyAllowList).Explain("n", JsonTokenType.Number).Action.Should().Be("writes \"#number#*****\""); -#pragma warning restore CS0618 + JsonObserver.Obj(Tagged(MaskTag.Hash)).Explain("s").Should().Match(e => e.Outcome == PathOutcome.Masked && e.Rule == "default policy Tagged(Hash)" && e.Action == "Mask(MaskTag.Hash)"); JsonObserver.Obj(NullList).Explain("s").Action.Should().Be("writes null"); - JsonObserver.Obj((ref Utf8JsonReader _, JsonWriter w, JsonObserveringEmptyContext _, ref PropertyPath _) => w.WriteNullValue()) - .Explain("s").Should().Match(e => e.Outcome == JsonPathOutcome.Custom && e.Rule == "custom default policy"); - JsonObserver.Array(BlockList).Explain("[2]", JsonTokenType.Number).Outcome.Should().Be(JsonPathOutcome.Unchanged); - JsonObserver.Array(a => a.MaskAny("x")).Explain("[0]").Rule.Should().Be("any item"); - JsonObserver.Array(BlockList).Explain("a").Outcome.Should().Be(JsonPathOutcome.Invalid); + JsonObserver.Obj(JsonValuePolicy.Custom((ref JsonValueContext __c) => { var w = __c.Writer; w.WriteNullValue(); })) + .Explain("s").Should().Match(e => e.Outcome == PathOutcome.Custom && e.Rule == "custom default policy"); + JsonObserver.Array(BlockList).Explain("[2]", ValueKind.Number).Outcome.Should().Be(PathOutcome.Unchanged); + JsonObserver.Array(a => a.Mask("x")).Explain("[0]").Rule.Should().Be("any item"); + JsonObserver.Array(BlockList).Explain("a").Outcome.Should().Be(PathOutcome.Invalid); } [Fact] @@ -85,24 +85,24 @@ public void Shape_Explained() { var observer = JsonShapeTests.Observer(); - observer.Explain("name").Should().Match(e => e.Outcome == JsonPathOutcome.Unchanged && e.Rule == "shape Scalar"); - observer.Explain("card").Should().Match(e => e.Outcome == JsonPathOutcome.Masked && e.Action == "MaskTag.Last4"); - observer.Explain("password", JsonTokenType.Null).Action.Should().Be("keeps null"); - observer.Explain("orders[1].secretCode").Should().Match(e => e.Outcome == JsonPathOutcome.Masked && e.Steps.Count == 4); + observer.Explain("name").Should().Match(e => e.Outcome == PathOutcome.Unchanged && e.Rule == "shape Scalar"); + observer.Explain("card").Should().Match(e => e.Outcome == PathOutcome.Masked && e.Action == "MaskTag.Last4"); + observer.Explain("password", ValueKind.Null).Action.Should().Be("keeps null"); + observer.Explain("orders[1].secretCode").Should().Match(e => e.Outcome == PathOutcome.Masked && e.Steps.Count == 4); observer.Explain("orders[1].extra").Rule.Should().Be("unknown member (MaskWhole)"); - observer.Explain("byCode.K1.sku").Outcome.Should().Be(JsonPathOutcome.Unchanged); - observer.Explain("unknown.deep").Should().Match(e => e.Outcome == JsonPathOutcome.Masked && e.Rule.Contains("Opaque")); + observer.Explain("byCode.K1.sku").Outcome.Should().Be(PathOutcome.Unchanged); + observer.Explain("unknown.deep").Should().Match(e => e.Outcome == PathOutcome.Masked && e.Rule.Contains("Opaque")); observer.Explain("name.first").Rule.Should().Contain("where the path has an object"); observer.Explain("extra").Rule.Should().Be("shape Opaque"); - observer.Explain("orders", JsonTokenType.StartArray).Outcome.Should().Be(JsonPathOutcome.Unchanged); - observer.Explain("NAME").Outcome.Should().Be(JsonPathOutcome.Unchanged); - observer.Explain("NAME", options: new JsonObserverOptions(PropertyNameCaseInsensitive: false)).Outcome.Should().Be(JsonPathOutcome.Masked); - JsonShapeTests.Observer(shapeOptions: new JsonShapeOptions(UnknownMemberPolicy.Descend)).Explain("unknown.deep.x") - .Should().Match(e => e.Outcome == JsonPathOutcome.Masked && e.Rule == "unknown member (Descend)"); - JsonShapeTests.Observer(shapeOptions: new JsonShapeOptions(UnknownMemberPolicy.PassThrough)).Explain("unknown.deep.x") - .Outcome.Should().Be(JsonPathOutcome.Unchanged); - JsonShapeTests.Observer(shapeOptions: new JsonShapeOptions(KeepNulls: false)).Explain("password", JsonTokenType.Null) - .Outcome.Should().Be(JsonPathOutcome.Masked); + observer.Explain("orders", ValueKind.Array).Outcome.Should().Be(PathOutcome.Unchanged); + observer.Explain("NAME").Outcome.Should().Be(PathOutcome.Unchanged); + observer.Explain("NAME", options: new JsonObserverOptions { NameCaseInsensitive = false }).Outcome.Should().Be(PathOutcome.Masked); + JsonShapeTests.Observer(shapeOptions: new JsonShapeOptions { Unknown = UnknownMemberPolicy.Descend }).Explain("unknown.deep.x") + .Should().Match(e => e.Outcome == PathOutcome.Masked && e.Rule == "unknown member (Descend)"); + JsonShapeTests.Observer(shapeOptions: new JsonShapeOptions { Unknown = UnknownMemberPolicy.PassThrough }).Explain("unknown.deep.x") + .Outcome.Should().Be(PathOutcome.Unchanged); + JsonShapeTests.Observer(shapeOptions: new JsonShapeOptions { KeepNulls = false }).Explain("password", ValueKind.Null) + .Outcome.Should().Be(PathOutcome.Masked); } [Theory] @@ -131,7 +131,7 @@ public void Path_Invalid_Throws(string path) [Fact] public void ValueKind_NotAValue_Throws() { - var explain = () => Lines.Explain("a", JsonTokenType.PropertyName); + var explain = () => Lines.Explain("a", (ValueKind)42); explain.Should().Throw(); } @@ -159,13 +159,13 @@ void Walk(JsonElement before, JsonElement after, string path) default: var kind = before.ValueKind switch { - JsonValueKind.Number => JsonTokenType.Number, - JsonValueKind.Null => JsonTokenType.Null, - _ => JsonTokenType.String, + JsonValueKind.Number => ValueKind.Number, + JsonValueKind.Null => ValueKind.Null, + _ => ValueKind.String, }; var explanation = observer.Explain(path, kind); var changed = before.GetRawText() != after.GetRawText(); - (explanation.Outcome == JsonPathOutcome.Masked).Should().Be(changed, explanation.ToString()); + (explanation.Outcome == PathOutcome.Masked).Should().Be(changed, explanation.ToString()); checkedLeaves++; break; } diff --git a/DragoAnt.System.Text.Json.Observer.Tests.Shared/JsonMaskingTests.cs b/DragoAnt.System.Text.Json.Observer.Tests.Shared/JsonMaskingTests.cs index 2954a83..2de4d40 100644 --- a/DragoAnt.System.Text.Json.Observer.Tests.Shared/JsonMaskingTests.cs +++ b/DragoAnt.System.Text.Json.Observer.Tests.Shared/JsonMaskingTests.cs @@ -1,7 +1,8 @@ using System.Globalization; using Bogus; -using DragoAnt.System.Text.Json.Observer.Strategies; -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; +using DragoAnt.Observer; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; namespace DragoAnt.System.Text.Json.Observer.Tests.Shared; @@ -23,43 +24,43 @@ public JsonMaskingTests(ITestOutputHelper outputHelper) private readonly JsonObserver _requestMasking = GetRequestMasking(BlockList); - internal static JsonObserver GetRequestMasking(JsonObserverValueDelegate defaultValuePolicy) + internal static JsonObserver GetRequestMasking(JsonValuePolicy defaultValuePolicy) { - return JsonObserver.Obj(Relative(policyBuilder => policyBuilder - .Match(PropMatches.EndsWith("card"), "saved", "id").MaskStr(MaskingRules.CustomerId) - .Match("card", "number").MaskStr(MaskingRules.CardNumber) - .Match("user", "entered").MaskStr((_, _) => string.Empty) - .Match("recurringTemplate", "id").MaskStr(MaskingRules.RecurringTemplateId) - .Match(PropMatches.Contains("cardHolder")).MaskStr(MaskingRules.FullName) - .Match(PropMatches.StartsWith("order"), "description").MaskStr(MaskingRules.OrderDescription) - .Match(PropMatches.StartsWith("customer"), "id").MaskStr(MaskingRules.CustomerId) - .Match(PropMatches.StartsWith("customer"), "birthDate").MaskStr(MaskingRules.BirthDate) - .Match(PropMatches.Contains("ipAddress")).MaskStr(MaskingRules.Ip) - .Match(PropMatches.Contains("email")).MaskStr(MaskingRules.Email) - .Match(PropMatches.Contains("phone")).MaskStr(MaskingRules.Phone) - .Match(PropMatches.Contains("documentNumber")).MaskStr(MaskingRules.DocumentNumber) - .Match(PropMatches.Contains("firstName")).MaskStr(MaskingRules.Name) - .Match(PropMatches.Contains("lastName")).MaskStr(MaskingRules.Name) - .Match(PropMatches.Contains("address")).MaskStr(MaskingRules.Full) - .Match(PropMatches.Contains("accountNumber")).MaskStr(MaskingRules.AccountNumber) + return JsonObserver.Obj(AnyDepth(policyBuilder => policyBuilder + .Path(Names.EndsWith("card"), "saved", "id").Mask(MaskingRules.CustomerId, MaskNulls.Mask) + .Path("card", "number").Mask(MaskingRules.CardNumber, MaskNulls.Mask) + .Path("user", "entered").Mask((_, _) => string.Empty, MaskNulls.Mask) + .Path("recurringTemplate", "id").Mask(MaskingRules.RecurringTemplateId, MaskNulls.Mask) + .Match(Names.Contains("cardHolder")).Mask(MaskingRules.FullName, MaskNulls.Mask) + .Path(Names.StartsWith("order"), "description").Mask(MaskingRules.OrderDescription, MaskNulls.Mask) + .Path(Names.StartsWith("customer"), "id").Mask(MaskingRules.CustomerId, MaskNulls.Mask) + .Path(Names.StartsWith("customer"), "birthDate").Mask(MaskingRules.BirthDate, MaskNulls.Mask) + .Match(Names.Contains("ipAddress")).Mask(MaskingRules.Ip, MaskNulls.Mask) + .Match(Names.Contains("email")).Mask(MaskingRules.Email, MaskNulls.Mask) + .Match(Names.Contains("phone")).Mask(MaskingRules.Phone, MaskNulls.Mask) + .Match(Names.Contains("documentNumber")).Mask(MaskingRules.DocumentNumber, MaskNulls.Mask) + .Match(Names.Contains("firstName")).Mask(MaskingRules.Name, MaskNulls.Mask) + .Match(Names.Contains("lastName")).Mask(MaskingRules.Name, MaskNulls.Mask) + .Match(Names.Contains("address")).Mask(MaskingRules.Full, MaskNulls.Mask) + .Match(Names.Contains("accountNumber")).Mask(MaskingRules.AccountNumber, MaskNulls.Mask) , defaultValuePolicy)); } private readonly JsonObserver _ignoreNullsRequestMasking = GetRequestUnmasking(NullList); - internal static JsonObserver GetRequestUnmasking(JsonObserverValueDelegate defaultValuePolicy) + internal static JsonObserver GetRequestUnmasking(JsonValuePolicy defaultValuePolicy) { return JsonObserver.Obj(b => b .Match("routing").Obj(sb => sb.Match("method").Unmasked()), - Relative(policyBuilder => policyBuilder - .Match(PropMatches.EndsWith("card"), "saved", "id").Unmasked() - .Match("card", "number").Unmasked() - .Match(PropMatches.Contains("cardHolder")).Unmasked() - .Match(PropMatches.StartsWith("customer"), "id").Unmasked() - .Match(PropMatches.StartsWith("customer"), "birthDate").Unmasked() - .Match(PropMatches.Contains("ipAddress")).Unmasked() - .Match(PropMatches.Contains("email")).Unmasked(), + AnyDepth(policyBuilder => policyBuilder + .Path(Names.EndsWith("card"), "saved", "id").Unmasked() + .Path("card", "number").Unmasked() + .Match(Names.Contains("cardHolder")).Unmasked() + .Path(Names.StartsWith("customer"), "id").Unmasked() + .Path(Names.StartsWith("customer"), "birthDate").Unmasked() + .Match(Names.Contains("ipAddress")).Unmasked() + .Match(Names.Contains("email")).Unmasked(), defaultValuePolicy)); } @@ -144,7 +145,7 @@ internal static JsonObserver GetRequestUnmasking(JsonObserverValueDelegate; +using DragoAnt.Observer; +using static DragoAnt.Observer.ValuePolicy; namespace DragoAnt.System.Text.Json.Observer.Tests.Shared; @@ -34,10 +34,10 @@ public sealed class ReadContext .Match("contractId").ReadInt((v, c) => c.ContractId = v) .Match("contractId2").ReadRaw((v, c) => c.ContractId2 = v) .Match("method").ReadStr((v, c) => c.Method = v)) - .Match("session", "user", "entered").ReadStr((v, c) => c.User = v), - Relative(builder => builder - .Match(PropMatches.EndsWith("card"), "saved", "id").ReadStr((v, c) => c.SavedCardValue = v) - .Match(PropMatches.Contains("ipAddress")).ReadStr((v, c) => c.Ip = v))); + .Path("session", "user", "entered").ReadStr((v, c) => c.User = v), + JsonValuePolicy.AnyDepth(builder => builder + .Path(Names.EndsWith("card"), "saved", "id").ReadStr((v, c) => c.SavedCardValue = v) + .Match(Names.Contains("ipAddress")).ReadStr((v, c) => c.Ip = v))); private static readonly Dictionary SensitiveValues = new() { @@ -162,9 +162,9 @@ public void Read_ForAllRules_Debug() var requestMasking = JsonObserver.Obj(b => b .Match("routing").Obj(routingB => routingB .Match("contractId").ReadInt((v, c) => c.ContractId = v)) - .Match("session", "user", "entered").ReadStr((v, c) => c.User = v), - Relative(b => b - .Match(PropMatches.EndsWith("card"), "saved", "id").ReadStr((v, c) => c.SavedCardValue = v))); + .Path("session", "user", "entered").ReadStr((v, c) => c.User = v), + JsonValuePolicy.AnyDepth(b => b + .Path(Names.EndsWith("card"), "saved", "id").ReadStr((v, c) => c.SavedCardValue = v))); // Act var readContext = Read(TestJson, requestMasking); diff --git a/DragoAnt.System.Text.Json.Observer.Tests.Shared/JsonShapeMetadataTests.cs b/DragoAnt.System.Text.Json.Observer.Tests.Shared/JsonShapeMetadataTests.cs index 73fc063..1df3d74 100644 --- a/DragoAnt.System.Text.Json.Observer.Tests.Shared/JsonShapeMetadataTests.cs +++ b/DragoAnt.System.Text.Json.Observer.Tests.Shared/JsonShapeMetadataTests.cs @@ -1,7 +1,7 @@ using System.Reflection; using System.Text.Json.Serialization; using System.Text.Json.Serialization.Metadata; -using DragoAnt.System.Text.Json.Observer.Strategies; +using DragoAnt.Observer; namespace DragoAnt.System.Text.Json.Observer.Tests.Shared; diff --git a/DragoAnt.System.Text.Json.Observer.Tests.Shared/JsonShapeTests.cs b/DragoAnt.System.Text.Json.Observer.Tests.Shared/JsonShapeTests.cs index fdeb329..3683865 100644 --- a/DragoAnt.System.Text.Json.Observer.Tests.Shared/JsonShapeTests.cs +++ b/DragoAnt.System.Text.Json.Observer.Tests.Shared/JsonShapeTests.cs @@ -1,6 +1,6 @@ using System.Text.Json.Serialization; using System.Text.Json.Serialization.Metadata; -using DragoAnt.System.Text.Json.Observer.Strategies; +using DragoAnt.Observer; namespace DragoAnt.System.Text.Json.Observer.Tests.Shared; @@ -15,7 +15,7 @@ internal static JsonObserver Observer(JsonSerializerOptions? options = null, Jso private static MaskTag? Classify(JsonPropertyInfo property) => property.AttributeProvider?.GetCustomAttributes(typeof(SensitiveAttribute), true).OfType().FirstOrDefault() is { } sensitive - ? new MaskTag(sensitive.Kind) + ? MaskTag.Create(sensitive.Kind) : null; [Fact] @@ -76,18 +76,18 @@ public void AllowList_OpaqueAndMismatchedTypes_Masked() => public void AllowList_Nulls_KeptOrMasked() { Observer().Mask("""{"unknown":null,"password":null}""").Should().Be("""{"unknown":null,"password":null}"""); - Observer(shapeOptions: new JsonShapeOptions(KeepNulls: false)).Mask("""{"unknown":null,"password":null,"name":null}""") + Observer(shapeOptions: new JsonShapeOptions { KeepNulls = false }).Mask("""{"unknown":null,"password":null,"name":null}""") .Should().Be("""{"unknown":"***","password":"***","name":null}"""); } [Fact] public void Unknown_Descend_ShowsNamesMasksValues() => - Observer(shapeOptions: new JsonShapeOptions(UnknownMemberPolicy.Descend)).Mask("""{"unknown":{"a":"x","b":[1,{"c":true}]}}""") + Observer(shapeOptions: new JsonShapeOptions { Unknown = UnknownMemberPolicy.Descend }).Mask("""{"unknown":{"a":"x","b":[1,{"c":true}]}}""") .Should().Be("""{"unknown":{"a":"***","b":["***",{"c":"***"}]}}"""); [Fact] public void Unknown_PassThrough_KeepsValuesMasksSensitive() => - Observer(shapeOptions: new JsonShapeOptions(UnknownMemberPolicy.PassThrough)).Mask("""{"unknown":{"a":"x","b":[1]},"password":"p"}""") + Observer(shapeOptions: new JsonShapeOptions { Unknown = UnknownMemberPolicy.PassThrough }).Mask("""{"unknown":{"a":"x","b":[1]},"password":"p"}""") .Should().Be("""{"unknown":{"a":"x","b":[1]},"password":"***"}"""); [Fact] @@ -112,16 +112,12 @@ public void BytesApi_TruncatedShape_ReturnsSafePrefix() } [Fact] - public void Legacy_AllowList_Behaviour_Documented() + public void AllowList_Behaviour_Documented() { const string json = """{"s":"x","n":1,"b":true,"z":null}"""; -#pragma warning disable CS0618 - var legacy = JsonObserver.Obj(_ => { }, JsonObserverValuePolicies.LegacyAllowList); -#pragma warning restore CS0618 - var allowList = JsonObserver.Obj(_ => { }, JsonObserverValuePolicies.AllowList); + var allowList = JsonObserver.Obj(_ => { }, ValuePolicy.AllowList); var byDefault = JsonObserver.Obj(_ => { }); - legacy.Mask(json).Should().Be("""{"s":"#str#*****","n":"#number#*****","b":true,"z":null}"""); allowList.Mask(json).Should().Be("""{"s":"***","n":"***","b":"***","z":null}"""); byDefault.Mask(json).Should().Be(allowList.Mask(json)); } @@ -144,7 +140,7 @@ public sealed class Customer [Sensitive(MaskKind.Last4)] public string? Card { get; set; } - [Sensitive(MaskKind.Omit)] + [Sensitive(MaskKind.Null)] public int? Pin { get; set; } [Sensitive] diff --git a/DragoAnt.System.Text.Json.Observer.Tests.Shared/JsonWriterSpanTests.cs b/DragoAnt.System.Text.Json.Observer.Tests.Shared/JsonWriterSpanTests.cs index d2e43c8..0cc52c8 100644 --- a/DragoAnt.System.Text.Json.Observer.Tests.Shared/JsonWriterSpanTests.cs +++ b/DragoAnt.System.Text.Json.Observer.Tests.Shared/JsonWriterSpanTests.cs @@ -1,14 +1,16 @@ -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; namespace DragoAnt.System.Text.Json.Observer.Tests.Shared; public abstract class JsonWriterSpanTests { - private static JsonObserver Writing(JsonObserverValueDelegate rule) => + private static JsonObserver Writing(JsonValueRule rule) => JsonObserver.Obj(b => b.Match("a").MaskValue(rule), BlockList); - private static void EverySpanOverload(ref Utf8JsonReader reader, JsonWriter writer, JsonObserveringEmptyContext context, ref PropertyPath path) + private static void EverySpanOverload(ref JsonValueContext context) { + var writer = context.Writer; writer.WriteStartObject(); writer.WritePropertyName("chars".AsSpan()); writer.WriteStringValue("é\"x".AsSpan()); @@ -39,16 +41,15 @@ public void SpanOverloads_BytesApiMatchesStringApi() [Fact] public void SpanOverloads_IgnoreNulls_WritesPendingNames() => - Writing(EverySpanOverload).Mask("""{"a":0}""", new JsonObserverOptions(IgnoreNulls: true)) + Writing(EverySpanOverload).Mask("""{"a":0}""", new JsonObserverOptions { IgnoreNulls = true }) .Should().Be("""{"a":{"chars":"é\"x","b64":"AQID+g==","d":1.25,"nan":"NaN","none":""}}"""); [Fact] public void CharSpan_LongerThanMaxValueBytes_Cut() { - var observer = Writing((ref Utf8JsonReader _, JsonWriter writer, JsonObserveringEmptyContext _, ref PropertyPath _) => - writer.WriteStringValue("abcdefgh".AsSpan())); + var observer = Writing((ref JsonValueContext __c) => { var writer = __c.Writer; writer.WriteStringValue("abcdefgh".AsSpan()); }); - var masked = observer.Mask("""{"a":0}""", out var result, new JsonObserverOptions(MaxValueBytes: 4)); + var masked = observer.Mask("""{"a":0}""", out var result, new JsonObserverOptions { MaxValueBytes = 4 }); masked.Should().Be("""{"a":"abcd…"}"""); result.Status.Should().Be(MaskStatus.Truncated); @@ -57,17 +58,16 @@ public void CharSpan_LongerThanMaxValueBytes_Cut() [Fact] public void Base64_LongerThanMaxValueBytes_Cut() { - var observer = Writing((ref Utf8JsonReader _, JsonWriter writer, JsonObserveringEmptyContext _, ref PropertyPath _) => - writer.WriteBase64StringValue([1, 2, 3, 4, 5, 6])); + var observer = Writing((ref JsonValueContext __c) => { var writer = __c.Writer; writer.WriteBase64StringValue([1, 2, 3, 4, 5, 6]); }); - observer.Mask("""{"a":0}""", new JsonObserverOptions(MaxValueBytes: 4)).Should().Be("""{"a":"AQID…"}"""); + observer.Mask("""{"a":0}""", new JsonObserverOptions { MaxValueBytes = 4 }).Should().Be("""{"a":"AQID…"}"""); } [Fact] public void SpanOverloads_ReadOnly_WriteNothing() { - var observer = JsonObserver.Obj(b => b.Match("a").MaskValue(EverySpanOverload), JsonObserverValuePolicies.BlockList); + var observer = JsonObserver.Obj(b => b.Match("a").MaskValue(EverySpanOverload), ValuePolicy.BlockList); - observer.Read("""{"a":0}""", JsonObserveringEmptyContext.Instance).Status.Should().Be(MaskStatus.Masked); + observer.Read("""{"a":0}""", NoContext.Instance).Status.Should().Be(MaskStatus.Masked); } } diff --git a/DragoAnt.System.Text.Json.Observer.Tests.Shared/LeakTests.cs b/DragoAnt.System.Text.Json.Observer.Tests.Shared/LeakTests.cs index 94e008d..c2c9047 100644 --- a/DragoAnt.System.Text.Json.Observer.Tests.Shared/LeakTests.cs +++ b/DragoAnt.System.Text.Json.Observer.Tests.Shared/LeakTests.cs @@ -1,7 +1,7 @@ using System.Buffers; using System.Text; using System.Text.RegularExpressions; -using DragoAnt.System.Text.Json.Observer.Strategies; +using DragoAnt.Observer; namespace DragoAnt.System.Text.Json.Observer.Tests.Shared; @@ -9,9 +9,9 @@ public abstract class LeakTests { private const string Ssn = """{"ssn":"123-45-6789","x":"y"}"""; - private static JsonObserverValueDelegate AllowList => JsonObserverValuePolicies.AllowList; - private static JsonObserverValueDelegate BlockList => JsonObserverValuePolicies.BlockList; - private static JsonObserverValueDelegate NullList => JsonObserverValuePolicies.NullList; + private static JsonValuePolicy AllowList => ValuePolicy.AllowList; + private static JsonValuePolicy BlockList => ValuePolicy.BlockList; + private static JsonValuePolicy NullList => ValuePolicy.NullList; [Fact] public void Read_UnderAllowList_WritesTheValueMasked() @@ -47,7 +47,7 @@ public void Read_UnderNullList_WritesNull() public void Read_InRelativePolicy_UnderAllowList_WritesTheValueMasked() { var holder = new Holder(); - var observer = JsonObserver.Obj(JsonObserverValuePolicies.Relative(b => b.Match("ssn").ReadStr(Keep))); + var observer = JsonObserver.Obj(JsonValuePolicy.AnyDepth(b => b.Match("ssn").ReadStr(Keep))); observer.Mask("""{"person":{"ssn":"123-45-6789"}}""", holder).Should().Be("""{"person":{"ssn":"***"}}"""); holder.Value.Should().Be("123-45-6789"); @@ -78,7 +78,7 @@ public void Read_ThenMask_OnOneMatch_ReadsAndMasks() { var holder = new Holder(); - JsonObserver.Obj(r => r.Match("ssn").ReadStr(Keep).MaskAny(MaskTag.Last4), BlockList).Mask(Ssn, holder) + JsonObserver.Obj(r => r.Match("ssn").ReadStr(Keep).Mask(MaskTag.Last4), BlockList).Mask(Ssn, holder) .Should().Be("""{"ssn":"***6789","x":"y"}"""); holder.Value.Should().Be("123-45-6789"); } @@ -88,7 +88,7 @@ public void Mask_ThenRead_OnOneMatch_ReadsAndMasks() { var holder = new Holder(); - JsonObserver.Obj(r => r.Match("ssn").MaskAny(MaskTag.Full).Match("ssn").ReadStr(Keep), BlockList).Mask(Ssn, holder) + JsonObserver.Obj(r => r.Match("ssn").Mask(MaskTag.Full).Match("ssn").ReadStr(Keep), BlockList).Mask(Ssn, holder) .Should().Be("""{"ssn":"***","x":"y"}"""); holder.Value.Should().Be("123-45-6789"); } @@ -97,8 +97,8 @@ public void Mask_ThenRead_OnOneMatch_ReadsAndMasks() public void Read_ThenMask_InRelativePolicy_ReadsAndMasks() { var holder = new Holder(); - var observer = JsonObserver.Obj(JsonObserverValuePolicies.Relative( - b => b.Match("ssn").ReadStr(Keep).MaskAny(MaskTag.Full), BlockList)); + var observer = JsonObserver.Obj(JsonValuePolicy.AnyDepth( + b => b.Match("ssn").ReadStr(Keep).Mask(MaskTag.Full), BlockList)); observer.Mask("""{"person":{"ssn":"123-45-6789","x":"y"}}""", holder).Should().Be("""{"person":{"ssn":"***","x":"y"}}"""); holder.Value.Should().Be("123-45-6789"); @@ -120,7 +120,7 @@ public void ArrayRead_ThenUnmaskedOrMask_DecidesTheItem() var masked = new Holder(); JsonObserver.Array(a => a.ReadStr(Keep).Unmasked()).Mask("""["123-45-6789"]""", unmasked).Should().Be("""["123-45-6789"]"""); - JsonObserver.Array(a => a.MaskAny(MaskTag.Last4).ReadStr(Keep), BlockList).Mask("""["123-45-6789"]""", masked) + JsonObserver.Array(a => a.Mask(MaskTag.Last4).ReadStr(Keep), BlockList).Mask("""["123-45-6789"]""", masked) .Should().Be("""["***6789"]"""); unmasked.Value.Should().Be("123-45-6789"); masked.Value.Should().Be("123-45-6789"); @@ -131,7 +131,7 @@ public void Read_ExtractionOnly_StillReads() { var holder = new Holder(); - JsonObserver.Obj(r => r.Match("ssn").MaskAny(MaskTag.Full).Match("ssn").ReadStr(Keep)).Read(Ssn, holder) + JsonObserver.Obj(r => r.Match("ssn").Mask(MaskTag.Full).Match("ssn").ReadStr(Keep)).Read(Ssn, holder) .Status.Should().Be(MaskStatus.Masked); holder.Value.Should().Be("123-45-6789"); } @@ -140,11 +140,11 @@ public void Read_ExtractionOnly_StillReads() public void Explain_Read_ReportsWhatIsWritten() { JsonObserver.Obj(r => r.Match("ssn").ReadStr(Keep), AllowList).Explain("ssn").Outcome - .Should().Be(JsonPathOutcome.Masked); + .Should().Be(PathOutcome.Masked); JsonObserver.Obj(r => r.Match("ssn").ReadStr(Keep).Unmasked(), AllowList).Explain("ssn").Outcome - .Should().Be(JsonPathOutcome.Read); - JsonObserver.Obj(r => r.Match("ssn").ReadStr(Keep).MaskAny(MaskTag.Full), BlockList).Explain("ssn").Outcome - .Should().Be(JsonPathOutcome.Masked); + .Should().Be(PathOutcome.Read); + JsonObserver.Obj(r => r.Match("ssn").ReadStr(Keep).Mask(MaskTag.Full), BlockList).Explain("ssn").Outcome + .Should().Be(PathOutcome.Masked); } public static TheoryData MaskFunctionKinds => ["MaskAny", "MaskStr", "MaskRawValue"]; @@ -155,12 +155,12 @@ public void MaskFunction_UnderValueCap_ReceivesTheWholeValue(string kind) { var observer = JsonObserver.Obj(r => _ = kind switch { - "MaskAny" => r.Match("card").MaskAny((s, _) => "***" + s![^4..]), - "MaskStr" => r.Match("card").MaskStr((s, _) => "***" + s![^4..]), - _ => r.Match("card").MaskRawValue((s, _) => "***" + s![^4..]), + "MaskAny" => r.Match("card").Mask((s, _) => "***" + s![^4..]), + "MaskStr" => r.Match("card").Mask((s, _) => "***" + s![^4..], MaskNulls.Mask), + _ => r.Match("card").Mask((s, _) => "***" + s![^4..], MaskNulls.Mask), }); - observer.Mask("""{"card":"1111222233334444"}""", out var result, new JsonObserverOptions(MaxValueBytes: 8)) + observer.Mask("""{"card":"1111222233334444"}""", out var result, new JsonObserverOptions { MaxValueBytes = 8 }) .Should().Be("""{"card":"***4444"}"""); result.Status.Should().Be(MaskStatus.Masked); } @@ -168,11 +168,11 @@ public void MaskFunction_UnderValueCap_ReceivesTheWholeValue(string kind) [Fact] public void MaskFunction_UnderValueCap_ReceivesTheWholeValue_FromSegments() { - var observer = JsonObserver.Obj(r => r.Match("card").MaskAny((s, _) => "***" + s![^4..])); + var observer = JsonObserver.Obj(r => r.Match("card").Mask((s, _) => "***" + s![^4..])); var output = new ArrayBufferWriter(); var result = observer.Mask(SequenceInputTests.Split("""{"card":"1111222233334444"}"""u8.ToArray(), 3), output, - new JsonObserverOptions(MaxValueBytes: 8)); + new JsonObserverOptions { MaxValueBytes = 8 }); Encoding.UTF8.GetString(output.WrittenSpan).Should().Be("""{"card":"***4444"}"""); result.Status.Should().Be(MaskStatus.Masked); @@ -181,9 +181,9 @@ public void MaskFunction_UnderValueCap_ReceivesTheWholeValue_FromSegments() [Fact] public void MaskOutput_LongerThanValueCap_IsNotCut() { - var observer = JsonObserver.Obj(r => r.Match("card").MaskAny((_, _) => "replaced-by-a-long-mask")); + var observer = JsonObserver.Obj(r => r.Match("card").Mask((_, _) => "replaced-by-a-long-mask")); - observer.Mask("""{"card":"1"}""", out var result, new JsonObserverOptions(MaxValueBytes: 8)) + observer.Mask("""{"card":"1"}""", out var result, new JsonObserverOptions { MaxValueBytes = 8 }) .Should().Be("""{"card":"replaced-by-a-long-mask"}"""); result.Status.Should().Be(MaskStatus.Masked); } @@ -191,22 +191,22 @@ public void MaskOutput_LongerThanValueCap_IsNotCut() [Fact] public void Hash_UnderValueCap_IsNotCut() { - var observer = JsonObserver.Obj(r => r.Match("card").MaskAny(MaskTag.Hash)); - var options = new JsonObserverOptions(MaxValueBytes: 8, HashKey: "0123456789abcdef0123456789abcdef"u8.ToArray()); + var observer = JsonObserver.Obj(r => r.Match("card").Mask(MaskTag.Hash)); + var options = new JsonObserverOptions { MaxValueBytes = 8, HashKey = "0123456789abcdef0123456789abcdef"u8.ToArray() }; var masked = observer.Mask("""{"card":"1111222233334444"}""", out var result, options)!; var uncapped = observer.Mask("""{"card":"1111222233334444"}""", options with { MaxValueBytes = int.MaxValue }); masked.Should().Be(uncapped); - JsonDocument.Parse(masked).RootElement.GetProperty("card").GetString().Should().StartWith("hash:").And.NotContain("…"); + JsonDocument.Parse(masked).RootElement.GetProperty("card").GetString().Should().HaveLength(24).And.EndWith("==").And.NotContain("…"); result.Status.Should().Be(MaskStatus.Masked); } [Fact] public void UnmaskedValue_UnderValueCap_IsStillCut() { - JsonObserver.Obj(JsonObserverValuePolicies.BlockList) - .Mask("""{"note":"1111222233334444"}""", out var result, new JsonObserverOptions(MaxValueBytes: 8)) + JsonObserver.Obj(ValuePolicy.BlockList) + .Mask("""{"note":"1111222233334444"}""", out var result, new JsonObserverOptions { MaxValueBytes = 8 }) .Should().Be("""{"note":"11112222…"}"""); result.Status.Should().Be(MaskStatus.Truncated); } @@ -230,17 +230,17 @@ public void UnmaskedValue_UnderValueCap_IsStillCut() [MemberData(nameof(CaseTruthTable))] public void CaseOption_ReachesEveryMatcher(string matcher, bool caseInsensitive, string expectedMasked) { - PropMatchingStrategy match = matcher switch + NameMatch match = matcher switch { "Match" => "driverLicense", - "Function" => new PropMatchingStrategy((name, comparison) => string.Equals(name, "driverLicense", comparison)), - _ => PropMatches.Regex(new Regex("^driverLicense$")), + "Function" => new NameMatch((name, comparison) => string.Equals(name, "driverLicense", comparison)), + _ => Names.Regex(new Regex("^driverLicense$")), }; var observer = matcher == "RelativeRegex" - ? JsonObserver.Obj(JsonObserverValuePolicies.Relative(b => b.Match(match).MaskAny(MaskTag.Full), JsonObserverValuePolicies.BlockList)) - : JsonObserver.Obj(r => r.Match(match).MaskAny(MaskTag.Full), JsonObserverValuePolicies.BlockList); + ? JsonObserver.Obj(JsonValuePolicy.AnyDepth(b => b.Match(match).Mask(MaskTag.Full), ValuePolicy.BlockList)) + : JsonObserver.Obj(r => r.Match(match).Mask(MaskTag.Full), ValuePolicy.BlockList); - var output = observer.Mask(CaseCorpus, new JsonObserverOptions(PropertyNameCaseInsensitive: caseInsensitive))!; + var output = observer.Mask(CaseCorpus, new JsonObserverOptions { NameCaseInsensitive = caseInsensitive })!; var masked = JsonDocument.Parse(output).RootElement.EnumerateObject() .Select((p, i) => (Key: $"A{i + 1}", Value: p.Value.GetString())) @@ -253,10 +253,10 @@ public void CaseOption_ReachesEveryMatcher(string matcher, bool caseInsensitive, public void Regex_WithExplicitIgnoreCase_StaysCaseInsensitive() { var observer = JsonObserver.Obj( - r => r.Match(PropMatches.Regex(new Regex("^token$", RegexOptions.IgnoreCase))).MaskAny(MaskTag.Full), - JsonObserverValuePolicies.BlockList); + r => r.Match(Names.Regex(new Regex("^token$", RegexOptions.IgnoreCase))).Mask(MaskTag.Full), + ValuePolicy.BlockList); - observer.Mask("""{"Token":"a"}""", new JsonObserverOptions(PropertyNameCaseInsensitive: false)).Should().Be("""{"Token":"***"}"""); + observer.Mask("""{"Token":"a"}""", new JsonObserverOptions { NameCaseInsensitive = false }).Should().Be("""{"Token":"***"}"""); } private static void Keep(string? value, Holder holder) => holder.Value = value; diff --git a/DragoAnt.System.Text.Json.Observer.Tests.Shared/LimitsTests.cs b/DragoAnt.System.Text.Json.Observer.Tests.Shared/LimitsTests.cs index 3fa1a83..01d2260 100644 --- a/DragoAnt.System.Text.Json.Observer.Tests.Shared/LimitsTests.cs +++ b/DragoAnt.System.Text.Json.Observer.Tests.Shared/LimitsTests.cs @@ -1,6 +1,7 @@ using System.Buffers; using System.Text; -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; namespace DragoAnt.System.Text.Json.Observer.Tests.Shared; @@ -9,12 +10,12 @@ public abstract class LimitsTests private static readonly JsonObserver Observer = JsonObserver.Any( _ => { }, _ => { }, - Relative(b => b.Match("password").MaskAny("***"), BlockList)); + AnyDepth(b => b.Match("password").Mask("***"), BlockList)); [Fact] public void MaxValueBytes_CutValue_ReportsTruncatedWithWholeDocument() { - var (result, output) = BytesApiTests.Mask(Observer, """{"note":"abcdefghij","n":1}""", new JsonObserverOptions(MaxValueBytes: 5)); + var (result, output) = BytesApiTests.Mask(Observer, """{"note":"abcdefghij","n":1}""", new JsonObserverOptions { MaxValueBytes = 5 }); result.Status.Should().Be(MaskStatus.Truncated); result.FailedAtByte.Should().Be(-1); @@ -23,14 +24,14 @@ public void MaxValueBytes_CutValue_ReportsTruncatedWithWholeDocument() [Fact] public void MaxValueBytes_NoValueCut_ReportsMasked() - => BytesApiTests.Mask(Observer, """{"note":"abcde"}""", new JsonObserverOptions(MaxValueBytes: 5)).Result.Status.Should().Be(MaskStatus.Masked); + => BytesApiTests.Mask(Observer, """{"note":"abcde"}""", new JsonObserverOptions { MaxValueBytes = 5 }).Result.Status.Should().Be(MaskStatus.Masked); [Fact] public void MaxValueBytes_StrategyOutput_IsNotCut() { - var observer = JsonObserver.Obj(b => b.Match("a").MaskStr((_, _) => new string('x', 100)), BlockList); + var observer = JsonObserver.Obj(b => b.Match("a").Mask((_, _) => new string('x', 100), MaskNulls.Mask), BlockList); - var (result, output) = BytesApiTests.Mask(observer, """{"a":"v"}""", new JsonObserverOptions(MaxValueBytes: 4)); + var (result, output) = BytesApiTests.Mask(observer, """{"a":"v"}""", new JsonObserverOptions { MaxValueBytes = 4 }); result.Status.Should().Be(MaskStatus.Masked); output.Should().Be($"{{\"a\":\"{new string('x', 100)}\"}}"); @@ -40,10 +41,10 @@ public void MaxValueBytes_StrategyOutput_IsNotCut() public void MaxValueBytes_LongCustomRuleString_SurrogatePairNotSplit() { var observer = JsonObserver.Obj( - b => b.Match("a").MaskValue((ref Utf8JsonReader _, JsonWriter w, JsonObserveringEmptyContext _, ref PropertyPath _) => w.WriteStringValue("ab\U0001F600cd")), + b => b.Match("a").MaskValue((ref JsonValueContext __c) => { var w = __c.Writer; w.WriteStringValue("ab\U0001F600cd"); }), BlockList); - var (_, output) = BytesApiTests.Mask(observer, """{"a":"v"}""", new JsonObserverOptions(MaxValueBytes: 4)); + var (_, output) = BytesApiTests.Mask(observer, """{"a":"v"}""", new JsonObserverOptions { MaxValueBytes = 4 }); JsonDocument.Parse(output).RootElement.GetProperty("a").GetString().Should().Be("ab…"); } @@ -53,10 +54,10 @@ public void MaskAny_Strategy_HugeValue_ReceivesTheWholeValue() { var secret = new string('s', 5 * 1024 * 1024); var utf8 = Encoding.UTF8.GetBytes($$"""{"password":"{{secret}}","n":1}"""); - var observer = JsonObserver.Obj(Relative(b => b.Match("password").MaskAny((v, _) => v is null ? null : "len:" + v.Length), BlockList)); + var observer = JsonObserver.Obj(AnyDepth(b => b.Match("password").Mask((v, _) => v is null ? null : "len:" + v.Length), BlockList)); var output = new ArrayBufferWriter(1024); - var result = observer.Mask(utf8, output, new JsonObserverOptions(MaxValueBytes: 256)); + var result = observer.Mask(utf8, output, new JsonObserverOptions { MaxValueBytes = 256 }); result.Status.Should().Be(MaskStatus.Masked); Encoding.UTF8.GetString(output.WrittenSpan).Should().Be($$"""{"password":"len:{{secret.Length}}","n":1}"""); @@ -75,7 +76,7 @@ public void MaxOutputBytes_Reached_StopsReadingTheInput() var observer = JsonObserver.Array(a => a.Obj(o => o.Match("user").ReadStr((_, c) => c.Count++))); var utf8 = Encoding.UTF8.GetBytes(builder.Append(']').ToString()); - var result = observer.Mask(utf8, new ArrayBufferWriter(), counter, new JsonObserverOptions(MaxOutputBytes: 100)); + var result = observer.Mask(utf8, new ArrayBufferWriter(), counter, new JsonObserverOptions { MaxOutputBytes = 100 }); result.Status.Should().Be(MaskStatus.Truncated); counter.Count.Should().BeLessThan(20); @@ -90,7 +91,7 @@ public void ReadInt_NonInteger_KeepsTokenAndReadsNull(string number) var context = new Counter { Value = 7 }; var observer = JsonObserver.Obj( b => b.Match("n").ReadInt((v, c) => c.Value = v), - JsonObserverValuePolicies.Relative(r => r.Match("password").MaskAny("***"), JsonObserverValuePolicies.BlockList)); + JsonValuePolicy.AnyDepth(r => r.Match("password").Mask("***"), ValuePolicy.BlockList)); var json = $$"""{"n":{{number}},"password":"p"}"""; var output = new ArrayBufferWriter(); diff --git a/DragoAnt.System.Text.Json.Observer.Tests.Shared/MaskAndExtractTests.cs b/DragoAnt.System.Text.Json.Observer.Tests.Shared/MaskAndExtractTests.cs index 065c79f..66915d9 100644 --- a/DragoAnt.System.Text.Json.Observer.Tests.Shared/MaskAndExtractTests.cs +++ b/DragoAnt.System.Text.Json.Observer.Tests.Shared/MaskAndExtractTests.cs @@ -13,7 +13,7 @@ public abstract class MaskAndExtractTests .Match("name").ReadStr((v, c) => c.Name = v) .Match("raw").ReadRaw((v, c) => c.Raw = v) .Match("none").ReadInt((v, c) => c.None = v ?? -1), - JsonObserverValuePolicies.Relative(r => r.Match("password").MaskAny("***"), JsonObserverValuePolicies.BlockList)); + JsonValuePolicy.AnyDepth(r => r.Match("password").Mask("***"), ValuePolicy.BlockList)); [Fact] public void MaskAndExtract_KeepsJsonTypes() diff --git a/DragoAnt.System.Text.Json.Observer.Tests.Shared/MaskTagTests.cs b/DragoAnt.System.Text.Json.Observer.Tests.Shared/MaskTagTests.cs index f627def..190728a 100644 --- a/DragoAnt.System.Text.Json.Observer.Tests.Shared/MaskTagTests.cs +++ b/DragoAnt.System.Text.Json.Observer.Tests.Shared/MaskTagTests.cs @@ -1,6 +1,7 @@ using System.Text; -using DragoAnt.System.Text.Json.Observer.Strategies; -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; +using DragoAnt.Observer; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; namespace DragoAnt.System.Text.Json.Observer.Tests.Shared; @@ -9,11 +10,11 @@ public abstract class MaskTagTests private static readonly JsonObserver Observer = JsonObserver.Any( _ => { }, _ => { }, - Relative(b => b - .Match("full").MaskAny(MaskTag.Full) - .Match("last4").MaskAny(MaskTag.Last4) - .Match("hash").MaskAny(MaskTag.Hash) - .Match("omit").MaskAny(MaskKind.Omit), + AnyDepth(b => b + .Match("full").Mask(MaskTag.Full) + .Match("last4").Mask(MaskTag.Last4) + .Match("hash").Mask(MaskTag.Hash) + .Match("omit").Mask(MaskKind.Null), BlockList)); private static JsonElement Mask(string json, JsonObserverOptions? options = null) @@ -45,14 +46,14 @@ public void Hash_StableAcrossCalls() var other = Mask("""{"hash":"S3cr3u"}""").GetProperty("hash").GetString(); var escaped = Mask("{\"hash\":\"S" + (char)92 + "u0033cr3t\"}").GetProperty("hash").GetString(); - first.Should().StartWith("hash:").And.HaveLength(21).And.Be(second).And.Be(escaped).And.NotBe(other); + first.Should().HaveLength(24).And.EndWith("==").And.Be(second).And.Be(escaped).And.NotBe(other); } [Fact] public void Hash_KeyedDiffers() { - var keyA = new JsonObserverOptions(HashKey: Encoding.UTF8.GetBytes("key-a")); - var keyB = new JsonObserverOptions(HashKey: Encoding.UTF8.GetBytes("key-b")); + var keyA = new JsonObserverOptions { HashKey = Encoding.UTF8.GetBytes("key-a") }; + var keyB = new JsonObserverOptions { HashKey = Encoding.UTF8.GetBytes("key-b") }; var a = Mask("""{"hash":"S3cr3t"}""", keyA).GetProperty("hash").GetString(); var again = Mask("""{"hash":"S3cr3t"}""", keyA with { }).GetProperty("hash").GetString(); @@ -73,24 +74,24 @@ public void Tag_PassedToCustomStrategy() var (_, output) = BytesApiTests.Mask( Observer, """{"full":"a","hash":7,"last4":{"x":1},"omit":true}""", - new JsonObserverOptions(MaskStrategy: strategy)); + new JsonObserverOptions { Strategy = strategy }); output.Should().Be("""{"full":"?","hash":"?","last4":"?","omit":"?"}"""); strategy.Calls.Should().Equal( "Full String a", "Hash Number 7", - "Last4 StartObject ", - "Omit True true"); + "Last4 Object ", + "Null Boolean true"); } private sealed record Classification(string Taxonomy, string Name); private static readonly Classification Pii = new("Demo", "Pii"); - private static readonly JsonObserver KeyedObserver = JsonObserver.Obj(Relative(b => b - .Match("email").MaskAny(MaskTag.Custom(Pii)) - .Match("token").MaskAny(new MaskTag(MaskKind.Hash, "secret")) - .Match("plain").MaskAny(MaskTag.Last4), + private static readonly JsonObserver KeyedObserver = JsonObserver.Obj(AnyDepth(b => b + .Match("email").Mask(MaskTag.Custom(Pii)) + .Match("token").Mask(MaskTag.Create(MaskKind.Hash, "secret")) + .Match("plain").Mask(MaskTag.Last4), BlockList)); [Fact] @@ -101,7 +102,7 @@ public void CustomKey_ReachesStrategyWithoutCasts() var (_, output) = BytesApiTests.Mask( KeyedObserver, """{"email":"a@b.c","token":"t0k3n","plain":"12345678"}""", - new JsonObserverOptions(MaskStrategy: strategy)); + new JsonObserverOptions { Strategy = strategy }); output.Should().Be("""{"email":"Pii","token":"secret","plain":"none"}"""); } @@ -114,7 +115,7 @@ public void CustomKey_DefaultStrategy_FallsBackToKind() var masked = KeyedObserver.Mask("""{"email":"a@b.c","token":"t0k3n","plain":"12345678"}"""); - masked.Should().StartWith("{\"email\":\"***\",\"token\":\"hash:").And.EndWith("\",\"plain\":\"***5678\"}"); + masked.Should().MatchRegex("""^\{"email":"\*\*\*","token":"[A-Za-z0-9+/]{22}==","plain":"\*\*\*5678"\}$"""); } [Fact] @@ -128,7 +129,7 @@ public void MaskTag_KeyEqualityAndAccessors() custom.TryGetKey(out var key).Should().BeTrue(); key.Should().Be(Pii); custom.TryGetKey(out _).Should().BeFalse(); - ((MaskTag)MaskKind.Last4).Should().Be(MaskTag.Last4).And.Be(new MaskTag(MaskKind.Last4, null)); + ((MaskTag)MaskKind.Last4).Should().Be(MaskTag.Last4).And.Be(MaskTag.Create(MaskKind.Last4, null)); MaskTag.Last4.Key.Should().BeNull(); var build = () => MaskTag.Custom(null!); build.Should().Throw(); @@ -140,12 +141,12 @@ public void ContextStrategy_ReceivesPropertyNameAndPath() var strategy = new DiscriminatingStrategy(); var rules = JsonObserver.Obj( root => root - .Match("user").Obj(u => u.Match("email").MaskAny(MaskTag.Custom(Pii))) - .Match("tags").Array(t => t.MaskAny(MaskTag.Hash)), - Relative(b => b.Match("phone").MaskAny(MaskTag.Last4), BlockList)); + .Match("user").Obj(u => u.Match("email").Mask(MaskTag.Custom(Pii))) + .Match("tags").Array(t => t.Mask(MaskTag.Hash)), + AnyDepth(b => b.Match("phone").Mask(MaskTag.Last4), BlockList)); var shape = JsonObserver.FromShape(JsonShape.Object( ("cards", JsonShape.Array(JsonShape.Object(("number", JsonShape.Masked(MaskTag.Last4))))))); - var options = new JsonObserverOptions(MaskStrategy: strategy); + var options = new JsonObserverOptions { Strategy = strategy }; rules.Mask("""{"user":{"email":"a@b.c"},"tags":["x"],"o":{"phone":"12"}}""", options) .Should().Be("""{"user":{"email":"a@b.c:email"},"tags":["x:"],"o":{"phone":"12:phone"}}"""); @@ -165,63 +166,66 @@ public void OldSignatureOnly_StillCalled() { var strategy = new RecordingStrategy(); - Observer.Mask("""{"full":"a"}""", new JsonObserverOptions(MaskStrategy: strategy)).Should().Be("""{"full":"?"}"""); + Observer.Mask("""{"full":"a"}""", new JsonObserverOptions { Strategy = strategy }).Should().Be("""{"full":"?"}"""); strategy.Calls.Should().Equal("Full String a"); } [Fact] - public void NoOverride_BehavesLikeDefault() => - Observer.Mask("""{"last4":"4111111111111111","omit":1}""", new JsonObserverOptions(MaskStrategy: new NoOverrideStrategy())) + public void Delegating_BehavesLikeDefault() => + Observer.Mask("""{"last4":"4111111111111111","omit":1}""", new JsonObserverOptions { Strategy = new NoOverrideStrategy() }) .Should().Be("""{"last4":"***1111","omit":null}"""); - private sealed class NoOverrideStrategy : Utf8MaskStrategy; + private sealed class NoOverrideStrategy : ValueMaskStrategy + { + public override void Mask(in MaskContext context, MaskValueWriter output) => Default.Mask(context, output); + } - private sealed class DiscriminatingStrategy : Utf8MaskStrategy + private sealed class DiscriminatingStrategy : ValueMaskStrategy { public List Calls { get; } = []; - public override void Mask(in Utf8MaskContext context, JsonWriter writer) + public override void Mask(in MaskContext context, MaskValueWriter output) { - var name = Encoding.UTF8.GetString(context.PropertyName); + var name = Encoding.UTF8.GetString(context.Name); Calls.Add($"{context.Tag.Kind} {context.Path.ToString()} {(context.IsArrayItem ? "[]" : name)}"); - if (context.TokenType is JsonTokenType.String) + if (context.Kind is ValueKind.String) { - writer.WriteStringValue($"{Encoding.UTF8.GetString(context.Value)}:{name}"); + output.String($"{Encoding.UTF8.GetString(context.Value)}:{name}"); } else { - Utf8MaskStrategy.Default.Mask(context, writer); + Default.Mask(context, output); } } } - private sealed class KeyedStrategy : Utf8MaskStrategy + private sealed class KeyedStrategy : ValueMaskStrategy { - public override void Mask(ReadOnlySpan value, JsonTokenType tokenType, MaskTag tag, JsonWriter writer, JsonObserverOptions options) + public override void Mask(in MaskContext context, MaskValueWriter output) { - switch (tag.Key) + switch (context.Tag.Key) { case Classification classification: - writer.WriteStringValue(classification.Name); + output.String(classification.Name); break; case string name: - writer.WriteStringValue(name); + output.String(name); break; default: - writer.WriteStringValue("none"); + output.String("none"); break; } } } - private sealed class RecordingStrategy : Utf8MaskStrategy + private sealed class RecordingStrategy : ValueMaskStrategy { public List Calls { get; } = []; - public override void Mask(ReadOnlySpan value, JsonTokenType tokenType, MaskTag tag, JsonWriter writer, JsonObserverOptions options) + public override void Mask(in MaskContext context, MaskValueWriter output) { - Calls.Add($"{tag.Kind} {tokenType} {Encoding.UTF8.GetString(value)}"); - writer.WriteStringValue("?"); + Calls.Add($"{context.Tag.Kind} {context.Kind} {Encoding.UTF8.GetString(context.Value)}"); + output.String("?"); } } } diff --git a/DragoAnt.System.Text.Json.Observer.Tests.Shared/NameMatchingTests.cs b/DragoAnt.System.Text.Json.Observer.Tests.Shared/NameMatchingTests.cs index f527c35..366ccc5 100644 --- a/DragoAnt.System.Text.Json.Observer.Tests.Shared/NameMatchingTests.cs +++ b/DragoAnt.System.Text.Json.Observer.Tests.Shared/NameMatchingTests.cs @@ -1,6 +1,7 @@ using System.Text; -using DragoAnt.System.Text.Json.Observer.Strategies; -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; +using DragoAnt.Observer; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; namespace DragoAnt.System.Text.Json.Observer.Tests.Shared; @@ -9,13 +10,13 @@ public abstract class NameMatchingTests private static readonly JsonObserver Observer = JsonObserver.Any( _ => { }, _ => { }, - Relative(b => b - .Match("password").MaskStr((_, _) => "***") - .Match(PropMatches.OneOf("pin", "cvv")).MaskStr((_, _) => "***") - .Match(PropMatches.EndsWith("Token")).MaskStr((_, _) => "***") - .Match(PropMatches.StartsWith("secret")).MaskStr((_, _) => "***") - .Match(PropMatches.Contains("Email")).MaskStr((_, _) => "***") - .Match("card", "number").MaskStr((_, _) => "***"), + AnyDepth(b => b + .Match("password").Mask((_, _) => "***", MaskNulls.Mask) + .Match(Names.OneOf("pin", "cvv")).Mask((_, _) => "***", MaskNulls.Mask) + .Match(Names.EndsWith("Token")).Mask((_, _) => "***", MaskNulls.Mask) + .Match(Names.StartsWith("secret")).Mask((_, _) => "***", MaskNulls.Mask) + .Match(Names.Contains("Email")).Mask((_, _) => "***", MaskNulls.Mask) + .Path("card", "number").Mask((_, _) => "***", MaskNulls.Mask), BlockList)); [Theory] @@ -40,11 +41,11 @@ public void NonAsciiName_StillMatches() => public void CustomStrategy_GetsDecodedName() { var names = new List(); - var observer = JsonObserver.Obj(_ => { }, Relative(b => b.Match(new PropMatchingStrategy(n => + var observer = JsonObserver.Obj(_ => { }, AnyDepth(b => b.Match(new NameMatch(n => { names.Add(n); return false; - })).MaskStr((_, _) => "***"), BlockList)); + })).Mask((_, _) => "***", MaskNulls.Mask), BlockList)); observer.Mask("""{"a\u0062":1,"c":[2]}"""); @@ -56,12 +57,12 @@ public void OneOf_NoClosureAllocation() { var observer = JsonObserver.Obj( _ => { }, - JsonObserverValuePolicies.Relative(b => b - .Match(PropMatches.OneOf("pin", "cvv", "password")).ReadStr((_, c) => c.Hits++) + JsonValuePolicy.AnyDepth(b => b + .Match(Names.OneOf("pin", "cvv", "password")).ReadStr((_, c) => c.Hits++) .Match("accessToken").ReadStr((_, c) => c.Hits++) - .Match(PropMatches.EndsWith("Token")).ReadStr((_, c) => c.Hits++) - .Match(PropMatches.StartsWith("secret")).ReadStr((_, c) => c.Hits++) - .Match(PropMatches.Contains("email")).ReadStr((_, c) => c.Hits++))); + .Match(Names.EndsWith("Token")).ReadStr((_, c) => c.Hits++) + .Match(Names.StartsWith("secret")).ReadStr((_, c) => c.Hits++) + .Match(Names.Contains("email")).ReadStr((_, c) => c.Hits++))); var json = new StringBuilder("{"); for (var i = 0; i < 200; i++) { diff --git a/DragoAnt.System.Text.Json.Observer.Tests.Shared/NestedArrayRuleTests.cs b/DragoAnt.System.Text.Json.Observer.Tests.Shared/NestedArrayRuleTests.cs index cae9888..379a090 100644 --- a/DragoAnt.System.Text.Json.Observer.Tests.Shared/NestedArrayRuleTests.cs +++ b/DragoAnt.System.Text.Json.Observer.Tests.Shared/NestedArrayRuleTests.cs @@ -1,14 +1,15 @@ using DragoAnt.System.Text.Json.Observer.Builders; -using DragoAnt.System.Text.Json.Observer.Strategies; -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; +using DragoAnt.Observer; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; namespace DragoAnt.System.Text.Json.Observer.Tests.Shared; public abstract class NestedArrayRuleTests { - private static readonly PropMatchingStrategy AnyItem = new(_ => true); + private static readonly NameMatch AnyItem = new(_ => true); - private static void Line(JsonObjBuilder line, bool allowList) + private static void Line(JsonObjBuilder line, bool allowList) { if (allowList) { @@ -16,11 +17,11 @@ private static void Line(JsonObjBuilder line, bool } else { - line.Match("qty").MaskAny("***"); + line.Match("qty").Mask("***"); } } - private static JsonObserverValueDelegate Policy(bool allowList) => allowList ? AllowList : BlockList; + private static JsonValuePolicy Policy(bool allowList) => allowList ? AllowList : BlockList; [Theory] [InlineData(false)] @@ -73,8 +74,8 @@ public void ObjInRootArray_Unchanged(bool allowList) [Fact] public void AnyItemWorkaround_StillMatches() { - var byPath = JsonObserver.Obj(root => root.Match("lines", AnyItem, "qty").MaskAny("***"), BlockList); - var byRelative = JsonObserver.Obj(Relative(rules => rules.Match("lines", AnyItem, "qty").MaskAny("***"), BlockList)); + var byPath = JsonObserver.Obj(root => root.Path("lines", AnyItem, "qty").Mask("***"), BlockList); + var byRelative = JsonObserver.Obj(AnyDepth(rules => rules.Path("lines", AnyItem, "qty").Mask("***"), BlockList)); const string json = """{"lines":[{"qty":5,"sku":"A"}]}"""; byPath.Mask(json).Should().Be("""{"lines":[{"qty":"***","sku":"A"}]}"""); diff --git a/DragoAnt.System.Text.Json.Observer.Tests.Shared/NestingTests.cs b/DragoAnt.System.Text.Json.Observer.Tests.Shared/NestingTests.cs index 7368bbd..f3f117c 100644 --- a/DragoAnt.System.Text.Json.Observer.Tests.Shared/NestingTests.cs +++ b/DragoAnt.System.Text.Json.Observer.Tests.Shared/NestingTests.cs @@ -1,6 +1,7 @@ using System.Buffers; using System.Text; -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; namespace DragoAnt.System.Text.Json.Observer.Tests.Shared; @@ -11,7 +12,7 @@ public abstract class NestingTests private static readonly JsonObserver Observer = JsonObserver.Any( _ => { }, _ => { }, - Relative(b => b.Match("password").MaskStr((_, _) => "***"), BlockList)); + AnyDepth(b => b.Match("password").Mask((_, _) => "***", MaskNulls.Mask), BlockList)); internal static string Nested(int levels) { @@ -48,23 +49,9 @@ public void Mask_When10000Levels_ReturnsInvalid() var utf8 = Encoding.UTF8.GetBytes(Nested(10_000)); var result = Observer.Mask(utf8, new ArrayBufferWriter()); - var deep = Observer.Mask(utf8, new ArrayBufferWriter(), new JsonObserverOptions(MaxDepth: 20_000)); + var deep = Observer.Mask(utf8, new ArrayBufferWriter(), new JsonObserverOptions { MaxDepth = 20_000 }); result.Status.Should().Be(MaskStatus.Invalid); deep.Status.Should().BeOneOf(MaskStatus.Invalid, MaskStatus.Masked); } - - [Fact] - public void PropertyPath_ReturnsRentedArray() - { - var pool = ArrayPool.Shared; - var probe = pool.Rent(16); - pool.Return(probe); - - Observer.Mask("{\"password\":\"x\"}"); - - var again = pool.Rent(16); - pool.Return(again); - again.Should().BeSameAs(probe); - } } diff --git a/DragoAnt.System.Text.Json.Observer.Tests.Shared/PassThroughTests.cs b/DragoAnt.System.Text.Json.Observer.Tests.Shared/PassThroughTests.cs index 8317964..cfb582c 100644 --- a/DragoAnt.System.Text.Json.Observer.Tests.Shared/PassThroughTests.cs +++ b/DragoAnt.System.Text.Json.Observer.Tests.Shared/PassThroughTests.cs @@ -1,4 +1,5 @@ -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; namespace DragoAnt.System.Text.Json.Observer.Tests.Shared; @@ -7,7 +8,7 @@ public abstract class PassThroughTests private static readonly JsonObserver Observer = JsonObserver.Any( _ => { }, _ => { }, - Relative(b => b.Match("password").MaskStr((_, _) => "***"), BlockList)); + AnyDepth(b => b.Match("password").Mask((_, _) => "***", MaskNulls.Mask), BlockList)); [Theory] [InlineData("""{"a":1e2}""")] diff --git a/DragoAnt.System.Text.Json.Observer.Tests.Shared/PropertyPathTests.cs b/DragoAnt.System.Text.Json.Observer.Tests.Shared/PropertyPathTests.cs index ab79e2e..c1db700 100644 --- a/DragoAnt.System.Text.Json.Observer.Tests.Shared/PropertyPathTests.cs +++ b/DragoAnt.System.Text.Json.Observer.Tests.Shared/PropertyPathTests.cs @@ -1,18 +1,19 @@ using System.Text; -using DragoAnt.System.Text.Json.Observer.Strategies; -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; +using DragoAnt.Observer; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; namespace DragoAnt.System.Text.Json.Observer.Tests.Shared; public abstract class PropertyPathTests { - private static List Collect(string json, Func, JsonObserver> build) + private static List Collect(string json, Func, JsonObserver> build) { var seen = new List(); - var observer = build((ref Utf8JsonReader reader, JsonWriter writer, JsonObserveringEmptyContext _, ref PropertyPath path) => + var observer = build((ref JsonValueContext c) => { - seen.Add(path.ToString()); - writer.WriteStringValue("x"); + seen.Add(c.Path.ToString()); + c.Writer.WriteStringValue("x"); }); observer.Mask(json); return seen; @@ -23,7 +24,7 @@ public void ToString_KeepsArrayIndices() { var seen = Collect( """{"items":[{"sku":"a"},{"sku":"b"},{"sku":"c","tags":["x","y"]}],"m":[[1,2],[3]]}""", - rule => JsonObserver.Obj(Relative(b => b.Match(new PropMatchingStrategy(_ => true)).MaskValue(rule), BlockList))); + rule => JsonObserver.Obj(AnyDepth(b => b.Match(new NameMatch(_ => true)).MaskValue(rule), BlockList))); seen.Should().Equal( "items[0].sku", "items[1].sku", "items[2].sku", "items[2].tags[0]", "items[2].tags[1]", @@ -35,7 +36,7 @@ public void ToString_RootArrayAndSpecialNames() { var seen = Collect( """[{"a.b":1,"it's":2,"":3,"x[1]":4,"é":5}]""", - rule => JsonObserver.Array(root => root.Obj(o => o.Match(new PropMatchingStrategy(_ => true)).MaskValue(rule)), BlockList)); + rule => JsonObserver.Array(root => root.Obj(o => o.Match(new NameMatch(_ => true)).MaskValue(rule)), BlockList)); seen.Should().Equal("[0]['a.b']", "[0]['it\\'s']", "[0]['']", "[0]['x[1]']", "[0].é"); } @@ -44,17 +45,19 @@ public void ToString_RootArrayAndSpecialNames() public void Accessors_IndexAndUtf8Name() { var seen = new List(); - var observer = JsonObserver.Obj(Relative(b => b.Match("sku").MaskValue((ref Utf8JsonReader _, JsonWriter writer, JsonObserveringEmptyContext _, ref PropertyPath path) => + var observer = JsonObserver.Obj(AnyDepth(b => b.Match("sku").MaskValue((ref JsonValueContext c) => { - path.TryGetArrayIndex(1, out var index).Should().BeTrue(); - path.TryGetArrayIndex(0, out _).Should().BeFalse(); - path.TryGetArrayIndex(7, out _).Should().BeFalse(); - path.IsArrayItem(1).Should().BeTrue(); - path.IsArrayItem(2).Should().BeFalse(); - path.TryGetPropertyNameUtf8(2, out var name).Should().BeTrue(); - path.TryGetPropertyNameUtf8(1, out _).Should().BeFalse(); - path.TryGetPropertyNameUtf8(-1, out _).Should().BeFalse(); - path.GetPropertyName(1).Should().BeNull(); + var path = c.Path; + var writer = c.Writer; + path.TryGetItemIndex(1, out var index).Should().BeTrue(); + path.TryGetItemIndex(0, out _).Should().BeFalse(); + path.TryGetItemIndex(7, out _).Should().BeFalse(); + path.IsItem(1).Should().BeTrue(); + path.IsItem(2).Should().BeFalse(); + path.TryGetName(2, out var name).Should().BeTrue(); + path.TryGetName(1, out _).Should().BeFalse(); + path.TryGetName(-1, out _).Should().BeFalse(); + path.GetName(1).Should().BeNull(); seen.Add($"{index}:{Encoding.UTF8.GetString(name)}"); writer.WriteStringValue("x"); }), BlockList)); @@ -66,9 +69,9 @@ public void Accessors_IndexAndUtf8Name() [Fact] public void Matching_ArrayItemSegment_StillOneLevel() { - var anyItem = new PropMatchingStrategy(n => n is null); + var anyItem = new NameMatch(n => n is null); - JsonObserver.Obj(root => root.Match("lines", anyItem, "qty").MaskAny("***"), BlockList) + JsonObserver.Obj(root => root.Path("lines", anyItem, "qty").Mask("***"), BlockList) .Mask("""{"lines":[{"qty":1},{"qty":2}],"qty":3}""") .Should().Be("""{"lines":[{"qty":"***"},{"qty":"***"}],"qty":3}"""); } diff --git a/DragoAnt.System.Text.Json.Observer.Tests.Shared/RelativeNullTests.cs b/DragoAnt.System.Text.Json.Observer.Tests.Shared/RelativeNullTests.cs index 5034b29..30ae134 100644 --- a/DragoAnt.System.Text.Json.Observer.Tests.Shared/RelativeNullTests.cs +++ b/DragoAnt.System.Text.Json.Observer.Tests.Shared/RelativeNullTests.cs @@ -1,16 +1,17 @@ -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; namespace DragoAnt.System.Text.Json.Observer.Tests.Shared; public abstract class RelativeNullTests { - private static string? Mark(string? value, JsonObserveringEmptyContext _) => value is null ? "was-null" : "x"; + private static string? Mark(string? value, NoContext _) => value is null ? "was-null" : "x"; [Fact] public void MaskStr_Null_CalledForRelativeLikeAbsolute() { - var absolute = JsonObserver.Obj(root => root.Match("a").Obj(a => a.Match("p").MaskStr(Mark)), BlockList); - var relative = JsonObserver.Obj(Relative(rules => rules.Match("p").MaskStr(Mark), BlockList)); + var absolute = JsonObserver.Obj(root => root.Match("a").Obj(a => a.Match("p").Mask(Mark, MaskNulls.Mask)), BlockList); + var relative = JsonObserver.Obj(AnyDepth(rules => rules.Match("p").Mask(Mark, MaskNulls.Mask), BlockList)); const string json = """{"a":{"p":null,"q":null}}"""; absolute.Mask(json).Should().Be("""{"a":{"p":"was-null","q":null}}"""); @@ -19,17 +20,17 @@ public void MaskStr_Null_CalledForRelativeLikeAbsolute() [Fact] public void TypedMask_Null_CalledForRelative() => - JsonObserver.Obj(Relative(rules => rules + JsonObserver.Obj(AnyDepth(rules => rules .Match("i").MaskInt((v, _) => v is null ? "i-null" : "i") .Match("b").MaskBool((v, _) => v is null ? "b-null" : "b") - .Match("r").MaskRawValue((v, _) => v is null ? "r-null" : "r"), + .Match("r").Mask((v, _) => v is null ? "r-null" : "r", MaskNulls.Mask), BlockList)) .Mask("""{"x":{"i":null,"b":null,"r":null}}""") .Should().Be("""{"x":{"i":"i-null","b":"b-null","r":"r-null"}}"""); [Fact] public void MaskAny_Null_StaysNullForRelative() => - JsonObserver.Obj(Relative(rules => rules.Match("p").MaskAny((_, _) => "called"), BlockList)) + JsonObserver.Obj(AnyDepth(rules => rules.Match("p").Mask((_, _) => "called"), BlockList)) .Mask("""{"p":null,"o":{"p":null}}""") .Should().Be("""{"p":null,"o":{"p":null}}"""); @@ -37,9 +38,9 @@ public void MaskAny_Null_StaysNullForRelative() => public void ReadStr_Null_ReadForRelative() { var context = new Holder(); - var observer = JsonObserver.Obj(JsonObserverValuePolicies.Relative( + var observer = JsonObserver.Obj(JsonValuePolicy.AnyDepth( rules => rules.Match("p").ReadStr((v, c) => c.Calls.Add(v ?? "")), - JsonObserverValuePolicies.BlockList)); + ValuePolicy.BlockList)); observer.Mask("""{"p":null,"o":{"p":"v"}}""", context).Should().Be("""{"p":null,"o":{"p":"v"}}"""); context.Calls.Should().Equal("", "v"); @@ -47,7 +48,7 @@ public void ReadStr_Null_ReadForRelative() [Fact] public void Null_NoRelativeRule_KeptByDefaultPolicy() => - JsonObserver.Obj(Relative(rules => rules.Match("p").MaskStr(Mark), AllowList)) + JsonObserver.Obj(AnyDepth(rules => rules.Match("p").Mask(Mark, MaskNulls.Mask), AllowList)) .Mask("""{"q":null,"s":"x"}""") .Should().Be("""{"q":null,"s":"***"}"""); diff --git a/DragoAnt.System.Text.Json.Observer.Tests.Shared/RobustnessTests.cs b/DragoAnt.System.Text.Json.Observer.Tests.Shared/RobustnessTests.cs index d7124e4..c93dad0 100644 --- a/DragoAnt.System.Text.Json.Observer.Tests.Shared/RobustnessTests.cs +++ b/DragoAnt.System.Text.Json.Observer.Tests.Shared/RobustnessTests.cs @@ -1,8 +1,9 @@ using System.Buffers; using System.Text; using System.Text.Json.Serialization.Metadata; -using DragoAnt.System.Text.Json.Observer.Strategies; -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; +using DragoAnt.Observer; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; namespace DragoAnt.System.Text.Json.Observer.Tests.Shared; @@ -13,7 +14,7 @@ public abstract class RobustnessTests private static readonly JsonObserver Observer = JsonObserver.Any( _ => { }, _ => { }, - Relative(b => b.Match("password").MaskAny("***").Match("pin").MaskStr("***"), BlockList)); + AnyDepth(b => b.Match("password").Mask("***").Match("pin").Mask("***", MaskNulls.Mask), BlockList)); private static readonly JsonObserver AllowListObserver = JsonObserver.Any( o => o.Match("user").Unmasked().Match("ok").Unmasked(), @@ -96,7 +97,7 @@ public void MaxDepthOption_Respected(int maxDepth, MaskStatus expected) { var json = new string('[', 20) + new string(']', 20); - BytesApiTests.Mask(Observer, json, new JsonObserverOptions(MaxDepth: maxDepth)).Result.Status.Should().Be(expected); + BytesApiTests.Mask(Observer, json, new JsonObserverOptions { MaxDepth = maxDepth }).Result.Status.Should().Be(expected); } [Fact] @@ -113,7 +114,7 @@ public void MaxOutputBytes_ExactOutputLength_Masked() const string json = """{"password":"x","n":1}"""; var full = BytesApiTests.Mask(Observer, json).Output; - var (result, output) = BytesApiTests.Mask(Observer, json, new JsonObserverOptions(MaxOutputBytes: Encoding.UTF8.GetByteCount(full))); + var (result, output) = BytesApiTests.Mask(Observer, json, new JsonObserverOptions { MaxOutputBytes = Encoding.UTF8.GetByteCount(full) }); result.Status.Should().Be(MaskStatus.Masked); output.Should().Be(full); @@ -146,7 +147,7 @@ public void Concurrency_SharedObserver_BytesApi_SameOutput() public void Concurrency_SharedShapeObserver_SameOutput() { var observer = JsonObserver.FromShape(JsonShape.Object(("id", JsonShape.Scalar), ("password", JsonShape.Masked(MaskTag.Hash)))); - var options = new JsonObserverOptions(HashKey: "key"u8.ToArray()); + var options = new JsonObserverOptions { HashKey = "key"u8.ToArray() }; var expected = observer.Mask("""{"id":1,"password":"p","x":2}""", options); var results = new string?[8 * 500]; diff --git a/DragoAnt.System.Text.Json.Observer.Tests.Shared/RuleCoverageTests.cs b/DragoAnt.System.Text.Json.Observer.Tests.Shared/RuleCoverageTests.cs index af3ea31..ef36ea4 100644 --- a/DragoAnt.System.Text.Json.Observer.Tests.Shared/RuleCoverageTests.cs +++ b/DragoAnt.System.Text.Json.Observer.Tests.Shared/RuleCoverageTests.cs @@ -1,7 +1,8 @@ using System.Globalization; using System.Text.RegularExpressions; -using DragoAnt.System.Text.Json.Observer.Strategies; -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; +using DragoAnt.Observer; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; namespace DragoAnt.System.Text.Json.Observer.Tests.Shared; @@ -44,23 +45,23 @@ public void TypedStrategies_NullResult_WritesNull() [Fact] public void PropMatchesRegex_MatchesName() - => JsonObserver.Obj(b => b.Match(PropMatches.Regex(new Regex("^pass", RegexOptions.IgnoreCase))).MaskStr("***"), BlockList) + => JsonObserver.Obj(b => b.Match(Names.Regex(new Regex("^pass", RegexOptions.IgnoreCase))).Mask("***", MaskNulls.Mask), BlockList) .Mask("""{"Password":"p","passcode":"c","bypass":"b"}""").Should().Be("""{"Password":"***","passcode":"***","bypass":"b"}"""); [Fact] public void PropMatchesOneOf_InAbsoluteRule() - => JsonObserver.Obj(b => b.Match(PropMatches.OneOf("pin", "cvv")).MaskAny("***"), BlockList) + => JsonObserver.Obj(b => b.Match(Names.OneOf("pin", "cvv")).Mask("***"), BlockList) .Mask("""{"PIN":1,"cvv":2,"n":3}""").Should().Be("""{"PIN":"***","cvv":"***","n":3}"""); [Fact] public void StringMaskingStrategy_RegexReplacementAndEvaluator() { var digits = new Regex("[0-9]"); - var replaced = JsonObserver.Obj(b => b.Match("a").MaskStr(StringMaskingStrategy.Regex(digits, "#")), BlockList); + var replaced = JsonObserver.Obj(b => b.Match("a").Mask(StringMaskingStrategy.Regex(digits, "#"), MaskNulls.Mask), BlockList); var evaluated = JsonObserver.Obj( - b => b.Match("a").MaskStr(StringMaskingStrategy.Regex(digits, m => m.Index < 2 ? m.Value : "*")), + b => b.Match("a").Mask(StringMaskingStrategy.Regex(digits, m => m.Index < 2 ? m.Value : "*"), MaskNulls.Mask), BlockList); - var implicitRegex = JsonObserver.Obj(b => b.Match("a").MaskStr(digits), BlockList); + var implicitRegex = JsonObserver.Obj(b => b.Match("a").Mask(digits, MaskNulls.Mask), BlockList); replaced.Mask("""{"a":"ab12"}""").Should().Be("""{"a":"ab##"}"""); evaluated.Mask("""{"a":"1234"}""").Should().Be("""{"a":"12**"}"""); @@ -72,9 +73,8 @@ public void StringMaskingStrategy_RegexReplacementAndEvaluator() public void CustomDelegate_PropertyPathApi() { var seen = new List(); - var observer = JsonObserver.Obj(Relative(b => b.Match("c").MaskValue((ref Utf8JsonReader _, JsonWriter writer, JsonObserveringEmptyContext _, ref PropertyPath path) => - { - seen.Add($"{path.Length}|{path.GetPropertyName(0)}|{path.GetPropertyNameReverse(0)}|{path.GetPropertyNameReverse(1)}|{path.ToString()}|{path.GetPropertyName(9)}"); + var observer = JsonObserver.Obj(AnyDepth(b => b.Match("c").MaskValue((ref JsonValueContext __c) => { var writer = __c.Writer; var path = __c.Path; + seen.Add($"{path.Length}|{path.GetName(0)}|{path.GetNameFromEnd(0)}|{path.GetNameFromEnd(1)}|{path.ToString()}|{path.GetName(9)}"); writer.WriteStringValue("x"); }), BlockList)); @@ -95,7 +95,7 @@ public void NullList_KeepsStructure() [Fact] public void Last4_LongStringAndNumber_ShowTail() { - var observer = JsonObserver.Obj(Relative(b => b.Match("c").MaskAny(MaskTag.Last4), BlockList)); + var observer = JsonObserver.Obj(AnyDepth(b => b.Match("c").Mask(MaskTag.Last4), BlockList)); observer.Mask("""{"c":"4111111111111111"}""").Should().Be("""{"c":"***1111"}"""); observer.Mask("""{"c":4111111111111111}""").Should().Be("""{"c":"***1111"}"""); @@ -106,14 +106,14 @@ public void Last4_LongStringAndNumber_ShowTail() [InlineData("1234567", "***")] [InlineData("12345678", "***5678")] public void Last4_ShortValue_MaskedFully(string value, string expected) - => JsonObserver.Obj(Relative(b => b.Match("c").MaskAny(MaskTag.Last4), BlockList)) + => JsonObserver.Obj(AnyDepth(b => b.Match("c").Mask(MaskTag.Last4), BlockList)) .Mask($$"""{"c":"{{value}}"}""").Should().Be($$"""{"c":"{{expected}}"}"""); [Fact] public void Hash_NumberLiteralAndStringOfSameText_Equal() { - var observer = JsonObserver.Obj(Relative(b => b.Match("h").MaskAny(MaskTag.Hash), BlockList)); - var options = new JsonObserverOptions(HashKey: "k"u8.ToArray()); + var observer = JsonObserver.Obj(AnyDepth(b => b.Match("h").Mask(MaskTag.Hash), BlockList)); + var options = new JsonObserverOptions { HashKey = "k"u8.ToArray() }; var number = observer.Mask("""{"h":1}""", options); var text = observer.Mask("""{"h":"1"}""", options); @@ -125,22 +125,22 @@ public void Hash_NumberLiteralAndStringOfSameText_Equal() [Fact] public void Omit_InArray_WritesNull() - => JsonObserver.Array(a => a.MaskAny(MaskTag.Omit)).Mask("""["secret",1,{"a":1}]""").Should().Be("[null,null,null]"); + => JsonObserver.Array(a => a.Mask(MaskTag.Null)).Mask("""["secret",1,{"a":1}]""").Should().Be("[null,null,null]"); [Fact] - public void CustomUtf8MaskStrategy_ThatThrows_InvalidNoLeak() + public void CustomValueMaskStrategy_ThatThrows_InvalidNoLeak() { - var observer = JsonObserver.Obj(Relative(b => b.Match("p").MaskAny(MaskTag.Full), BlockList)); + var observer = JsonObserver.Obj(AnyDepth(b => b.Match("p").Mask(MaskTag.Full), BlockList)); - var output = observer.Mask("""{"a":1,"p":"secret","b":2}""", out var result, new JsonObserverOptions(MaskStrategy: new ThrowingStrategy())); + var output = observer.Mask("""{"a":1,"p":"secret","b":2}""", out var result, new JsonObserverOptions { Strategy = new ThrowingStrategy() }); result.Status.Should().Be(MaskStatus.Invalid); output.Should().Be("""{"a":1}"""); } - private sealed class ThrowingStrategy : Utf8MaskStrategy + private sealed class ThrowingStrategy : ValueMaskStrategy { - public override void Mask(ReadOnlySpan value, JsonTokenType tokenType, MaskTag tag, JsonWriter writer, JsonObserverOptions options) + public override void Mask(in MaskContext context, MaskValueWriter output) => throw new InvalidOperationException("boom"); } } diff --git a/DragoAnt.System.Text.Json.Observer.Tests.Shared/SensitiveRuleTests.cs b/DragoAnt.System.Text.Json.Observer.Tests.Shared/SensitiveRuleTests.cs index 27b9614..19fe890 100644 --- a/DragoAnt.System.Text.Json.Observer.Tests.Shared/SensitiveRuleTests.cs +++ b/DragoAnt.System.Text.Json.Observer.Tests.Shared/SensitiveRuleTests.cs @@ -1,16 +1,17 @@ -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; namespace DragoAnt.System.Text.Json.Observer.Tests.Shared; public abstract class SensitiveRuleTests { - private static readonly JsonObserver AbsoluteMaskStr = JsonObserver.Obj(b => b.Match("pin").MaskStr("***"), BlockList); + private static readonly JsonObserver AbsoluteMaskStr = JsonObserver.Obj(b => b.Match("pin").Mask("***", MaskNulls.Mask), BlockList); - private static readonly JsonObserver RelativeMaskStr = JsonObserver.Obj(Relative(b => b.Match("pin").MaskStr("***"), BlockList)); + private static readonly JsonObserver RelativeMaskStr = JsonObserver.Obj(AnyDepth(b => b.Match("pin").Mask("***", MaskNulls.Mask), BlockList)); - private static readonly JsonObserver AbsoluteMaskRaw = JsonObserver.Obj(b => b.Match("pin").MaskRawValue((_, _) => "***"), BlockList); + private static readonly JsonObserver AbsoluteMaskRaw = JsonObserver.Obj(b => b.Match("pin").Mask((_, _) => "***", MaskNulls.Mask), BlockList); - private static readonly JsonObserver RelativeMaskRaw = JsonObserver.Obj(Relative(b => b.Match("pin").MaskRawValue((_, _) => "***"), BlockList)); + private static readonly JsonObserver RelativeMaskRaw = JsonObserver.Obj(AnyDepth(b => b.Match("pin").Mask((_, _) => "***", MaskNulls.Mask), BlockList)); public static TheoryData SensitiveValues => new() { @@ -66,11 +67,11 @@ public void TypedMask_OtherValueTypes_NeverExposed(string rule) public void MaskStr_StrategyInput_PerValueType() { var seen = new List(); - var observer = JsonObserver.Obj(b => b.Match("pin").MaskStr((v, _) => + var observer = JsonObserver.Obj(b => b.Match("pin").Mask((v, _) => { seen.Add(v); return "***"; - }), BlockList); + }, MaskNulls.Mask), BlockList); observer.Mask("""{"pin":"a\u0062c"}"""); observer.Mask("""{"pin":12.50}"""); @@ -83,7 +84,7 @@ public void MaskStr_StrategyInput_PerValueType() [Fact] public void MaskStr_NullStrategyResult_WritesNull() - => JsonObserver.Obj(b => b.Match("pin").MaskStr((_, _) => null), BlockList) + => JsonObserver.Obj(b => b.Match("pin").Mask((_, _) => null, MaskNulls.Mask), BlockList) .Mask("""{"pin":[1,2]}""").Should().Be("""{"pin":null}"""); [Fact] diff --git a/DragoAnt.System.Text.Json.Observer.Tests.Shared/SequenceInputTests.cs b/DragoAnt.System.Text.Json.Observer.Tests.Shared/SequenceInputTests.cs index 0a5fcec..9128677 100644 --- a/DragoAnt.System.Text.Json.Observer.Tests.Shared/SequenceInputTests.cs +++ b/DragoAnt.System.Text.Json.Observer.Tests.Shared/SequenceInputTests.cs @@ -1,7 +1,8 @@ using System.Buffers; using System.Text; -using DragoAnt.System.Text.Json.Observer.Strategies; -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; +using DragoAnt.Observer; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; namespace DragoAnt.System.Text.Json.Observer.Tests.Shared; @@ -11,23 +12,23 @@ namespace DragoAnt.System.Text.Json.Observer.Tests.Shared; /// public abstract class SequenceInputTests { - private static readonly PropMatchingStrategy AnyItem = new(_ => true); + private static readonly NameMatch AnyItem = new(_ => true); - private static readonly JsonObserver Tags = JsonObserver.Obj(Relative(b => b - .Match("full").MaskAny(MaskTag.Full) - .Match("last4").MaskAny(MaskTag.Last4) - .Match("hash").MaskAny(MaskTag.Hash) - .Match("omit").MaskAny(MaskKind.Omit), + private static readonly JsonObserver Tags = JsonObserver.Obj(AnyDepth(b => b + .Match("full").Mask(MaskTag.Full) + .Match("last4").Mask(MaskTag.Last4) + .Match("hash").Mask(MaskTag.Hash) + .Match("omit").Mask(MaskKind.Null), BlockList)); private static readonly JsonObserver Nested = JsonObserver.Obj( - root => root.Match("lines").Array(l => l.Obj(x => x.Match("qty").MaskAny("***").Match("sku").MaskStr((v, _) => v + "!"))), - Relative(b => b.Match("lines", AnyItem, "note").MaskRawValue((v, _) => "<" + v + ">"), BlockList)); + root => root.Match("lines").Array(l => l.Obj(x => x.Match("qty").Mask("***").Match("sku").Mask((v, _) => v + "!", MaskNulls.Mask))), + AnyDepth(b => b.Path("lines", AnyItem, "note").Mask((v, _) => "<" + v + ">", MaskNulls.Mask), BlockList)); - private static readonly JsonObserver NonAscii = JsonObserver.Obj(Relative(b => b - .Match("пароль").MaskAny("1") - .Match(PropMatches.Contains("ключ")).MaskInt((v, _) => $"{v}") - .Match("password").MaskAny("2"), + private static readonly JsonObserver NonAscii = JsonObserver.Obj(AnyDepth(b => b + .Match("пароль").Mask("1") + .Match(Names.Contains("ключ")).MaskInt((v, _) => $"{v}") + .Match("password").Mask("2"), AllowList)); private static readonly Dictionary Cases = BuildCases(); @@ -37,7 +38,7 @@ public abstract class SequenceInputTests var cases = new Dictionary { ["golden-request"] = (JsonMaskingTests.GetRequestMasking(BlockList), JsonMaskingTests.TestJson, null), - ["golden-ignore-nulls"] = (JsonMaskingTests.GetRequestUnmasking(NullList), JsonMaskingTests.TestJson, new JsonObserverOptions(IgnoreNulls: true, Indented: true)), + ["golden-ignore-nulls"] = (JsonMaskingTests.GetRequestUnmasking(NullList), JsonMaskingTests.TestJson, new JsonObserverOptions { IgnoreNulls = true, Indented = true }), ["tags"] = (Tags, """{"full":{"a":[1,2]},"last4":"4111111111111111","hash":"S3cr3t","omit":12.5e3,"x":"y"}""", null), ["shape"] = (JsonShapeTests.Observer(), """{"id":1,"orders":[{"sku":"A1","secretCode":"x","extra":1}],"byCode":{"K1":{"sku":"B"}},"card":"4111111111111111","e_mail":"a@b.c","unknown":{"deep":true}}""", null), ["nested-array"] = (Nested, """{"lines":[{"qty":5,"sku":"A","note":"n\"1"},{"qty":-7.25,"sku":"Bé"}],"total":12}""", null), @@ -47,9 +48,9 @@ public abstract class SequenceInputTests ["truncated"] = (BytesApiTests.Observer, """{"user":"bob","password":"S3cr3t","card":{"pin":"123""", null), ["invalid"] = (BytesApiTests.Observer, """{"user":"bob",,"password":"x"}""", null), ["not-json"] = (BytesApiTests.Observer, " 42", null), - ["max-output"] = (BytesApiTests.Observer, BytesApiTests.Payloads[0], new JsonObserverOptions(MaxOutputBytes: 60)), - ["max-value"] = (BytesApiTests.Observer, """{"text":"éééééééé","password":"x"}""", new JsonObserverOptions(MaxValueBytes: 5)), - ["case-sensitive"] = (NonAscii, """{"PASSWORD":"x","password":"y"}""", new JsonObserverOptions(PropertyNameCaseInsensitive: false)), + ["max-output"] = (BytesApiTests.Observer, BytesApiTests.Payloads[0], new JsonObserverOptions { MaxOutputBytes = 60 }), + ["max-value"] = (BytesApiTests.Observer, """{"text":"éééééééé","password":"x"}""", new JsonObserverOptions { MaxValueBytes = 5 }), + ["case-sensitive"] = (NonAscii, """{"PASSWORD":"x","password":"y"}""", new JsonObserverOptions { NameCaseInsensitive = false }), ["nested-17"] = (BytesApiTests.Observer, NestingTests.Nested(17).Replace("password", "pin", StringComparison.Ordinal), null), }; @@ -124,7 +125,7 @@ public void Mask_SingleSegmentAndEmpty_SameAsSpan() .Should().Be(BytesApiTests.Observer.Mask(utf8, spanOutput)); sequenceOutput.WrittenSpan.SequenceEqual(spanOutput.WrittenSpan).Should().BeTrue(); BytesApiTests.Observer.Mask(ReadOnlySequence.Empty, new ArrayBufferWriter()) - .Should().Be(new MaskResult(MaskStatus.NotJson, 0, 0)); + .Should().Be(new MaskResult { Status = MaskStatus.Unrecognized, BytesWritten = 0, FailedAtByte = 0 }); BytesApiTests.Observer.Mask(Split([0xEF, 0xBB], 1), new ArrayBufferWriter()) .Should().Be(BytesApiTests.Observer.Mask([0xEF, 0xBB], new ArrayBufferWriter())); } @@ -161,8 +162,8 @@ public void Read_SingleSegment_UsesSpanPath() context.Values.Should().Equal("a"); } - private static JsonObserverValueDelegate ReadRules(Action> init) => - JsonObserverValuePolicies.Relative(init, JsonObserverValuePolicies.BlockList); + private static JsonValuePolicy ReadRules(Action> init) => + JsonValuePolicy.AnyDepth(init, ValuePolicy.BlockList); public sealed class Extracted { diff --git a/DragoAnt.System.Text.Json.Observer.Tests.Shared/ShapeCoverageTests.cs b/DragoAnt.System.Text.Json.Observer.Tests.Shared/ShapeCoverageTests.cs index 4f4a5d3..d8a7cc9 100644 --- a/DragoAnt.System.Text.Json.Observer.Tests.Shared/ShapeCoverageTests.cs +++ b/DragoAnt.System.Text.Json.Observer.Tests.Shared/ShapeCoverageTests.cs @@ -1,7 +1,7 @@ using System.Collections.Immutable; using System.Text.Json.Serialization; using System.Text.Json.Serialization.Metadata; -using DragoAnt.System.Text.Json.Observer.Strategies; +using DragoAnt.Observer; namespace DragoAnt.System.Text.Json.Observer.Tests.Shared; @@ -15,7 +15,7 @@ private static JsonObserver Observer(JsonSerializerOptions? options = null, J private static MaskTag? Classify(JsonPropertyInfo property) => property.AttributeProvider?.GetCustomAttributes(typeof(JsonShapeTests.SensitiveAttribute), true) .OfType().FirstOrDefault() is { } sensitive - ? new MaskTag(sensitive.Kind) + ? MaskTag.Create(sensitive.Kind) : null; [Fact] diff --git a/DragoAnt.System.Text.Json.Observer.Tests.Shared/StringApiTests.cs b/DragoAnt.System.Text.Json.Observer.Tests.Shared/StringApiTests.cs index 678e541..81b0125 100644 --- a/DragoAnt.System.Text.Json.Observer.Tests.Shared/StringApiTests.cs +++ b/DragoAnt.System.Text.Json.Observer.Tests.Shared/StringApiTests.cs @@ -1,4 +1,5 @@ -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; namespace DragoAnt.System.Text.Json.Observer.Tests.Shared; @@ -9,7 +10,7 @@ public abstract class StringApiTests private static readonly JsonObserver Observer = JsonObserver.Any( _ => { }, _ => { }, - Relative(b => b.Match("password").MaskAny("***"), BlockList)); + AnyDepth(b => b.Match("password").Mask("***"), BlockList)); public static TheoryData Inputs => [ @@ -79,7 +80,7 @@ public void Mask_InvalidUtf16Surrogate_NeverThrows() [InlineData("""{"password":"x"}""", MaskStatus.Masked, -1)] [InlineData("""{"password":"x","a":[1""", MaskStatus.Truncated, 21)] [InlineData("""{"a": }""", MaskStatus.Invalid, 6)] - [InlineData("42", MaskStatus.NotJson, 0)] + [InlineData("42", MaskStatus.Unrecognized, 0)] public void Mask_OutResult_ReportsStatus(string json, MaskStatus status, long failedAt) { var output = Observer.Mask(json, out var result); @@ -93,14 +94,14 @@ public void Mask_OutResult_ReportsStatus(string json, MaskStatus status, long fa public void Mask_Null_ReturnsNullAndNotJson() { Observer.Mask(null, out var result).Should().BeNull(); - result.Status.Should().Be(MaskStatus.NotJson); + result.Status.Should().Be(MaskStatus.Unrecognized); } [Fact] public void Mask_UsesOptions_LikeBytesApi() { - var observer = JsonObserver.Obj(Relative(b => b.Match("h").MaskAny(Strategies.MaskTag.Hash), BlockList)); - var options = new JsonObserverOptions(HashKey: "k"u8.ToArray(), MaxValueBytes: 3); + var observer = JsonObserver.Obj(AnyDepth(b => b.Match("h").Mask(MaskTag.Hash), BlockList)); + var options = new JsonObserverOptions { HashKey = "k"u8.ToArray(), MaxValueBytes = 3 }; const string json = """{"h":"secret","note":"abcdef"}"""; var text = observer.Mask(json, out var result, options); @@ -114,22 +115,22 @@ public void Mask_UsesOptions_LikeBytesApi() public void IgnoreNulls_DropsNullPropertiesItemsAndEmptiedContainers() => JsonObserver.Obj(BlockList).Mask( """{"a":null,"b":{"c":null},"d":[null,1,null],"e":[null],"f":{"g":{"h":null}},"i":"x"}""", - new JsonObserverOptions(IgnoreNulls: true)) + new JsonObserverOptions { IgnoreNulls = true }) .Should().Be("""{"d":[1],"i":"x"}"""); [Fact] public void IgnoreNulls_AllNull_KeepsEmptyRoot() - => JsonObserver.Obj(BlockList).Mask("""{"a":null}""", new JsonObserverOptions(IgnoreNulls: true)).Should().Be("{}"); + => JsonObserver.Obj(BlockList).Mask("""{"a":null}""", new JsonObserverOptions { IgnoreNulls = true }).Should().Be("{}"); [Fact] public void IgnoreNulls_OmitTag_DropsProperty() - => JsonObserver.Obj(Relative(b => b.Match("p").MaskAny(Strategies.MaskTag.Omit), BlockList)) - .Mask("""{"p":"x","q":1}""", new JsonObserverOptions(IgnoreNulls: true)).Should().Be("""{"q":1}"""); + => JsonObserver.Obj(AnyDepth(b => b.Match("p").Mask(MaskTag.Null), BlockList)) + .Mask("""{"p":"x","q":1}""", new JsonObserverOptions { IgnoreNulls = true }).Should().Be("""{"q":1}"""); [Fact] public void Indented_TruncatedOutput_StillValidJson() { - var output = Observer.Mask($$"""{"user":"bob","password":"{{Secret}}","a":{"b":[1,2""", new JsonObserverOptions(Indented: true)); + var output = Observer.Mask($$"""{"user":"bob","password":"{{Secret}}","a":{"b":[1,2""", new JsonObserverOptions { Indented = true }); output.Should().Contain(Environment.NewLine).And.NotContain(Secret); JsonDocument.Parse(output!).RootElement.GetProperty("a").GetProperty("b").GetArrayLength().Should().Be(1); @@ -140,7 +141,7 @@ public void Indented_TruncatedOutput_StillValidJson() [InlineData("""{"user":"bob","x":""", MaskStatus.Truncated, 1)] [InlineData("""{"x": ],"user":"bob"}""", MaskStatus.Invalid, 0)] [InlineData("[]", MaskStatus.Invalid, 0)] - [InlineData("1", MaskStatus.NotJson, 0)] + [InlineData("1", MaskStatus.Unrecognized, 0)] public void Read_ReportsStatus(string json, MaskStatus status, int count) { var observer = JsonObserver.Obj(b => b.Match("user").ReadStr((_, c) => c.Count++)); diff --git a/DragoAnt.System.Text.Json.Observer.Tests.Shared/WholeValueMaskingTests.cs b/DragoAnt.System.Text.Json.Observer.Tests.Shared/WholeValueMaskingTests.cs index 6f2a756..db4e47c 100644 --- a/DragoAnt.System.Text.Json.Observer.Tests.Shared/WholeValueMaskingTests.cs +++ b/DragoAnt.System.Text.Json.Observer.Tests.Shared/WholeValueMaskingTests.cs @@ -1,4 +1,5 @@ -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; namespace DragoAnt.System.Text.Json.Observer.Tests.Shared; @@ -7,10 +8,10 @@ public abstract class WholeValueMaskingTests private static readonly JsonObserver Relatively = JsonObserver.Any( _ => { }, _ => { }, - Relative(b => b.Match("secret").MaskAny("***"), BlockList)); + AnyDepth(b => b.Match("secret").Mask("***"), BlockList)); private static readonly JsonObserver Absolutely = JsonObserver.Obj( - b => b.Match("secret").MaskAny("***").Match("list").Array(a => a.MaskAny("***")), + b => b.Match("secret").Mask("***").Match("list").Array(a => a.Mask("***")), BlockList); [Fact] @@ -41,7 +42,7 @@ public void AbsoluteRule_MasksAnyValue() => [Fact] public void Strategy_ReceivesScalarTextAndNullForContainer() { - var observer = JsonObserver.Obj(_ => { }, Relative(b => b.Match("v").MaskAny((v, _) => $"[{v ?? "container"}]"), BlockList)); + var observer = JsonObserver.Obj(_ => { }, AnyDepth(b => b.Match("v").Mask((v, _) => $"[{v ?? "container"}]"), BlockList)); observer.Mask("""{"a":{"v":42},"b":{"v":"text"},"c":{"v":false},"d":{"v":[1]}}""") .Should().Be("""{"a":{"v":"[42]"},"b":{"v":"[text]"},"c":{"v":"[false]"},"d":{"v":"[container]"}}"""); @@ -50,7 +51,7 @@ public void Strategy_ReceivesScalarTextAndNullForContainer() [Fact] public void MaskRawValue_SensitiveBool_Masked() { - var observer = JsonObserver.Obj(_ => { }, Relative(b => b.Match("pin").MaskRawValue((_, _) => "***"), BlockList)); + var observer = JsonObserver.Obj(_ => { }, AnyDepth(b => b.Match("pin").Mask((_, _) => "***", MaskNulls.Mask), BlockList)); observer.Mask("""{"pin":true}""").Should().Be("""{"pin":"***"}"""); } diff --git a/DragoAnt.System.Text.Json.Observer/BoundedJsonWriter.cs b/DragoAnt.System.Text.Json.Observer/BoundedJsonWriter.cs index af0005e..c7cf94c 100644 --- a/DragoAnt.System.Text.Json.Observer/BoundedJsonWriter.cs +++ b/DragoAnt.System.Text.Json.Observer/BoundedJsonWriter.cs @@ -47,7 +47,7 @@ private BoundedJsonWriter(JsonObserverOptions options) public bool Exhausted { get; private set; } /// - /// At least one string value was cut to . + /// At least one string value was cut to . /// public bool ValuesTruncated { get; private set; } diff --git a/DragoAnt.System.Text.Json.Observer/Builders/JsonAnyDepthBuilder.cs b/DragoAnt.System.Text.Json.Observer/Builders/JsonAnyDepthBuilder.cs new file mode 100644 index 0000000..6184fe8 --- /dev/null +++ b/DragoAnt.System.Text.Json.Observer/Builders/JsonAnyDepthBuilder.cs @@ -0,0 +1,316 @@ +namespace DragoAnt.System.Text.Json.Observer.Builders; + +/// +/// Rules of an any-depth policy (): +/// tests a property's own name and the last levels of its path, wherever it is +/// nested. The first rule that matches wins. +/// +/// Type that read rules write extracted values to. +public readonly struct JsonAnyDepthBuilder +{ + private readonly List> _policies = []; + private readonly JsonValuePolicy _fallback; + + internal JsonAnyDepthBuilder(JsonValuePolicy fallback) + { + _fallback = fallback; + } + + /// + /// Starts a rule for every property with this name, at any depth; finish it with a rule method. + /// + /// Property name test: a string for an exact name, or one of . + public PropertyMaskingStrategyBuilder Match(NameMatch match) => + new(this, new NamePathMatch([match], isPath: false)); + + /// + /// Starts a rule for the properties whose path ends with these names, at any depth; finish it with a rule method. + /// + /// Name tests for the last levels of the path, for example "card", "number" for any …card.number. + public PropertyMaskingStrategyBuilder Path(params NameMatch[] path) => + new(this, new NamePathMatch(path, isPath: true)); + + internal static ObserveRule Build(JsonAnyDepthBuilder builder) => builder.Build(); + + internal static JsonObserverItem[] BuildItems(JsonAnyDepthBuilder builder) => [.. builder._policies]; + + private JsonAnyDepthBuilder AddAnyProp(JsonPropertyPathMatchDelegate propNameMatch, ObserveRule policy, RuleInfo info) + { + _policies.Add(new JsonObserverItem((int depth, ref JsonWalk walk, JsonTokenType _) => propNameMatch(depth, ref walk), policy) { Info = info }); + return this; + } + + private JsonAnyDepthBuilder AddValueProp(JsonPropertyPathMatchDelegate propNameMatch, ObserveRule policy, RuleInfo info) + { + _policies.Add(new JsonObserverItem(ValueMatch(propNameMatch), policy) { Info = info }); + return this; + } + + private JsonAnyDepthBuilder AddReadProp(JsonPropertyPathMatchDelegate propNameMatch, JsonObserverItem.ReadValue read, RuleInfo info) + { + _policies.Add(JsonObserverItem.Read(ValueMatch(propNameMatch), read, info)); + return this; + } + + private static JsonPropertyMatchDelegate ValueMatch(JsonPropertyPathMatchDelegate propNameMatch) => + (int depth, ref JsonWalk walk, JsonTokenType type) => + { + var (success, nextDepth) = propNameMatch(depth, ref walk); + + if (!success || !type.IsValueToken()) + { + return (false, 0); + } + + return (true, nextDepth); + }; + + private ObserveRule Build() => JsonObserverItem.ApplyValuePolicy([.. _policies], _fallback.Rule); + + /// + /// A rule in progress: says what happens to the value of the matched property. + /// + public readonly ref struct PropertyMaskingStrategyBuilder + { + private readonly JsonAnyDepthBuilder _builder; + private readonly NamePathMatch _propNameMatch; + + internal PropertyMaskingStrategyBuilder(JsonAnyDepthBuilder builder, NamePathMatch propNameMatch) + { + _builder = builder; + _propNameMatch = propNameMatch; + } + + /// + /// Masks the whole value, whatever its JSON type, with the call's + /// (), which receives ; a null value stays + /// null, and an object or array is masked whole without being read. + /// + /// How the value is masked, for example . + public JsonAnyDepthBuilder Mask(MaskTag tag) + => MaskWhole(JsonObserverItem.ApplyTagPolicy(tag), RuleText.Tag(tag), keepsNull: true); + + /// + /// Masks the whole value, whatever its JSON type, with instead of the call's + /// strategy; a null value stays null. + /// + /// Strategy of this rule. + /// Tag handed to the strategy; by default. + /// is null. + public JsonAnyDepthBuilder Mask(ValueMaskStrategy strategy, MaskTag tag = default) + => MaskWhole(JsonObserverItem.ApplyTagPolicy(tag, strategy ?? throw new ArgumentNullException(nameof(strategy))), RuleText.Strategy(tag), keepsNull: true); + + /// + /// Masks the whole value, whatever its JSON type, with a replacement: a string arrives decoded, a number or + /// boolean as its JSON literal ("12.50", "true"), and an object or array is skipped unread and + /// arrives as null. The replacement receives the whole value and its result is not cut by + /// . + /// + /// + /// Replacement: a constant string, a whose matches become *, + /// or a function; a null result writes null. + /// + /// Whether a null value stays null (the default) or is passed to the replacement. + public JsonAnyDepthBuilder Mask(StringMaskingStrategy strategy, MaskNulls nulls = MaskNulls.Keep) + => MaskWhole(JsonObserverItem.ApplyFunctionPolicy(strategy, strategy.Constant, nulls), RuleText.Function(strategy.Constant, nulls), nulls == MaskNulls.Keep); + + /// + /// Masks the whole value with a function of the value and the context; see . + /// + /// Returns the replacement string; null writes null. + /// Whether a null value stays null (the default) or is passed to the function. + public JsonAnyDepthBuilder Mask(Func strategy, MaskNulls nulls = MaskNulls.Keep) + => Mask((StringMaskingStrategy)strategy, nulls); + + /// + /// Masks the whole value with a function of the value; see . + /// + /// Returns the replacement string; null writes null. + /// Whether a null value stays null (the default) or is passed to the function. + public JsonAnyDepthBuilder Mask(Func strategy, MaskNulls nulls = MaskNulls.Keep) + => Mask(StringMaskingStrategy.From(strategy), nulls); + + /// + /// Hands a string or null value to ; a value of another type is not read. + /// Reading does not decide the output: the default policy writes the value unless + /// or a mask method is chained, or another rule writes the same match. + /// + /// Receives the decoded value and the context. + public ReadRuleBuilder ReadStr(Action strategy) + => Read(JsonObserverItem.ReadStr(strategy), RuleText.ReadStr); + + /// + /// Masks the whole value with , whatever its JSON type. The strategy receives the number + /// when it fits , and null for anything else: another type, a fractional or too large number, + /// null, or an object or array, which is skipped unread. + /// + /// Returns the replacement string; null writes null. + public JsonAnyDepthBuilder MaskInt(Func strategy) + => MaskWhole(JsonObserverItem.ApplyIntPolicy(strategy), "MaskInt(function)"); + + /// + /// Hands a number or null value to ; a number that does not fit + /// arrives as null, and a value of another type is not read. Reading does not decide the output: the default + /// policy writes the value unless or a mask method is chained. + /// + /// Receives the value and the context. + public ReadRuleBuilder ReadInt(Action strategy) + => Read(JsonObserverItem.ReadInt(strategy), RuleText.ReadNumber("ReadInt")); + + /// + /// Masks the whole value with , whatever its JSON type. The strategy receives the number + /// when it fits , and null for anything else. + /// + /// Returns the replacement string; null writes null. + public JsonAnyDepthBuilder MaskLong(Func strategy) + => MaskWhole(JsonObserverItem.ApplyLongPolicy(strategy), "MaskLong(function)"); + + /// + /// Hands a number or null value to ; a number that does not fit + /// arrives as null, and a value of another type is not read. Reading does not decide the output. + /// + /// Receives the value and the context. + public ReadRuleBuilder ReadLong(Action strategy) + => Read(JsonObserverItem.ReadLong(strategy), RuleText.ReadNumber("ReadLong")); + + /// + /// Masks the whole value with , whatever its JSON type. The strategy receives the number + /// when it fits , and null for anything else. + /// + /// Returns the replacement string; null writes null. + public JsonAnyDepthBuilder MaskDecimal(Func strategy) + => MaskWhole(JsonObserverItem.ApplyDecimalPolicy(strategy), "MaskDecimal(function)"); + + /// + /// Hands a number or null value to ; a number out of the range + /// arrives as null, and a value of another type is not read. Reading does not decide the output. + /// + /// Receives the value, parsed with the invariant culture, and the context. + public ReadRuleBuilder ReadDecimal(Action strategy) + => Read(JsonObserverItem.ReadDecimal(strategy), RuleText.ReadNumber("ReadDecimal")); + + /// + /// Masks the whole value with , whatever its JSON type. The strategy receives + /// true or false, and null for anything else, including an object or array, which is skipped unread. + /// + /// Returns the replacement string; null writes null. + public JsonAnyDepthBuilder MaskBool(Func strategy) + => MaskWhole(JsonObserverItem.ApplyBoolPolicy(strategy), "MaskBool(function)"); + + /// + /// Hands a boolean or null value to ; a value of another type is not read. + /// Reading does not decide the output. + /// + /// Receives the value and the context. + public ReadRuleBuilder ReadBool(Action strategy) + => Read(JsonObserverItem.ReadBool(strategy), RuleText.ReadBool); + + /// + /// Hands a string, number, boolean or null value to as its raw JSON text; an object + /// or array is not read. Reading does not decide the output. + /// + /// Receives the raw text (a string without quotes, escapes kept) and the context. + public ReadRuleBuilder ReadRaw(Action strategy) + => Read(JsonObserverItem.ReadRaw(strategy), RuleText.ReadRaw); + + /// + /// Writes the value of the matched property unchanged. Applies to strings, numbers, booleans and null; + /// an object or array gets the next matching rule or the default policy. + /// + public JsonAnyDepthBuilder Unmasked() => + Value(( + ref Utf8JsonReader reader, + JsonWriter writer, + TContext context, + int depth, + ref JsonWalk walk, + ValueRule _) => + BuiltInPolicies.BlockList(ref reader, writer, context, ref walk), RuleText.Unmasked, PathOutcome.Unchanged); + + /// + /// Custom rule for a string, number, boolean or null value of the matched property; an object or array + /// gets the next matching rule or the default policy. + /// + /// Called with the reader on the value; it must write exactly one value. + /// is null. + public JsonAnyDepthBuilder MaskValue(JsonValueRule rule) + => Value(JsonObserverItem.ApplyCustomRule(rule ?? throw new ArgumentNullException(nameof(rule))), RuleText.CustomValue, PathOutcome.Custom); + + internal JsonAnyDepthBuilder MaskWhole(ObserveRule policy, string action, bool keepsNull = false) + => _builder.AddAnyProp(Match, policy, Info(action, PathOutcome.Masked) with { KeepsNull = keepsNull }); + + private ReadRuleBuilder Read(JsonObserverItem.ReadValue read, string action) + => new(this, _builder.AddReadProp(Match, read, Info(action, PathOutcome.Read))); + + private JsonAnyDepthBuilder Value(ObserveRule policy, string action, PathOutcome outcome) + => _builder.AddValueProp(Match, policy, Info(action, outcome)); + + private JsonPropertyPathMatchDelegate Match => _propNameMatch.RelativeMatch; + + private RuleInfo Info(string action, PathOutcome outcome) => new(_propNameMatch.Describe(), action, outcome); + } + + /// + /// A read rule just added. A read rule does not decide what is written: the default policy writes the value unless + /// or a mask method is chained here, which applies to the same match. and + /// start the next rule. + /// + public readonly ref struct ReadRuleBuilder + { + private readonly PropertyMaskingStrategyBuilder _rule; + private readonly JsonAnyDepthBuilder _builder; + + internal ReadRuleBuilder(PropertyMaskingStrategyBuilder rule, JsonAnyDepthBuilder builder) + { + _rule = rule; + _builder = builder; + } + + /// + public PropertyMaskingStrategyBuilder Match(NameMatch match) => _builder.Match(match); + + /// + public PropertyMaskingStrategyBuilder Path(params NameMatch[] path) => _builder.Path(path); + + /// + /// Writes the read value unchanged instead of through the default policy. + /// + public JsonAnyDepthBuilder Unmasked() => _rule.Unmasked(); + + /// + public JsonAnyDepthBuilder Mask(MaskTag tag) => _rule.Mask(tag); + + /// + public JsonAnyDepthBuilder Mask(ValueMaskStrategy strategy, MaskTag tag = default) => _rule.Mask(strategy, tag); + + /// + public JsonAnyDepthBuilder Mask(StringMaskingStrategy strategy, MaskNulls nulls = MaskNulls.Keep) => _rule.Mask(strategy, nulls); + + /// + public JsonAnyDepthBuilder Mask(Func strategy, MaskNulls nulls = MaskNulls.Keep) => _rule.Mask(strategy, nulls); + + /// + public JsonAnyDepthBuilder Mask(Func strategy, MaskNulls nulls = MaskNulls.Keep) => _rule.Mask(strategy, nulls); + + /// + public JsonAnyDepthBuilder MaskInt(Func strategy) => _rule.MaskInt(strategy); + + /// + public JsonAnyDepthBuilder MaskLong(Func strategy) => _rule.MaskLong(strategy); + + /// + public JsonAnyDepthBuilder MaskDecimal(Func strategy) => _rule.MaskDecimal(strategy); + + /// + public JsonAnyDepthBuilder MaskBool(Func strategy) => _rule.MaskBool(strategy); + + /// + public JsonAnyDepthBuilder MaskValue(JsonValueRule rule) => _rule.MaskValue(rule); + + /// + /// The rules added so far, to keep adding to them. + /// + /// The read rule just added. + public static implicit operator JsonAnyDepthBuilder(ReadRuleBuilder rule) => rule._builder; + } +} diff --git a/DragoAnt.System.Text.Json.Observer/Builders/JsonArrayBuilder.cs b/DragoAnt.System.Text.Json.Observer/Builders/JsonArrayBuilder.cs index c12f71d..e76cd2d 100644 --- a/DragoAnt.System.Text.Json.Observer/Builders/JsonArrayBuilder.cs +++ b/DragoAnt.System.Text.Json.Observer/Builders/JsonArrayBuilder.cs @@ -1,4 +1,3 @@ -using DragoAnt.System.Text.Json.Observer.Strategies; using static System.Text.Json.JsonTokenType; namespace DragoAnt.System.Text.Json.Observer.Builders; @@ -13,9 +12,9 @@ public readonly struct JsonArrayBuilder private const string ValueItem = "string, number, boolean or null item"; private readonly List> _policies = []; - private readonly JsonObserverValueDelegate? _builderDefaultValuePolicy; + private readonly JsonValuePolicy? _builderDefaultValuePolicy; - internal JsonArrayBuilder(JsonObserverValueDelegate? builderDefaultValuePolicy) + internal JsonArrayBuilder(JsonValuePolicy? builderDefaultValuePolicy) { _builderDefaultValuePolicy = builderDefaultValuePolicy; } @@ -27,10 +26,10 @@ internal JsonArrayBuilder(JsonObserverValueDelegate? builderDefaultVal /// Policy for the objects' values no rule matches; the enclosing one when null. public JsonArrayBuilder Obj( Action> init, - JsonObserverValueDelegate? defaultValuePolicy = null) + JsonValuePolicy? defaultValuePolicy = null) { var (policy, set) = JsonObserverItem.Obj(init, defaultValuePolicy ?? _builderDefaultValuePolicy); - return Add(type => type == StartObject, policy, new RuleInfo("object item", "Obj(...)", JsonPathOutcome.Unchanged, set)); + return Add(type => type == StartObject, policy, new RuleInfo("object item", "Obj(...)", PathOutcome.Unchanged, set)); } /// @@ -40,85 +39,80 @@ public JsonArrayBuilder Obj( /// Policy for the nested arrays' values no rule matches; the enclosing one when null. public JsonArrayBuilder Array( Action> init, - JsonObserverValueDelegate? defaultValuePolicy = null) + JsonValuePolicy? defaultValuePolicy = null) { var (policy, set) = JsonObserverItem.Array(init, defaultValuePolicy ?? _builderDefaultValuePolicy); - return Add(type => type == StartArray, policy, new RuleInfo("array item", "Array(...)", JsonPathOutcome.Unchanged, set)); + return Add(type => type == StartArray, policy, new RuleInfo("array item", "Array(...)", PathOutcome.Unchanged, set)); } - /// - public JsonArrayBuilder MaskStr(Func strategy) - => MaskStr((StringMaskingStrategy)strategy); + /// + public JsonArrayBuilder Mask(MaskTag tag) + => MaskWhole(JsonObserverItem.ApplyTagPolicy(tag), RuleText.Tag(tag), keepsNull: true); - /// - public JsonArrayBuilder MaskStr(StringMaskingStrategy strategy) => - MaskWhole(JsonObserverItem.ApplyStringPolicy(strategy, strategy.Constant), RuleText.Strategy("MaskStr", strategy.Constant)); + /// + public JsonArrayBuilder Mask(ValueMaskStrategy strategy, MaskTag tag = default) + => MaskWhole(JsonObserverItem.ApplyTagPolicy(tag, strategy ?? throw new ArgumentNullException(nameof(strategy))), RuleText.Strategy(tag), keepsNull: true); - /// + /// + public JsonArrayBuilder Mask(StringMaskingStrategy strategy, MaskNulls nulls = MaskNulls.Keep) + => MaskWhole(JsonObserverItem.ApplyFunctionPolicy(strategy, strategy.Constant, nulls), RuleText.Function(strategy.Constant, nulls), nulls == MaskNulls.Keep); + + /// + public JsonArrayBuilder Mask(Func strategy, MaskNulls nulls = MaskNulls.Keep) + => Mask((StringMaskingStrategy)strategy, nulls); + + /// + public JsonArrayBuilder Mask(Func strategy, MaskNulls nulls = MaskNulls.Keep) + => Mask(StringMaskingStrategy.From(strategy), nulls); + + /// public JsonArrayBuilder ReadStr(Action strategy) => Read(JsonObserverItem.ReadStr(strategy), RuleText.ReadStr); - /// + /// public JsonArrayBuilder MaskInt(Func strategy) => MaskWhole(JsonObserverItem.ApplyIntPolicy(strategy), "MaskInt(function)"); - /// + /// public JsonArrayBuilder ReadInt(Action strategy) => Read(JsonObserverItem.ReadInt(strategy), RuleText.ReadNumber("ReadInt")); - /// + /// public JsonArrayBuilder MaskLong(Func strategy) => MaskWhole(JsonObserverItem.ApplyLongPolicy(strategy), "MaskLong(function)"); - /// + /// public JsonArrayBuilder ReadLong(Action strategy) => Read(JsonObserverItem.ReadLong(strategy), RuleText.ReadNumber("ReadLong")); - /// + /// public JsonArrayBuilder MaskDecimal(Func strategy) => MaskWhole(JsonObserverItem.ApplyDecimalPolicy(strategy), "MaskDecimal(function)"); - /// + /// public JsonArrayBuilder ReadDecimal(Action strategy) => Read(JsonObserverItem.ReadDecimal(strategy), RuleText.ReadNumber("ReadDecimal")); - /// + /// public JsonArrayBuilder MaskBool(Func strategy) => MaskWhole(JsonObserverItem.ApplyBoolPolicy(strategy), "MaskBool(function)"); - /// + /// public JsonArrayBuilder ReadBool(Action strategy) => Read(JsonObserverItem.ReadBool(strategy), RuleText.ReadBool); - /// - public JsonArrayBuilder MaskAny(Func strategy) - => MaskAny((StringMaskingStrategy)strategy); - - /// - public JsonArrayBuilder MaskAny(StringMaskingStrategy strategy) - => MaskWhole(JsonObserverItem.ApplyAnyPolicy(strategy, strategy.Constant), RuleText.Strategy("MaskAny", strategy.Constant)); - - /// - public JsonArrayBuilder MaskAny(MaskTag tag) - => MaskWhole(JsonObserverItem.ApplyTagPolicy(tag), RuleText.Tag(tag)); - - /// - public JsonArrayBuilder MaskRawValue(Func strategy) - => MaskWhole(JsonObserverItem.ApplyRawPolicy(strategy), "MaskRawValue(function)"); - - /// + /// public JsonArrayBuilder ReadRaw(Action strategy) => Read(JsonObserverItem.ReadRaw(strategy), RuleText.ReadRaw); - /// - public JsonArrayBuilder MaskValue(JsonObserverValueDelegate policy) => - MaskValue( - (ref Utf8JsonReader reader, JsonWriter writer, TContext context, int depth, ref PropertyPath propPath, JsonObserverValueDelegate _) => - policy(ref reader, writer, context, ref propPath)); - - /// - public JsonArrayBuilder MaskValue(JsonObserverDelegate policy) - => Add(type => type.IsValueToken(), policy, new RuleInfo(ValueItem, RuleText.CustomValue, JsonPathOutcome.Custom)); + /// + /// Custom rule for the string, number, boolean and null items; object and array items get the next rule or the default policy. + /// + /// Called with the reader on the item; it must write exactly one value. + /// is null. + public JsonArrayBuilder MaskValue(JsonValueRule rule) + => Add(type => type.IsValueToken(), JsonObserverItem.ApplyCustomRule(rule ?? throw new ArgumentNullException(nameof(rule))), + new RuleInfo(ValueItem, RuleText.CustomValue, PathOutcome.Custom)); /// /// Writes the string, number, boolean and null items unchanged; object and array items get the next rule or the default policy. @@ -126,33 +120,33 @@ public JsonArrayBuilder MaskValue(JsonObserverDelegate polic public JsonArrayBuilder Unmasked() => Add( type => type.IsValueToken(), - (ref Utf8JsonReader reader, JsonWriter writer, TContext context, int _, ref PropertyPath propPath, JsonObserverValueDelegate _) => - JsonObserverValuePolicies.BlockList(ref reader, writer, context, ref propPath), - new RuleInfo(ValueItem, RuleText.Unmasked, JsonPathOutcome.Unchanged)); + (ref Utf8JsonReader reader, JsonWriter writer, TContext context, int _, ref JsonWalk walk, ValueRule _) => + BuiltInPolicies.BlockList(ref reader, writer, context, ref walk), + new RuleInfo(ValueItem, RuleText.Unmasked, PathOutcome.Unchanged)); - internal JsonArrayBuilder MaskWhole(JsonObserverDelegate policy, string action) => - Add(_ => true, policy, new RuleInfo(AnyItem, action, JsonPathOutcome.Masked)); + internal JsonArrayBuilder MaskWhole(ObserveRule policy, string action, bool keepsNull = false) => + Add(_ => true, policy, new RuleInfo(AnyItem, action, PathOutcome.Masked, KeepsNull: keepsNull)); - internal static (JsonObserverDelegate Delegate, RuleSet Set) Build(JsonArrayBuilder builder) => builder.Build(); + internal static (ObserveRule Delegate, RuleSet Set) Build(JsonArrayBuilder builder) => builder.Build(); private JsonArrayBuilder Read(JsonObserverItem.ReadValue read, string action) { _policies.Add(JsonObserverItem.Read( - (int _, ref PropertyPath _, JsonTokenType type) => (type.IsValueToken(), 1), + (int _, ref JsonWalk _, JsonTokenType type) => (type.IsValueToken(), 1), read, - new RuleInfo(ValueItem, action, JsonPathOutcome.Read))); + new RuleInfo(ValueItem, action, PathOutcome.Read))); return this; } - private (JsonObserverDelegate, RuleSet) Build() + private (ObserveRule, RuleSet) Build() { JsonObserverItem[] items = [.. _policies]; - return (JsonObserverItem.ApplyArrayPolicy(items, _builderDefaultValuePolicy), new RuleSet(true, items, _builderDefaultValuePolicy)); + return (JsonObserverItem.ApplyArrayPolicy(items, _builderDefaultValuePolicy?.Rule), new RuleSet(true, items, _builderDefaultValuePolicy)); } - private JsonArrayBuilder Add(Func typeMatch, JsonObserverDelegate policy, RuleInfo info) + private JsonArrayBuilder Add(Func typeMatch, ObserveRule policy, RuleInfo info) { - _policies.Add(new JsonObserverItem((int _, ref PropertyPath _, JsonTokenType type) => (typeMatch(type), 1), policy) { Info = info }); + _policies.Add(new JsonObserverItem((int _, ref JsonWalk _, JsonTokenType type) => (typeMatch(type), 1), policy) { Info = info }); return this; } } diff --git a/DragoAnt.System.Text.Json.Observer/Builders/JsonObjBuilder.cs b/DragoAnt.System.Text.Json.Observer/Builders/JsonObjBuilder.cs index 3868fa4..f465551 100644 --- a/DragoAnt.System.Text.Json.Observer/Builders/JsonObjBuilder.cs +++ b/DragoAnt.System.Text.Json.Observer/Builders/JsonObjBuilder.cs @@ -1,19 +1,18 @@ -using DragoAnt.System.Text.Json.Observer.Strategies; using static System.Text.Json.JsonTokenType; namespace DragoAnt.System.Text.Json.Observer.Builders; /// -/// Rules for the properties of one JSON object. A property path given to -/// starts at this object; the first rule that matches a property wins. +/// Rules for the properties of one JSON object. tests a property of this object, +/// a nested property below it; the first rule that matches a property wins. /// /// Type that read rules write extracted values to. public readonly struct JsonObjBuilder { private readonly List> _policies = []; - private readonly JsonObserverValueDelegate? _builderDefaultValuePolicy; + private readonly JsonValuePolicy? _builderDefaultValuePolicy; - internal JsonObjBuilder(JsonObserverValueDelegate? builderDefaultValuePolicy) + internal JsonObjBuilder(JsonValuePolicy? builderDefaultValuePolicy) { _builderDefaultValuePolicy = builderDefaultValuePolicy; } @@ -21,29 +20,29 @@ internal JsonObjBuilder(JsonObserverValueDelegate? builderDefaultValue /// /// Starts a rule for the property this matches; finish it with a rule method. /// - /// Property name test: a string for an exact, case-insensitive name, or one of . - public PropertyMaskingStrategyBuilder Match(PropMatchingStrategy match) => - new(this, new PropertyPathMatch([match]), _builderDefaultValuePolicy); + /// Property name test: a string for an exact name, or one of . + public PropertyMaskingStrategyBuilder Match(NameMatch match) => + new(this, new NamePathMatch([match], isPath: false), _builderDefaultValuePolicy); /// /// Starts a rule for the nested property path this matches, one name test per level; finish it with a rule method. /// - /// Name tests from this object down, for example "card", "number" for card.number. - public PropertyMaskingStrategyBuilder Match(params PropMatchingStrategy[] match) => - new(this, new PropertyPathMatch(match), _builderDefaultValuePolicy); + /// Name tests from this object down, for example "card", "number" for card.number. + public PropertyMaskingStrategyBuilder Path(params NameMatch[] path) => + new(this, new NamePathMatch(path, isPath: true), _builderDefaultValuePolicy); - internal static (JsonObserverDelegate Delegate, RuleSet Set) Build(JsonObjBuilder builder) => builder.Build(); + internal static (ObserveRule Delegate, RuleSet Set) Build(JsonObjBuilder builder) => builder.Build(); - private (JsonObserverDelegate, RuleSet) Build() + private (ObserveRule, RuleSet) Build() { JsonObserverItem[] items = [.. _policies]; - return (JsonObserverItem.ApplyObjPolicy(items, _builderDefaultValuePolicy), new RuleSet(false, items, _builderDefaultValuePolicy)); + return (JsonObserverItem.ApplyObjPolicy(items, _builderDefaultValuePolicy?.Rule), new RuleSet(false, items, _builderDefaultValuePolicy)); } - private JsonObjBuilder AddAny(JsonPropertyPathMatchDelegate propNameMatch, JsonObserverDelegate policy, RuleInfo info) => - Add((int depth, ref PropertyPath path, JsonTokenType _) => propNameMatch(depth, ref path), policy, info); + private JsonObjBuilder AddAny(JsonPropertyPathMatchDelegate propNameMatch, ObserveRule policy, RuleInfo info) => + Add((int depth, ref JsonWalk walk, JsonTokenType _) => propNameMatch(depth, ref walk), policy, info); - private JsonObjBuilder AddValue(JsonPropertyPathMatchDelegate propNameMatch, JsonObserverDelegate policy, RuleInfo info) => + private JsonObjBuilder AddValue(JsonPropertyPathMatchDelegate propNameMatch, ObserveRule policy, RuleInfo info) => Add(ValueMatch(propNameMatch), policy, info); private JsonObjBuilder AddRead(JsonPropertyPathMatchDelegate propNameMatch, JsonObserverItem.ReadValue read, RuleInfo info) @@ -53,9 +52,9 @@ private JsonObjBuilder AddRead(JsonPropertyPathMatchDelegate propNameM } private static JsonPropertyMatchDelegate ValueMatch(JsonPropertyPathMatchDelegate propNameMatch) => - (int depth, ref PropertyPath path, JsonTokenType type) => + (int depth, ref JsonWalk walk, JsonTokenType type) => { - var (success, propDepth) = propNameMatch(depth, ref path); + var (success, propDepth) = propNameMatch(depth, ref walk); if (!success || !type.IsValueToken()) { @@ -65,7 +64,7 @@ private static JsonPropertyMatchDelegate ValueMatch(JsonPropertyPathMatchDelegat return (true, propDepth); }; - private JsonObjBuilder Add(JsonPropertyMatchDelegate propMatch, JsonObserverDelegate policy, RuleInfo info) + private JsonObjBuilder Add(JsonPropertyMatchDelegate propMatch, ObserveRule policy, RuleInfo info) { _policies.Add(new JsonObserverItem(propMatch, policy) { Info = info }); return this; @@ -77,100 +76,90 @@ private JsonObjBuilder Add(JsonPropertyMatchDelegate propMatch, JsonOb public readonly ref struct PropertyMaskingStrategyBuilder { private readonly JsonObjBuilder _builder; - private readonly PropertyPathMatch _propNameMatch; - private readonly JsonObserverValueDelegate? _builderDefaultValuePolicy; + private readonly NamePathMatch _propNameMatch; + private readonly JsonValuePolicy? _builderDefaultValuePolicy; internal PropertyMaskingStrategyBuilder( JsonObjBuilder builder, - PropertyPathMatch propNameMatch, - JsonObserverValueDelegate? builderDefaultValuePolicy) + NamePathMatch propNameMatch, + JsonValuePolicy? builderDefaultValuePolicy) { _builder = builder; _propNameMatch = propNameMatch; _builderDefaultValuePolicy = builderDefaultValuePolicy; } - /// - public JsonObjBuilder MaskStr(Func strategy) - => MaskStr((StringMaskingStrategy)strategy); + /// + public JsonObjBuilder Mask(MaskTag tag) + => MaskWhole(JsonObserverItem.ApplyTagPolicy(tag), RuleText.Tag(tag), keepsNull: true); - /// - public JsonObjBuilder MaskStr(StringMaskingStrategy strategy) => - MaskWhole(JsonObserverItem.ApplyStringPolicy(strategy, strategy.Constant), RuleText.Strategy("MaskStr", strategy.Constant)); + /// + public JsonObjBuilder Mask(ValueMaskStrategy strategy, MaskTag tag = default) + => MaskWhole(JsonObserverItem.ApplyTagPolicy(tag, strategy ?? throw new ArgumentNullException(nameof(strategy))), RuleText.Strategy(tag), keepsNull: true); - /// + /// + public JsonObjBuilder Mask(StringMaskingStrategy strategy, MaskNulls nulls = MaskNulls.Keep) + => MaskWhole(JsonObserverItem.ApplyFunctionPolicy(strategy, strategy.Constant, nulls), RuleText.Function(strategy.Constant, nulls), nulls == MaskNulls.Keep); + + /// + public JsonObjBuilder Mask(Func strategy, MaskNulls nulls = MaskNulls.Keep) + => Mask((StringMaskingStrategy)strategy, nulls); + + /// + public JsonObjBuilder Mask(Func strategy, MaskNulls nulls = MaskNulls.Keep) + => Mask(StringMaskingStrategy.From(strategy), nulls); + + /// public ReadRuleBuilder ReadStr(Action strategy) => Read(JsonObserverItem.ReadStr(strategy), RuleText.ReadStr); - /// + /// public JsonObjBuilder MaskInt(Func strategy) => MaskWhole(JsonObserverItem.ApplyIntPolicy(strategy), "MaskInt(function)"); - /// + /// public ReadRuleBuilder ReadInt(Action strategy) => Read(JsonObserverItem.ReadInt(strategy), RuleText.ReadNumber("ReadInt")); - /// + /// public JsonObjBuilder MaskLong(Func strategy) => MaskWhole(JsonObserverItem.ApplyLongPolicy(strategy), "MaskLong(function)"); - /// + /// public ReadRuleBuilder ReadLong(Action strategy) => Read(JsonObserverItem.ReadLong(strategy), RuleText.ReadNumber("ReadLong")); - /// + /// public JsonObjBuilder MaskDecimal(Func strategy) => MaskWhole(JsonObserverItem.ApplyDecimalPolicy(strategy), "MaskDecimal(function)"); - /// + /// public ReadRuleBuilder ReadDecimal(Action strategy) => Read(JsonObserverItem.ReadDecimal(strategy), RuleText.ReadNumber("ReadDecimal")); - /// + /// public JsonObjBuilder MaskBool(Func strategy) => MaskWhole(JsonObserverItem.ApplyBoolPolicy(strategy), "MaskBool(function)"); - /// + /// public ReadRuleBuilder ReadBool(Action strategy) => Read(JsonObserverItem.ReadBool(strategy), RuleText.ReadBool); - /// - public JsonObjBuilder MaskAny(Func strategy) - => MaskAny((StringMaskingStrategy)strategy); - - /// - public JsonObjBuilder MaskAny(StringMaskingStrategy strategy) - => MaskWhole(JsonObserverItem.ApplyAnyPolicy(strategy, strategy.Constant), RuleText.Strategy("MaskAny", strategy.Constant)); - - /// - public JsonObjBuilder MaskAny(MaskTag tag) - => MaskWhole(JsonObserverItem.ApplyTagPolicy(tag), RuleText.Tag(tag)); - - /// - public JsonObjBuilder MaskRawValue(Func strategy) - => MaskWhole(JsonObserverItem.ApplyRawPolicy(strategy), "MaskRawValue(function)"); - - /// + /// public ReadRuleBuilder ReadRaw(Action strategy) => Read(JsonObserverItem.ReadRaw(strategy), RuleText.ReadRaw); - /// - public JsonObjBuilder MaskValue(JsonObserverValueDelegate policy) => - MaskValue( - (ref Utf8JsonReader reader, JsonWriter writer, TContext context, int depth, ref PropertyPath propPath, JsonObserverValueDelegate _) => - policy(ref reader, writer, context, ref propPath)); - - /// - public JsonObjBuilder MaskValue(JsonObserverDelegate policy) - => _builder.AddValue(_propNameMatch.AbsoluteMatch, policy, Info(RuleText.CustomValue, JsonPathOutcome.Custom)); + /// + public JsonObjBuilder MaskValue(JsonValueRule rule) + => _builder.AddValue(_propNameMatch.AbsoluteMatch, JsonObserverItem.ApplyCustomRule(rule ?? throw new ArgumentNullException(nameof(rule))), Info(RuleText.CustomValue, PathOutcome.Custom)); - /// + /// public JsonObjBuilder Unmasked() => _builder.AddValue( _propNameMatch.AbsoluteMatch, - (ref Utf8JsonReader reader, JsonWriter writer, TContext context, int depth, ref PropertyPath propPath, JsonObserverValueDelegate _) => - JsonObserverValuePolicies.BlockList(ref reader, writer, context, ref propPath), - Info(RuleText.Unmasked, JsonPathOutcome.Unchanged)); + (ref Utf8JsonReader reader, JsonWriter writer, TContext context, int depth, ref JsonWalk walk, ValueRule _) => + BuiltInPolicies.BlockList(ref reader, writer, context, ref walk), + Info(RuleText.Unmasked, PathOutcome.Unchanged)); /// /// Rules for the matched property when its value is an object; a value of another type gets the next matching rule @@ -180,18 +169,18 @@ public JsonObjBuilder Unmasked() => /// Policy for the object's values no rule matches; the enclosing one when null. public JsonObjBuilder Obj( Action> init, - JsonObserverValueDelegate? defaultValuePolicy = null) + JsonValuePolicy? defaultValuePolicy = null) { var (policy, set) = JsonObserverItem.Obj(init, defaultValuePolicy ?? _builderDefaultValuePolicy); - return Container(StartObject, policy, Info("Obj(...)", JsonPathOutcome.Unchanged, set)); + return Container(StartObject, policy, Info("Obj(...)", PathOutcome.Unchanged, set)); } /// /// Custom handling of the matched property when its value is an object. /// - /// Called with the reader on the object's start; it must write the object and move past it. - public JsonObjBuilder Obj(JsonObserverDelegate policy) => - Container(StartObject, policy, Info("Obj(custom rule)", JsonPathOutcome.Custom)); + /// Called with the reader on the object's start; it must write the object and leave the reader on its end. + public JsonObjBuilder Obj(JsonValueRule rule) => + Container(StartObject, JsonObserverItem.ApplyCustomRule(rule ?? throw new ArgumentNullException(nameof(rule))), Info("Obj(custom rule)", PathOutcome.Custom)); /// /// Rules for the matched property when its value is an array; a value of another type gets the next matching rule @@ -201,31 +190,31 @@ public JsonObjBuilder Obj(JsonObserverDelegate policy) => /// Policy for the array's values no rule matches; the enclosing one when null. public JsonObjBuilder Array( Action> init, - JsonObserverValueDelegate? defaultValuePolicy = null) + JsonValuePolicy? defaultValuePolicy = null) { var (policy, set) = JsonObserverItem.Array(init, defaultValuePolicy ?? _builderDefaultValuePolicy); - return Container(StartArray, policy, Info("Array(...)", JsonPathOutcome.Unchanged, set)); + return Container(StartArray, policy, Info("Array(...)", PathOutcome.Unchanged, set)); } /// /// Custom handling of the matched property when its value is an array. /// - /// Called with the reader on the array's start; it must write the array and move past it. - public JsonObjBuilder Array(JsonObserverDelegate policy) => - Container(StartArray, policy, Info("Array(custom rule)", JsonPathOutcome.Custom)); + /// Called with the reader on the array's start; it must write the array and leave the reader on its end. + public JsonObjBuilder Array(JsonValueRule rule) => + Container(StartArray, JsonObserverItem.ApplyCustomRule(rule ?? throw new ArgumentNullException(nameof(rule))), Info("Array(custom rule)", PathOutcome.Custom)); - internal JsonObjBuilder MaskWhole(JsonObserverDelegate policy, string action) - => _builder.AddAny(_propNameMatch.AbsoluteMatch, policy, Info(action, JsonPathOutcome.Masked)); + internal JsonObjBuilder MaskWhole(ObserveRule policy, string action, bool keepsNull = false) + => _builder.AddAny(_propNameMatch.AbsoluteMatch, policy, Info(action, PathOutcome.Masked) with { KeepsNull = keepsNull }); private ReadRuleBuilder Read(JsonObserverItem.ReadValue read, string action) - => new(this, _builder.AddRead(_propNameMatch.AbsoluteMatch, read, Info(action, JsonPathOutcome.Read))); + => new(this, _builder.AddRead(_propNameMatch.AbsoluteMatch, read, Info(action, PathOutcome.Read))); - private JsonObjBuilder Container(JsonTokenType container, JsonObserverDelegate policy, RuleInfo info) + private JsonObjBuilder Container(JsonTokenType container, ObserveRule policy, RuleInfo info) { var match = _propNameMatch.AbsoluteMatch; - return _builder.Add((int depth, ref PropertyPath path, JsonTokenType type) => + return _builder.Add((int depth, ref JsonWalk walk, JsonTokenType type) => { - var (success, nextDepth) = match(depth, ref path); + var (success, nextDepth) = match(depth, ref walk); if (!success || type != container) { @@ -236,14 +225,14 @@ private JsonObjBuilder Container(JsonTokenType container, JsonObserver }, policy, info); } - private RuleInfo Info(string action, JsonPathOutcome outcome, RuleSet? child = null) => + private RuleInfo Info(string action, PathOutcome outcome, RuleSet? child = null) => new(_propNameMatch.Describe(), action, outcome, child); } /// /// A read rule just added. A read rule does not decide what is written: the default policy writes the value unless - /// or a mask method is chained here, which applies to the same match. - /// starts the next rule. + /// or a mask method is chained here, which applies to the same match. and + /// start the next rule. /// public readonly ref struct ReadRuleBuilder { @@ -256,52 +245,46 @@ internal ReadRuleBuilder(PropertyMaskingStrategyBuilder rule, JsonObjBuilder - public PropertyMaskingStrategyBuilder Match(PropMatchingStrategy match) => _builder.Match(match); + /// + public PropertyMaskingStrategyBuilder Match(NameMatch match) => _builder.Match(match); - /// - public PropertyMaskingStrategyBuilder Match(params PropMatchingStrategy[] match) => _builder.Match(match); + /// + public PropertyMaskingStrategyBuilder Path(params NameMatch[] path) => _builder.Path(path); /// /// Writes the read value unchanged instead of through the default policy. /// public JsonObjBuilder Unmasked() => _rule.Unmasked(); - /// - public JsonObjBuilder MaskStr(Func strategy) => _rule.MaskStr(strategy); + /// + public JsonObjBuilder Mask(MaskTag tag) => _rule.Mask(tag); - /// - public JsonObjBuilder MaskStr(StringMaskingStrategy strategy) => _rule.MaskStr(strategy); + /// + public JsonObjBuilder Mask(ValueMaskStrategy strategy, MaskTag tag = default) => _rule.Mask(strategy, tag); - /// + /// + public JsonObjBuilder Mask(StringMaskingStrategy strategy, MaskNulls nulls = MaskNulls.Keep) => _rule.Mask(strategy, nulls); + + /// + public JsonObjBuilder Mask(Func strategy, MaskNulls nulls = MaskNulls.Keep) => _rule.Mask(strategy, nulls); + + /// + public JsonObjBuilder Mask(Func strategy, MaskNulls nulls = MaskNulls.Keep) => _rule.Mask(strategy, nulls); + + /// public JsonObjBuilder MaskInt(Func strategy) => _rule.MaskInt(strategy); - /// + /// public JsonObjBuilder MaskLong(Func strategy) => _rule.MaskLong(strategy); - /// + /// public JsonObjBuilder MaskDecimal(Func strategy) => _rule.MaskDecimal(strategy); - /// + /// public JsonObjBuilder MaskBool(Func strategy) => _rule.MaskBool(strategy); - /// - public JsonObjBuilder MaskAny(Func strategy) => _rule.MaskAny(strategy); - - /// - public JsonObjBuilder MaskAny(StringMaskingStrategy strategy) => _rule.MaskAny(strategy); - - /// - public JsonObjBuilder MaskAny(MaskTag tag) => _rule.MaskAny(tag); - - /// - public JsonObjBuilder MaskRawValue(Func strategy) => _rule.MaskRawValue(strategy); - - /// - public JsonObjBuilder MaskValue(JsonObserverValueDelegate policy) => _rule.MaskValue(policy); - - /// - public JsonObjBuilder MaskValue(JsonObserverDelegate policy) => _rule.MaskValue(policy); + /// + public JsonObjBuilder MaskValue(JsonValueRule rule) => _rule.MaskValue(rule); /// /// The rules added so far, to keep adding to them. diff --git a/DragoAnt.System.Text.Json.Observer/Builders/JsonValuePolicyBuilder.cs b/DragoAnt.System.Text.Json.Observer/Builders/JsonValuePolicyBuilder.cs deleted file mode 100644 index 35687c5..0000000 --- a/DragoAnt.System.Text.Json.Observer/Builders/JsonValuePolicyBuilder.cs +++ /dev/null @@ -1,356 +0,0 @@ -using DragoAnt.System.Text.Json.Observer.Strategies; - -namespace DragoAnt.System.Text.Json.Observer.Builders; - -/// -/// Rules of a relative policy (): a property path given to -/// matches the end of a property's path at any depth. The first rule that matches wins. -/// -/// Type that read rules write extracted values to. -public readonly struct JsonValuePolicyBuilder -{ - private readonly bool _relative; - private readonly List> _policies = []; - private readonly JsonObserverValueDelegate? _builderDefaultValuePolicy; - - internal JsonValuePolicyBuilder(bool relative, JsonObserverValueDelegate? builderDefaultValuePolicy) - { - _relative = relative; - _builderDefaultValuePolicy = builderDefaultValuePolicy; - } - - /// - /// Starts a rule for the properties whose path ends with these names, at any depth; finish it with a rule method. - /// - /// - /// Name tests for the last levels of the path, for example "card", "number" for any …card.number: - /// a string for an exact, case-insensitive name, or one of . - /// - public PropertyMaskingStrategyBuilder Match(params PropMatchingStrategy[] match) => - new(this, new PropertyPathMatch(match), _builderDefaultValuePolicy); - - internal static JsonObserverDelegate Build(JsonValuePolicyBuilder builder) => builder.Build(); - - internal static JsonObserverItem[] BuildItems(JsonValuePolicyBuilder builder) => [.. builder._policies]; - - private JsonValuePolicyBuilder AddAnyProp(JsonPropertyPathMatchDelegate propNameMatch, JsonObserverDelegate policy, RuleInfo info) - { - _policies.Add(new JsonObserverItem((int depth, ref PropertyPath path, JsonTokenType _) => propNameMatch(depth, ref path), policy) { Info = info }); - return this; - } - - private JsonValuePolicyBuilder AddValueProp(JsonPropertyPathMatchDelegate propNameMatch, JsonObserverDelegate policy, RuleInfo info) - { - _policies.Add(new JsonObserverItem(ValueMatch(propNameMatch), policy) { Info = info }); - return this; - } - - private JsonValuePolicyBuilder AddReadProp(JsonPropertyPathMatchDelegate propNameMatch, JsonObserverItem.ReadValue read, RuleInfo info) - { - _policies.Add(JsonObserverItem.Read(ValueMatch(propNameMatch), read, info)); - return this; - } - - private static JsonPropertyMatchDelegate ValueMatch(JsonPropertyPathMatchDelegate propNameMatch) => - (int depth, ref PropertyPath path, JsonTokenType type) => - { - var (success, nextDepth) = propNameMatch(depth, ref path); - - if (!success || !type.IsValueToken()) - { - return (false, 0); - } - - return (true, nextDepth); - }; - - private JsonObserverDelegate Build() => JsonObserverItem.ApplyValuePolicy([.. _policies], _builderDefaultValuePolicy); - - /// - /// A rule in progress: says what happens to the value of the matched property. - /// - public readonly ref struct PropertyMaskingStrategyBuilder - { - private readonly JsonValuePolicyBuilder _builder; - private readonly PropertyPathMatch _propNameMatch; - private readonly JsonObserverValueDelegate? _builderDefaultValuePolicy; - - internal PropertyMaskingStrategyBuilder( - JsonValuePolicyBuilder builder, - PropertyPathMatch propNameMatch, - JsonObserverValueDelegate? builderDefaultValuePolicy) - { - _builder = builder; - _propNameMatch = propNameMatch; - _builderDefaultValuePolicy = builderDefaultValuePolicy; - } - - /// - /// Masks the whole value with , whatever its JSON type: a string arrives decoded, - /// a number or boolean as its JSON literal ("12.50", "true"), null as null, and an object - /// or array is skipped unread and arrives as null. The strategy receives the whole value and its result is not cut by - /// . - /// - /// Returns the replacement string; null writes null. - public JsonValuePolicyBuilder MaskStr(Func strategy) - => MaskStr((StringMaskingStrategy)strategy); - - /// - /// Masks the whole value with , whatever its JSON type: a string arrives decoded, - /// a number or boolean as its JSON literal ("12.50", "true"), null as null, and an object - /// or array is skipped unread and arrives as null. The strategy receives the whole value and its result is not cut by - /// . - /// - /// - /// Replacement: a constant string, a whose matches become *, - /// or a function; a null result writes null. - /// - public JsonValuePolicyBuilder MaskStr(StringMaskingStrategy strategy) - => MaskWhole(JsonObserverItem.ApplyStringPolicy(strategy, strategy.Constant), RuleText.Strategy("MaskStr", strategy.Constant)); - - /// - /// Hands a string or null value to ; a value of another type is not read. - /// Reading does not decide the output: the default policy writes the value unless - /// or a mask method is chained, or another rule writes the same match. - /// - /// Receives the decoded value and the context. - public ReadRuleBuilder ReadStr(Action strategy) - => Read(JsonObserverItem.ReadStr(strategy), RuleText.ReadStr); - - /// - /// Masks the whole value with , whatever its JSON type. The strategy receives the number - /// when it fits , and null for anything else: another type, a fractional or too large number, - /// null, or an object or array, which is skipped unread. - /// - /// Returns the replacement string; null writes null. - public JsonValuePolicyBuilder MaskInt(Func strategy) - => MaskWhole(JsonObserverItem.ApplyIntPolicy(strategy), "MaskInt(function)"); - - /// - /// Hands a number or null value to ; a number that does not fit - /// arrives as null, and a value of another type is not read. Reading does not decide the output: the default policy writes the value unless - /// or a mask method is chained, or another rule writes the same match. - /// - /// Receives the value and the context. - public ReadRuleBuilder ReadInt(Action strategy) - => Read(JsonObserverItem.ReadInt(strategy), RuleText.ReadNumber("ReadInt")); - - /// - /// Masks the whole value with , whatever its JSON type. The strategy receives the number - /// when it fits , and null for anything else: another type, a fractional or too large number, - /// null, or an object or array, which is skipped unread. - /// - /// Returns the replacement string; null writes null. - public JsonValuePolicyBuilder MaskLong(Func strategy) - => MaskWhole(JsonObserverItem.ApplyLongPolicy(strategy), "MaskLong(function)"); - - /// - /// Hands a number or null value to ; a number that does not fit - /// arrives as null, and a value of another type is not read. Reading does not decide the output: the default policy writes the value unless - /// or a mask method is chained, or another rule writes the same match. - /// - /// Receives the value and the context. - public ReadRuleBuilder ReadLong(Action strategy) - => Read(JsonObserverItem.ReadLong(strategy), RuleText.ReadNumber("ReadLong")); - - /// - /// Masks the whole value with , whatever its JSON type. The strategy receives the number - /// when it fits , and null for anything else: another type, a number out of range, - /// null, or an object or array, which is skipped unread. - /// - /// Returns the replacement string; null writes null. - public JsonValuePolicyBuilder MaskDecimal(Func strategy) - => MaskWhole(JsonObserverItem.ApplyDecimalPolicy(strategy), "MaskDecimal(function)"); - - /// - /// Hands a number or null value to ; a number out of the range - /// arrives as null, and a value of another type is not read. Reading does not decide the output: the default policy writes the value unless - /// or a mask method is chained, or another rule writes the same match. - /// - /// Receives the value, parsed with the invariant culture, and the context. - public ReadRuleBuilder ReadDecimal(Action strategy) - => Read(JsonObserverItem.ReadDecimal(strategy), RuleText.ReadNumber("ReadDecimal")); - - /// - /// Masks the whole value with , whatever its JSON type. The strategy receives - /// true or false, and null for anything else, including an object or array, which is skipped unread. - /// - /// Returns the replacement string; null writes null. - public JsonValuePolicyBuilder MaskBool(Func strategy) - => MaskWhole(JsonObserverItem.ApplyBoolPolicy(strategy), "MaskBool(function)"); - - /// - /// Hands a boolean or null value to ; a value of another type is not read. - /// Reading does not decide the output: the default policy writes the value unless - /// or a mask method is chained, or another rule writes the same match. - /// - /// Receives the value and the context. - public ReadRuleBuilder ReadBool(Action strategy) - => Read(JsonObserverItem.ReadBool(strategy), RuleText.ReadBool); - - /// - /// Masks the whole value with , whatever its JSON type: a string arrives decoded, - /// a number or boolean as its JSON literal, and an object or array is skipped unread and arrives as null. - /// A null value stays null without calling the strategy. The strategy receives the whole value and its result is not cut by - /// . - /// - /// Returns the replacement string; null writes null. - public JsonValuePolicyBuilder MaskAny(Func strategy) - => MaskAny((StringMaskingStrategy)strategy); - - /// - /// Masks the whole value with , whatever its JSON type: a string arrives decoded, - /// a number or boolean as its JSON literal, and an object or array is skipped unread and arrives as null. - /// A null value stays null without calling the strategy. The strategy receives the whole value and its result is not cut by - /// . - /// - /// - /// Replacement: a constant string, a whose matches become *, - /// or a function; a null result writes null. - /// - public JsonValuePolicyBuilder MaskAny(StringMaskingStrategy strategy) - => MaskWhole(JsonObserverItem.ApplyAnyPolicy(strategy, strategy.Constant), RuleText.Strategy("MaskAny", strategy.Constant)); - - /// - /// Masks the whole value, whatever its JSON type, with the of the call - /// (), which receives ; a null value stays null. - /// - /// How the value is masked, for example . - public JsonValuePolicyBuilder MaskAny(MaskTag tag) - => MaskWhole(JsonObserverItem.ApplyTagPolicy(tag), RuleText.Tag(tag)); - - /// - /// Like , but a string arrives as its raw JSON text, - /// escape sequences included and without quotes. - /// - /// Returns the replacement string; null writes null. - public JsonValuePolicyBuilder MaskRawValue(Func strategy) - => MaskWhole(JsonObserverItem.ApplyRawPolicy(strategy), "MaskRawValue(function)"); - - /// - /// Hands a string, number, boolean or null value to as its raw JSON text; an object - /// or array is not read. Reading does not decide the output: the default policy writes the value unless - /// or a mask method is chained, or another rule writes the same match. - /// - /// Receives the raw text (a string without quotes, escapes kept) and the context. - public ReadRuleBuilder ReadRaw(Action strategy) - => Read(JsonObserverItem.ReadRaw(strategy), RuleText.ReadRaw); - - /// - /// Writes the value of the matched property unchanged. Applies to strings, numbers, booleans and null; - /// an object or array gets the next matching rule or the default policy. - /// - public JsonValuePolicyBuilder Unmasked() => - Value(( - ref Utf8JsonReader reader, - JsonWriter writer, - TContext context, - int depth, - ref PropertyPath propPath, - JsonObserverValueDelegate _) => - JsonObserverValuePolicies.BlockList(ref reader, writer, context, ref propPath), RuleText.Unmasked, JsonPathOutcome.Unchanged); - - /// - /// Custom rule for a string, number, boolean or null value of the matched property; an object or array - /// gets the next matching rule or the default policy. - /// - /// Called with the reader on the value; it must write exactly one value. - public JsonValuePolicyBuilder MaskValue(JsonObserverValueDelegate policy) => - MaskValue(( - ref Utf8JsonReader reader, - JsonWriter writer, - TContext context, - int depth, - ref PropertyPath propPath, - JsonObserverValueDelegate _) => - policy(ref reader, writer, context, ref propPath)); - - /// - /// Custom rule for a string, number, boolean or null value of the matched property; an object or array - /// gets the next matching rule or the default policy. - /// - /// Called with the reader on the value; it must write exactly one value. - public JsonValuePolicyBuilder MaskValue(JsonObserverDelegate policy) - => Value(policy, RuleText.CustomValue, JsonPathOutcome.Custom); - - internal JsonValuePolicyBuilder MaskWhole(JsonObserverDelegate policy, string action) - => _builder.AddAnyProp(Match, policy, Info(action, JsonPathOutcome.Masked)); - - private ReadRuleBuilder Read(JsonObserverItem.ReadValue read, string action) - => new(this, _builder.AddReadProp(Match, read, Info(action, JsonPathOutcome.Read))); - - private JsonValuePolicyBuilder Value(JsonObserverDelegate policy, string action, JsonPathOutcome outcome) - => _builder.AddValueProp(Match, policy, Info(action, outcome)); - - private JsonPropertyPathMatchDelegate Match => _builder._relative ? _propNameMatch.RelativeMatch : _propNameMatch.AbsoluteMatch; - - private RuleInfo Info(string action, JsonPathOutcome outcome) => new(_propNameMatch.Describe(), action, outcome); - } - - /// - /// A read rule just added. A read rule does not decide what is written: the default policy writes the value unless - /// or a mask method is chained here, which applies to the same match. - /// starts the next rule. - /// - public readonly ref struct ReadRuleBuilder - { - private readonly PropertyMaskingStrategyBuilder _rule; - private readonly JsonValuePolicyBuilder _builder; - - internal ReadRuleBuilder(PropertyMaskingStrategyBuilder rule, JsonValuePolicyBuilder builder) - { - _rule = rule; - _builder = builder; - } - - /// - public PropertyMaskingStrategyBuilder Match(params PropMatchingStrategy[] match) => _builder.Match(match); - - /// - /// Writes the read value unchanged instead of through the default policy. - /// - public JsonValuePolicyBuilder Unmasked() => _rule.Unmasked(); - - /// - public JsonValuePolicyBuilder MaskStr(Func strategy) => _rule.MaskStr(strategy); - - /// - public JsonValuePolicyBuilder MaskStr(StringMaskingStrategy strategy) => _rule.MaskStr(strategy); - - /// - public JsonValuePolicyBuilder MaskInt(Func strategy) => _rule.MaskInt(strategy); - - /// - public JsonValuePolicyBuilder MaskLong(Func strategy) => _rule.MaskLong(strategy); - - /// - public JsonValuePolicyBuilder MaskDecimal(Func strategy) => _rule.MaskDecimal(strategy); - - /// - public JsonValuePolicyBuilder MaskBool(Func strategy) => _rule.MaskBool(strategy); - - /// - public JsonValuePolicyBuilder MaskAny(Func strategy) => _rule.MaskAny(strategy); - - /// - public JsonValuePolicyBuilder MaskAny(StringMaskingStrategy strategy) => _rule.MaskAny(strategy); - - /// - public JsonValuePolicyBuilder MaskAny(MaskTag tag) => _rule.MaskAny(tag); - - /// - public JsonValuePolicyBuilder MaskRawValue(Func strategy) => _rule.MaskRawValue(strategy); - - /// - public JsonValuePolicyBuilder MaskValue(JsonObserverValueDelegate policy) => _rule.MaskValue(policy); - - /// - public JsonValuePolicyBuilder MaskValue(JsonObserverDelegate policy) => _rule.MaskValue(policy); - - /// - /// The rules added so far, to keep adding to them. - /// - /// The read rule just added. - public static implicit operator JsonValuePolicyBuilder(ReadRuleBuilder rule) => rule._builder; - } -} diff --git a/DragoAnt.System.Text.Json.Observer/Builders/RuleText.cs b/DragoAnt.System.Text.Json.Observer/Builders/RuleText.cs index d2c6aeb..13fe29d 100644 --- a/DragoAnt.System.Text.Json.Observer/Builders/RuleText.cs +++ b/DragoAnt.System.Text.Json.Observer/Builders/RuleText.cs @@ -1,9 +1,7 @@ -using DragoAnt.System.Text.Json.Observer.Strategies; - namespace DragoAnt.System.Text.Json.Observer.Builders; /// -/// How rule actions read in a . +/// How rule actions read in a . /// internal static class RuleText { @@ -15,9 +13,14 @@ internal static class RuleText public static string ReadNumber(string method) => $"{method} (a number or null is read)"; - public static string Strategy(string method, string? constant) => - constant is null ? $"{method}(function)" : $"{method}(\"{constant}\")"; + public static string Function(string? constant, MaskNulls nulls) + { + var what = constant is null ? "function" : $"\"{constant}\""; + return nulls == MaskNulls.Mask ? $"Mask({what}, MaskNulls.Mask)" : $"Mask({what})"; + } public static string Tag(MaskTag tag) => - tag.Key is null ? $"MaskAny(MaskTag.{tag.Kind})" : $"MaskAny(MaskTag.{tag.Kind}, key {tag.Key})"; + tag.Key is null ? $"Mask(MaskTag.{tag.Kind})" : $"Mask(MaskTag.{tag.Kind}, key {tag.Key})"; + + public static string Strategy(MaskTag tag) => $"Mask(strategy, MaskTag.{tag.Kind})"; } diff --git a/DragoAnt.System.Text.Json.Observer/DragoAnt.System.Text.Json.Observer.csproj b/DragoAnt.System.Text.Json.Observer/DragoAnt.System.Text.Json.Observer.csproj index a4da6f9..cd56f87 100644 --- a/DragoAnt.System.Text.Json.Observer/DragoAnt.System.Text.Json.Observer.csproj +++ b/DragoAnt.System.Text.Json.Observer/DragoAnt.System.Text.Json.Observer.csproj @@ -4,6 +4,12 @@ true json;system.text.json;utf8jsonreader;masking;redaction;pii;extraction;streaming;logging + + + + + + diff --git a/DragoAnt.System.Text.Json.Observer/GlobalUsings.cs b/DragoAnt.System.Text.Json.Observer/GlobalUsings.cs index eefd540..2d77e88 100644 --- a/DragoAnt.System.Text.Json.Observer/GlobalUsings.cs +++ b/DragoAnt.System.Text.Json.Observer/GlobalUsings.cs @@ -1 +1,2 @@ -global using System.Text.Json; +global using System.Text.Json; +global using DragoAnt.Observer; diff --git a/DragoAnt.System.Text.Json.Observer/JsonObserver.cs b/DragoAnt.System.Text.Json.Observer/JsonObserver.cs index df1f786..76d074e 100644 --- a/DragoAnt.System.Text.Json.Observer/JsonObserver.cs +++ b/DragoAnt.System.Text.Json.Observer/JsonObserver.cs @@ -1,6 +1,7 @@ using System.Buffers; using System.Runtime.CompilerServices; using System.Text; +using System.Diagnostics.CodeAnalysis; using DragoAnt.System.Text.Json.Observer.Builders; // ReSharper disable MemberCanBePrivate.Global @@ -13,66 +14,66 @@ namespace DragoAnt.System.Text.Json.Observer; /// public sealed class JsonObserver { - private readonly JsonObserver _masking; + private readonly JsonObserver _masking; /// /// Creates an observer for a payload whose root is an object or an array. /// /// Adds the rules for a root object. /// Adds the rules for a root array. - /// Policy for values no rule matches; when null. + /// Policy for values no rule matches; when null. public static JsonObserver Any( - Action> initObj, - Action> initArray, - JsonObserverValueDelegate? defaultMasking = null) => - new(Any(initObj, initArray, defaultMasking)); + Action> initObj, + Action> initArray, + JsonValuePolicy? defaultMasking = null) => + new(Any(initObj, initArray, defaultMasking)); /// /// Creates an observer for a root object that applies one policy to every value, for example - /// . A root array is . + /// . A root array is . /// - /// Policy for every value; when null. - public static JsonObserver Obj(JsonObserverValueDelegate? defaultMasking) => - new(Obj(defaultMasking)); + /// Policy for every value; when null. + public static JsonObserver Obj(JsonValuePolicy? defaultMasking) => + new(Obj(defaultMasking)); /// /// Creates an observer for a root object. A root array is ; use for both. /// /// Adds the rules for the object's properties. - /// Policy for values no rule matches; when null. + /// Policy for values no rule matches; when null. public static JsonObserver Obj( - Action> init, - JsonObserverValueDelegate? defaultMasking = null) => - new(Obj(init, defaultMasking)); + Action> init, + JsonValuePolicy? defaultMasking = null) => + new(Obj(init, defaultMasking)); /// /// Creates an observer for a root array that applies one policy to every value. A root object is . /// - /// Policy for every value; when null. - public static JsonObserver Array(JsonObserverValueDelegate? defaultMasking) => - new(Array(defaultMasking)); + /// Policy for every value; when null. + public static JsonObserver Array(JsonValuePolicy? defaultMasking) => + new(Array(defaultMasking)); /// /// Creates an observer for a root array. A root object is ; use for both. /// /// Adds the rules for the array's items. - /// Policy for values no rule matches; when null. + /// Policy for values no rule matches; when null. public static JsonObserver Array( - Action> init, - JsonObserverValueDelegate? defaultMasking = null) => - new(Array(init, defaultMasking)); + Action> init, + JsonValuePolicy? defaultMasking = null) => + new(Array(init, defaultMasking)); /// /// Creates an observer that also extracts values into a , for a root object or array. /// /// Adds the rules for a root object. /// Adds the rules for a root array. - /// Policy for values no rule matches; when null. + /// Policy for values no rule matches; when null. /// Type that read rules write extracted values to. public static JsonObserver Any( Action> initObj, Action> initArray, - JsonObserverValueDelegate? defaultMasking = null) + JsonValuePolicy? defaultMasking = null) { var (masking, obj, array) = JsonObserverItem.Any(initObj, initArray, defaultMasking); return new JsonObserver(masking, new RuleExplainer(obj, array)); @@ -81,20 +82,20 @@ public static JsonObserver Any( /// /// Creates an observer with a context for a root object that applies one policy to every value. /// - /// Policy for every value; when null. + /// Policy for every value; when null. /// Type that read rules write extracted values to. - public static JsonObserver Obj(JsonObserverValueDelegate? defaultMasking) => + public static JsonObserver Obj(JsonValuePolicy? defaultMasking) => Obj(_ => { }, defaultMasking); /// /// Creates an observer that also extracts values into a , for a root object. /// /// Adds the rules for the object's properties. - /// Policy for values no rule matches; when null. + /// Policy for values no rule matches; when null. /// Type that read rules write extracted values to. public static JsonObserver Obj( Action> init, - JsonObserverValueDelegate? defaultMasking = null) + JsonValuePolicy? defaultMasking = null) { var (masking, set) = JsonObserverItem.Obj(init, defaultMasking); return new JsonObserver(masking, new RuleExplainer(set, null)); @@ -103,20 +104,20 @@ public static JsonObserver Obj( /// /// Creates an observer with a context for a root array that applies one policy to every value. /// - /// Policy for every value; when null. + /// Policy for every value; when null. /// Type that read rules write extracted values to. - public static JsonObserver Array(JsonObserverValueDelegate? defaultMasking) => + public static JsonObserver Array(JsonValuePolicy? defaultMasking) => Array(_ => { }, defaultMasking); /// /// Creates an observer that also extracts values into a , for a root array. /// /// Adds the rules for the array's items. - /// Policy for values no rule matches; when null. + /// Policy for values no rule matches; when null. /// Type that read rules write extracted values to. public static JsonObserver Array( Action> init, - JsonObserverValueDelegate? defaultMasking = null) + JsonValuePolicy? defaultMasking = null) { var (masking, set) = JsonObserverItem.Array(init, defaultMasking); return new JsonObserver(masking, new RuleExplainer(null, set)); @@ -133,10 +134,10 @@ public static JsonObserver FromShape(JsonShape shape, JsonShapeOptions? options { ArgumentNullException.ThrowIfNull(shape); var walker = new ShapeWalker(shape, options ?? JsonShapeOptions.Default); - return new JsonObserver(new JsonObserver(walker.Invoke, walker)); + return new JsonObserver(new JsonObserver(walker.Invoke, walker)); } - private JsonObserver(JsonObserver masking) + private JsonObserver(JsonObserver masking) { _masking = masking; } @@ -150,7 +151,7 @@ private JsonObserver(JsonObserver masking) /// Limits and output settings; when omitted. /// The masked JSON; empty when the text is not a JSON object or array; null for null. public string? Mask(string? json, JsonObserverOptions? options = null) - => _masking.Mask(json, JsonObserveringEmptyContext.Instance, out _, options); + => _masking.Mask(json, NoContext.Instance, out _, options); /// /// Masks a JSON text and reports what happened. Never throws: invalid or cut-off JSON yields its safe masked prefix, @@ -161,7 +162,7 @@ private JsonObserver(JsonObserver masking) /// Limits and output settings; when omitted. /// The masked JSON; empty when the text is not a JSON object or array; null for null. public string? Mask(string? json, out MaskResult result, JsonObserverOptions? options = null) - => _masking.Mask(json, JsonObserveringEmptyContext.Instance, out result, options); + => _masking.Mask(json, NoContext.Instance, out result, options); /// /// Masks a UTF-8 JSON payload into . Never throws: problems are reported in the result, @@ -172,7 +173,7 @@ private JsonObserver(JsonObserver masking) /// Limits and output settings; when omitted. /// Status, bytes written and the input offset where reading stopped. public MaskResult Mask(ReadOnlySpan utf8, IBufferWriter output, JsonObserverOptions? options = null) - => _masking.Mask(utf8, output, JsonObserveringEmptyContext.Instance, options); + => _masking.Mask(utf8, output, NoContext.Instance, options); /// /// Masks a UTF-8 JSON payload held in several buffers, for example read from a PipeReader, into @@ -185,11 +186,16 @@ public MaskResult Mask(ReadOnlySpan utf8, IBufferWriter output, Json /// Limits and output settings; when omitted. /// Status, bytes written and the input offset where reading stopped. public MaskResult Mask(in ReadOnlySequence utf8, IBufferWriter output, JsonObserverOptions? options = null) - => _masking.Mask(utf8, output, JsonObserveringEmptyContext.Instance, options); + => _masking.Mask(utf8, output, NoContext.Instance, options); /// - public JsonPathExplanation Explain(string path, JsonTokenType valueKind = JsonTokenType.String, JsonObserverOptions? options = null) + public PathExplanation Explain([StringSyntax(PathSyntax)] string path, ValueKind valueKind = ValueKind.String, JsonObserverOptions? options = null) => _masking.Explain(path, valueKind, options); + + /// + /// The syntax name of the paths Explain accepts, for editors that highlight string syntaxes. + /// + public const string PathSyntax = "DragoAnt.ObserverPath"; } /// @@ -199,11 +205,11 @@ public JsonPathExplanation Explain(string path, JsonTokenType valueKind = JsonTo /// Type that read rules write extracted values to. public sealed class JsonObserver { - private readonly JsonObserverDelegate _maskDelegate; + private readonly ObserveRule _maskDelegate; private readonly PathExplainer _explainer; private int _maxDepth = 6; - internal JsonObserver(JsonObserverDelegate maskDelegate, PathExplainer explainer) + internal JsonObserver(ObserveRule maskDelegate, PathExplainer explainer) { _maskDelegate = maskDelegate; _explainer = explainer; @@ -214,19 +220,16 @@ internal JsonObserver(JsonObserverDelegate maskDelegate, PathExplainer /// masking anything: useful to check a configuration, to document it, or to find out why a value was masked. /// /// - /// A JSON path such as items[2].sku, $.order.card.number or $['a.b']; the first segment decides - /// whether the root is an object or an array. - /// - /// - /// JSON type of the value at the path: a scalar type, , or - /// / for a container; rules can differ by type. + /// A concrete JSON path such as items[2].sku, $.order.card.number or $['a.b'] (no wildcards or + /// recursive descent); the first segment decides whether the root is an object or an array. /// - /// The call's options, for . + /// Type of the value at the path; rules can differ by type. + /// The call's options, for . /// The deciding rule, its action, the outcome and the steps that lead there. - /// is not a JSON path. - /// is not a value type or a container start. - public JsonPathExplanation Explain(string path, JsonTokenType valueKind = JsonTokenType.String, JsonObserverOptions? options = null) - => _explainer.Explain(path, valueKind, (options ?? JsonObserverOptions.Default).PropertyNameCaseInsensitive); + /// is not a concrete JSON path. + /// is not a defined value. + public PathExplanation Explain([StringSyntax(JsonObserver.PathSyntax)] string path, ValueKind valueKind = ValueKind.String, JsonObserverOptions? options = null) + => _explainer.Explain(path, valueKind, (options ?? JsonObserverOptions.Default).NameCaseInsensitive); private static ReadOnlySpan Utf8Bom => [0xEF, 0xBB, 0xBF]; @@ -254,7 +257,7 @@ public JsonPathExplanation Explain(string path, JsonTokenType valueKind = JsonTo { if (json is null) { - result = new MaskResult(MaskStatus.NotJson, 0, 0); + result = new MaskResult { Status = MaskStatus.Unrecognized }; return null; } @@ -270,7 +273,7 @@ public JsonPathExplanation Explain(string path, JsonTokenType valueKind = JsonTo } catch (Exception) { - result = new MaskResult(MaskStatus.Invalid, 0, 0); + result = new MaskResult { Status = MaskStatus.Invalid }; return string.Empty; } finally @@ -328,9 +331,9 @@ private MaskResult Mask(ref Utf8JsonReader reader, ReadOnlySpan input, IBu using var bounded = BoundedJsonWriter.Rent(options); using var ignoreNulls = options.IgnoreNulls ? IgnoreNullsJsonWriter.Rent(bounded) : null; var (status, failedAt) = Observe(ref reader, input, (JsonWriter?)ignoreNulls ?? bounded, context, options); - if (status == MaskStatus.NotJson) + if (status == MaskStatus.Unrecognized) { - return new MaskResult(MaskStatus.NotJson, 0, 0); + return new MaskResult { Status = MaskStatus.Unrecognized }; } try @@ -341,11 +344,11 @@ private MaskResult Mask(ref Utf8JsonReader reader, ReadOnlySpan input, IBu status = MaskStatus.Truncated; } - return new MaskResult(status, written, failedAt); + return new MaskResult { Status = status, BytesWritten = written, FailedAtByte = failedAt }; } catch (Exception) { - return new MaskResult(MaskStatus.Invalid, 0, failedAt); + return new MaskResult { Status = MaskStatus.Invalid, FailedAtByte = failedAt }; } } @@ -356,12 +359,12 @@ private MaskResult Mask(ref Utf8JsonReader reader, ReadOnlySpan input, IBu /// JSON text; it may be cut short. /// Receives the values read rules extract. /// Limits; when omitted. - /// Status and the UTF-8 offset where reading stopped; for null. + /// Status and the UTF-8 offset where reading stopped; for null. public MaskResult Read(string? json, TContext context, JsonObserverOptions? options = null) { if (json is null) { - return new MaskResult(MaskStatus.NotJson, 0, 0); + return new MaskResult { Status = MaskStatus.Unrecognized }; } byte[]? input = null; @@ -372,7 +375,7 @@ public MaskResult Read(string? json, TContext context, JsonObserverOptions? opti } catch (Exception) { - return new MaskResult(MaskStatus.Invalid, 0, 0); + return new MaskResult { Status = MaskStatus.Invalid }; } finally { @@ -397,7 +400,7 @@ public MaskResult Read(ReadOnlySpan utf8, TContext context, JsonObserverOp utf8 = SkipBom(utf8); var reader = CreateReader(utf8, options); var (status, failedAt) = Observe(ref reader, utf8, JsonWriter.Empty, context, options); - return new MaskResult(status, 0, failedAt); + return new MaskResult { Status = status, FailedAtByte = failedAt }; } /// @@ -419,7 +422,7 @@ public MaskResult Read(in ReadOnlySequence utf8, TContext context, JsonObs options ??= JsonObserverOptions.Default; var reader = CreateReader(SkipBom(utf8), options); var (status, failedAt) = Observe(ref reader, default, JsonWriter.Empty, context, options); - return new MaskResult(status, 0, failedAt); + return new MaskResult { Status = status, FailedAtByte = failedAt }; } private static ReadOnlySpan SkipBom(ReadOnlySpan utf8) => utf8.StartsWith(Utf8Bom) ? utf8[Utf8Bom.Length..] : utf8; @@ -456,15 +459,15 @@ private static Utf8JsonReader CreateReader(in ReadOnlySequence utf8, JsonO TContext context, JsonObserverOptions options) { - var propPath = new PropertyPath(_maxDepth, input) { PropertyNameCaseInsensitive = options.PropertyNameCaseInsensitive }; + var propPath = new JsonWalk(_maxDepth, input, options); try { if (!reader.Read() || reader.TokenType is not (JsonTokenType.StartObject or JsonTokenType.StartArray)) { - return (MaskStatus.NotJson, 0); + return (MaskStatus.Unrecognized, 0); } - _maskDelegate(ref reader, writer, context, 0, ref propPath, JsonObserverValuePolicies.Default); + _maskDelegate(ref reader, writer, context, 0, ref propPath, JsonValuePolicy.Default.Rule); UpdateMaxDepth(propPath.MaxLength); return propPath.Stopped || writer.Stopped ? (MaskStatus.Truncated, reader.BytesConsumed) diff --git a/DragoAnt.System.Text.Json.Observer/JsonObserverDelegate.cs b/DragoAnt.System.Text.Json.Observer/JsonObserverDelegate.cs index 5824e5e..366cd61 100644 --- a/DragoAnt.System.Text.Json.Observer/JsonObserverDelegate.cs +++ b/DragoAnt.System.Text.Json.Observer/JsonObserverDelegate.cs @@ -1,40 +1,36 @@ namespace DragoAnt.System.Text.Json.Observer; /// -/// Custom rule for one JSON value: reads the token the reader stands on and writes its replacement. +/// The engine's rule: writes the value the reader stands on, passing and the default policy on to nested rules. /// -/// Type that read rules write extracted values to. -/// Positioned on the value: a scalar, or the start of an object or array. -/// Receives exactly one value. -/// Context of the call. -/// Nesting level of the rule that matched; pass it on unchanged to nested rules. -/// Path of the value; a rule that moves past a container must leave the reader on its end. -/// Policy for values no rule matches. -public delegate void JsonObserverDelegate( +internal delegate void ObserveRule( ref Utf8JsonReader reader, JsonWriter writer, TContext context, int depth, - ref PropertyPath propPath, - JsonObserverValueDelegate defaultValue); + ref JsonWalk walk, + ValueRule defaultValue); /// -/// Policy for one string, number, boolean or null value, such as . +/// The engine's policy for one string, number, boolean or null value. /// -/// Type that read rules write extracted values to. -/// Positioned on the value. -/// Receives exactly one value. -/// Context of the call. -/// Path of the value. -public delegate void JsonObserverValueDelegate( +internal delegate void ValueRule( ref Utf8JsonReader reader, JsonWriter writer, TContext context, - ref PropertyPath propPath); + ref JsonWalk walk); internal delegate (bool success, int depth) JsonPropertyMatchDelegate( int depth, - ref PropertyPath propPath, + ref JsonWalk walk, JsonTokenType tokenType); -internal delegate (bool success, int depth) JsonPropertyPathMatchDelegate(int depth, ref PropertyPath propPath); +internal delegate (bool success, int depth) JsonPropertyPathMatchDelegate(int depth, ref JsonWalk walk); + +/// +/// Custom rule for one JSON value: reads the token stands on and +/// writes exactly one value to . +/// +/// Type that read rules write extracted values to. +/// The reader, the writer, the context, the path and the options of the call. +public delegate void JsonValueRule(ref JsonValueContext context); diff --git a/DragoAnt.System.Text.Json.Observer/JsonObserverItem.cs b/DragoAnt.System.Text.Json.Observer/JsonObserverItem.cs index 015b7a2..55511a0 100644 --- a/DragoAnt.System.Text.Json.Observer/JsonObserverItem.cs +++ b/DragoAnt.System.Text.Json.Observer/JsonObserverItem.cs @@ -2,7 +2,6 @@ using System.Runtime.CompilerServices; using System.Text; using DragoAnt.System.Text.Json.Observer.Builders; -using DragoAnt.System.Text.Json.Observer.Strategies; using static System.Text.Json.JsonTokenType; namespace DragoAnt.System.Text.Json.Observer; @@ -12,7 +11,7 @@ namespace DragoAnt.System.Text.Json.Observer; /// /// Property name matching delegate. /// Masking policy delegate. -internal sealed class JsonObserverItem(JsonPropertyMatchDelegate propMatch, JsonObserverDelegate masking) +internal sealed class JsonObserverItem(JsonPropertyMatchDelegate propMatch, ObserveRule masking) { /// /// What the rule tests and does, for explanations. @@ -25,10 +24,10 @@ internal sealed class JsonObserverItem(JsonPropertyMatchDelegate propM /// Init masking for object. /// Init masking for array. /// Default policy for unknown scenarios. - public static (JsonObserverDelegate Delegate, RuleSet Obj, RuleSet Array) Any( + public static (ObserveRule Delegate, RuleSet Obj, RuleSet Array) Any( Action> initObj, Action> initArray, - JsonObserverValueDelegate? defaultValueMasking) + JsonValuePolicy? defaultValueMasking) { var (objMasking, objSet) = Obj(initObj, defaultValueMasking); var (arrayMasking, arraySet) = Array(initArray, defaultValueMasking); @@ -38,8 +37,8 @@ public static (JsonObserverDelegate Delegate, RuleSet Obj, R JsonWriter writer, TContext context, int depth, - ref PropertyPath propPath, - JsonObserverValueDelegate valuePolicy) => + ref JsonWalk propPath, + ValueRule valuePolicy) => { switch (reader.TokenType) { @@ -71,9 +70,9 @@ public static (JsonObserverDelegate Delegate, RuleSet Obj, R /// /// Init masking for object. /// Default masking for unknown scenarios. - public static (JsonObserverDelegate Delegate, RuleSet Set) Obj( + public static (ObserveRule Delegate, RuleSet Set) Obj( Action> init, - JsonObserverValueDelegate? defaultValueMasking) + JsonValuePolicy? defaultValueMasking) { var builder = new JsonObjBuilder(defaultValueMasking); init(builder); @@ -85,9 +84,9 @@ public static (JsonObserverDelegate Delegate, RuleSet Set) O /// /// Masking condition builder. /// Default masking policy. - public static (JsonObserverDelegate Delegate, RuleSet Set) Array( + public static (ObserveRule Delegate, RuleSet Set) Array( Action> init, - JsonObserverValueDelegate? defaultValuePolicy) + JsonValuePolicy? defaultValuePolicy) { var builder = new JsonArrayBuilder(defaultValuePolicy); init(builder); @@ -166,23 +165,25 @@ private static void WriteByDefault( JsonWriter writer, TContext context, int _, - ref PropertyPath propPath, - JsonObserverValueDelegate defaultValuePolicy) => + ref JsonWalk propPath, + ValueRule defaultValuePolicy) => defaultValuePolicy(ref reader, writer, context, ref propPath); /// - /// Masks a value of any JSON type with the call's ; a container is skipped. + /// Masks a value of any JSON type with the call's (or ); + /// a container is masked whole and skipped; a null value stays null. /// /// Tag handed to the strategy. - public static JsonObserverDelegate ApplyTagPolicy(MaskTag tag) + /// Strategy of this rule; the call's when null. + public static ObserveRule ApplyTagPolicy(MaskTag tag, ValueMaskStrategy? strategy = null) { return ( ref Utf8JsonReader reader, JsonWriter writer, TContext _, int __, - ref PropertyPath propPath, - JsonObserverValueDelegate ___) => + ref JsonWalk propPath, + ValueRule ___) => { if (reader.TokenType is Null) { @@ -190,43 +191,38 @@ public static JsonObserverDelegate ApplyTagPolicy(MaskTag tag) return; } - TagMasking.Mask(ref reader, writer, tag, ref propPath); + TagMasking.Mask(ref reader, writer, tag, strategy ?? propPath.Options.Strategy ?? ValueMaskStrategy.Default, ref propPath); }; } /// - /// Masks a value of any JSON type: a string arrives decoded, a number or boolean as its literal, an object or array - /// is skipped unread and arrives as null; a null value stays null. + /// Masks a value of any JSON type with a function: a string arrives decoded, a number or boolean as its literal, an + /// object or array is skipped unread and arrives as null; with a null + /// value stays null without calling it. /// - public static JsonObserverDelegate ApplyAnyPolicy(Func maskingRule, string? constant = null) => + public static ObserveRule ApplyFunctionPolicy(Func maskingRule, string? constant, MaskNulls nulls) => ApplyMaskPolicy( constant is not null ? (ref Utf8JsonReader _, TContext _) => constant - : (ref Utf8JsonReader reader, TContext context) => maskingRule(ScalarText(ref reader, decode: true), context), - keepNull: true); + : (ref Utf8JsonReader reader, TContext context) => maskingRule(ScalarText(ref reader), context), + keepNull: nulls == MaskNulls.Keep); /// - /// Like , but the function is also called for null. + /// A custom rule: hands the value to with the enclosing default policy for + /// . /// - public static JsonObserverDelegate ApplyStringPolicy(Func maskingRule, string? constant = null) => - ApplyMaskPolicy( - constant is not null - ? (ref Utf8JsonReader _, TContext _) => constant - : (ref Utf8JsonReader reader, TContext context) => maskingRule(ScalarText(ref reader, decode: true), context), - keepNull: false); - - /// - /// Like , but a string arrives as its raw, still escaped JSON text. - /// - public static JsonObserverDelegate ApplyRawPolicy(Func maskingRule) => - ApplyMaskPolicy( - (ref Utf8JsonReader reader, TContext context) => maskingRule(ScalarText(ref reader, decode: false), context), - keepNull: false); + public static ObserveRule ApplyCustomRule(JsonValueRule rule) => + (ref Utf8JsonReader reader, JsonWriter writer, TContext context, int depth, ref JsonWalk walk, ValueRule defaultValue) => + { + var value = new JsonValueContext(ref reader, writer, context, depth, ref walk, defaultValue, null); + rule(ref value); + value.CopyBack(ref reader, ref walk); + }; /// /// Masks a value of any JSON type; the function receives true or false, and null for anything else. /// - public static JsonObserverDelegate ApplyBoolPolicy(Func maskingRule) => + public static ObserveRule ApplyBoolPolicy(Func maskingRule) => ApplyMaskPolicy( (ref Utf8JsonReader reader, TContext context) => maskingRule(reader.TokenType switch { @@ -239,7 +235,7 @@ public static JsonObserverDelegate ApplyBoolPolicy(Func /// Masks a value of any JSON type; the function receives a number that fits , and null for anything else. /// - public static JsonObserverDelegate ApplyIntPolicy(Func maskingRule) => + public static ObserveRule ApplyIntPolicy(Func maskingRule) => ApplyMaskPolicy( (ref Utf8JsonReader reader, TContext context) => maskingRule(reader.TokenType is Number && reader.TryGetInt32(out var value) ? value : null, context), @@ -248,7 +244,7 @@ public static JsonObserverDelegate ApplyIntPolicy(Func /// Masks a value of any JSON type; the function receives a number that fits , and null for anything else. /// - public static JsonObserverDelegate ApplyLongPolicy(Func maskingRule) => + public static ObserveRule ApplyLongPolicy(Func maskingRule) => ApplyMaskPolicy( (ref Utf8JsonReader reader, TContext context) => maskingRule(reader.TokenType is Number && reader.TryGetInt64(out var value) ? value : null, context), @@ -257,7 +253,7 @@ public static JsonObserverDelegate ApplyLongPolicy(Func /// Masks a value of any JSON type; the function receives a number that fits , and null for anything else. /// - public static JsonObserverDelegate ApplyDecimalPolicy(Func maskingRule) => + public static ObserveRule ApplyDecimalPolicy(Func maskingRule) => ApplyMaskPolicy( (ref Utf8JsonReader reader, TContext context) => maskingRule(reader.TokenType is Number && reader.TryGetDecimal(out var value) ? value : null, context), @@ -268,15 +264,15 @@ public static JsonObserverDelegate ApplyDecimalPolicy(Func /// Writes the function's replacement for the current value whatever its type, then moves past it; a container is never read. /// - private static JsonObserverDelegate ApplyMaskPolicy(MaskToken mask, bool keepNull) + private static ObserveRule ApplyMaskPolicy(MaskToken mask, bool keepNull) { return ( ref Utf8JsonReader reader, JsonWriter writer, TContext context, int _, - ref PropertyPath propPath, - JsonObserverValueDelegate __) => + ref JsonWalk propPath, + ValueRule __) => { if (keepNull && reader.TokenType is Null) { @@ -306,14 +302,14 @@ private static JsonObserverDelegate ApplyMaskPolicy(MaskToken mask, bo /// /// The whole text of a string, number or boolean token; null for anything else. /// - private static string? ScalarText(ref Utf8JsonReader reader, bool decode) + private static string? ScalarText(ref Utf8JsonReader reader) { if (reader.TokenType is not (JsonTokenType.String or Number or True or False)) { return null; } - if (decode && reader.TokenType is JsonTokenType.String && (reader.ValueIsEscaped || reader.HasValueSequence)) + if (reader.TokenType is JsonTokenType.String && (reader.ValueIsEscaped || reader.HasValueSequence)) { var length = reader.HasValueSequence ? checked((int)reader.ValueSequence.Length) : reader.ValueSpan.Length; var buffer = ArrayPool.Shared.Rent(length); @@ -335,7 +331,7 @@ private static JsonObserverDelegate ApplyMaskPolicy(MaskToken mask, bo /// /// Property masking policies. /// Value masking delegate. - public static JsonObserverDelegate ApplyValuePolicy(JsonObserverItem[] policies, JsonObserverValueDelegate? valuePolicy) + public static ObserveRule ApplyValuePolicy(JsonObserverItem[] policies, ValueRule? valuePolicy) { var defaultPolicy = GetApplyDefaultPolicy(valuePolicy, UnknownContainers.Create(policies)); var lastReader = LastReader(policies); @@ -345,8 +341,8 @@ public static JsonObserverDelegate ApplyValuePolicy(JsonObserverItem defaultValuePolicy) => + ref JsonWalk propPath, + ValueRule defaultValuePolicy) => { switch (reader.TokenType) { @@ -383,15 +379,15 @@ public static JsonObserverDelegate ApplyValuePolicy(JsonObserverItem ApplyObjPolicy(JsonObserverItem[] policies, JsonObserverValueDelegate? valuePolicy) + internal static ObserveRule ApplyObjPolicy(JsonObserverItem[] policies, ValueRule? valuePolicy) => ApplyObjPolicy(policies, valuePolicy, UnknownContainers.Create(policies)); - internal static JsonObserverDelegate ApplyArrayPolicy(JsonObserverItem[] policies, JsonObserverValueDelegate? valuePolicy) + internal static ObserveRule ApplyArrayPolicy(JsonObserverItem[] policies, ValueRule? valuePolicy) => ApplyArrayPolicy(policies, valuePolicy, UnknownContainers.Create(policies)); - private static JsonObserverDelegate ApplyObjPolicy( + private static ObserveRule ApplyObjPolicy( JsonObserverItem[] policies, - JsonObserverValueDelegate? valuePolicy, + ValueRule? valuePolicy, UnknownContainers unknown) { var defaultPolicy = GetApplyDefaultPolicy(valuePolicy, unknown); @@ -402,8 +398,8 @@ private static JsonObserverDelegate ApplyObjPolicy( JsonWriter writer, TContext context, int depth, - ref PropertyPath propPath, - JsonObserverValueDelegate defaultValuePolicy) => + ref JsonWalk propPath, + ValueRule defaultValuePolicy) => { var effective = valuePolicy ?? defaultValuePolicy; @@ -475,9 +471,9 @@ private static JsonObserverDelegate ApplyObjPolicy( }; } - private static JsonObserverDelegate ApplyArrayPolicy( + private static ObserveRule ApplyArrayPolicy( JsonObserverItem[] policies, - JsonObserverValueDelegate? valuePolicy, + ValueRule? valuePolicy, UnknownContainers unknown) { var defaultPolicy = GetApplyDefaultPolicy(valuePolicy, unknown); @@ -488,8 +484,8 @@ private static JsonObserverDelegate ApplyArrayPolicy( JsonWriter writer, TContext context, int depth, - ref PropertyPath propPath, - JsonObserverValueDelegate defaultValuePolicy) => + ref JsonWalk propPath, + ValueRule defaultValuePolicy) => { var effective = valuePolicy ?? defaultValuePolicy; if (reader.TokenType != StartArray) @@ -551,15 +547,15 @@ private static JsonObserverDelegate ApplyArrayPolicy( }; } - private static JsonObserverDelegate GetApplyDefaultPolicy(JsonObserverValueDelegate? valuePolicy, UnknownContainers unknown) + private static ObserveRule GetApplyDefaultPolicy(ValueRule? valuePolicy, UnknownContainers unknown) { return ( ref Utf8JsonReader reader, JsonWriter writer, TContext context, int depth, - ref PropertyPath propPath, - JsonObserverValueDelegate defaultValuePolicy) => + ref JsonWalk propPath, + ValueRule defaultValuePolicy) => { var effective = valuePolicy ?? defaultValuePolicy; switch (reader.TokenType) @@ -605,7 +601,7 @@ private static JsonObserverDelegate GetApplyDefaultPolicy(JsonObserver internal static (JsonObserverItem?, int depth) MatchPolicy( JsonObserverItem[] policies, int depth, - ref PropertyPath path, + ref JsonWalk path, JsonTokenType tokenType) { foreach (var policyItem in policies) @@ -641,7 +637,7 @@ internal static void RunReads( JsonObserverItem[] policies, int lastReader, int depth, - ref PropertyPath path, + ref JsonWalk path, ref Utf8JsonReader reader, TContext context) { @@ -655,15 +651,15 @@ internal static void RunReads( } } - internal (bool success, int depth) Match(int depth, ref PropertyPath propPath, JsonTokenType token) => propMatch(depth, ref propPath, token); + internal (bool success, int depth) Match(int depth, ref JsonWalk propPath, JsonTokenType token) => propMatch(depth, ref propPath, token); internal void Apply( ref Utf8JsonReader reader, JsonWriter writer, TContext context, int depth, - ref PropertyPath propPath, - JsonObserverValueDelegate defaultValue) => + ref JsonWalk propPath, + ValueRule defaultValue) => masking(ref reader, writer, context, depth, ref propPath, defaultValue); /// @@ -671,8 +667,8 @@ internal void Apply( /// private sealed class UnknownContainers { - public JsonObserverDelegate Obj { get; private set; } = null!; - public JsonObserverDelegate Array { get; private set; } = null!; + public ObserveRule Obj { get; private set; } = null!; + public ObserveRule Array { get; private set; } = null!; public static UnknownContainers Create(JsonObserverItem[] policies) { diff --git a/DragoAnt.System.Text.Json.Observer/JsonObserverOptions.cs b/DragoAnt.System.Text.Json.Observer/JsonObserverOptions.cs index 63d76cb..176d05e 100644 --- a/DragoAnt.System.Text.Json.Observer/JsonObserverOptions.cs +++ b/DragoAnt.System.Text.Json.Observer/JsonObserverOptions.cs @@ -1,44 +1,24 @@ -using DragoAnt.System.Text.Json.Observer.Strategies; - namespace DragoAnt.System.Text.Json.Observer; /// -/// Limits and output settings of one masking call; the same for the string and the UTF-8 API. +/// Limits and output settings of one JSON observer call; the same for the string and the UTF-8 API. The limits, +/// the hash key, the strategy and the case option are those of . /// -/// Output size limit in UTF-8 bytes; when reached the output is closed and the status is . -/// -/// Longest string value written, in UTF-8 bytes; a longer one is cut, ends with an ellipsis and makes the status . -/// It applies to values written unmasked: a masking function receives the whole value, and mask output, a hash included, is never cut. -/// -/// Deepest nesting accepted; a deeper payload is . -/// Write non-ASCII and HTML-sensitive characters unescaped, which keeps logs readable. -/// -/// Key of . When empty, a random key is used for the lifetime of the process, -/// so hashes correlate within one process only. -/// -/// Strategy for rules added with a ; when null. -/// Drop properties and array items whose value is null, and objects and arrays left empty by that. -/// Write the output indented. -/// -/// Match rule names, tests and shape properties ignoring case, as by default. Pass the -/// PropertyNameCaseInsensitive of the serializer's options to match names the way deserialization does. -/// With false a rule no longer catches a differently cased name: under a block list such a value is written -/// unchanged, under an allow list it is masked. -/// -public sealed record JsonObserverOptions( - int MaxOutputBytes = int.MaxValue, - int MaxValueBytes = int.MaxValue, - int MaxDepth = 64, - bool RelaxedEscaping = true, - ReadOnlyMemory HashKey = default, - Utf8MaskStrategy? MaskStrategy = null, - bool IgnoreNulls = false, - bool Indented = false, - bool PropertyNameCaseInsensitive = true) +public sealed record JsonObserverOptions : ObserverOptions { /// - /// Defaults: no size limits, depth 64, relaxed escaping, a per-process hash key, the built-in strategy, - /// null values kept, compact output and names matched ignoring case. + /// Defaults: no size limits, depth 64, relaxed escaping, compact output, a per-process hash key, the built-in + /// strategy, null values kept and names matched ignoring case. + /// + public static new JsonObserverOptions Default { get; } = new(); + + /// + /// Write non-ASCII and HTML-sensitive characters unescaped, which keeps logs readable; on by default. + /// + public bool RelaxedEscaping { get; init; } = true; + + /// + /// Write the output indented. /// - public static JsonObserverOptions Default { get; } = new(); + public bool Indented { get; init; } } diff --git a/DragoAnt.System.Text.Json.Observer/JsonObserverValuePolicies.cs b/DragoAnt.System.Text.Json.Observer/JsonObserverValuePolicies.cs deleted file mode 100644 index 26d310a..0000000 --- a/DragoAnt.System.Text.Json.Observer/JsonObserverValuePolicies.cs +++ /dev/null @@ -1,212 +0,0 @@ -using DragoAnt.System.Text.Json.Observer.Builders; -using static System.Text.Json.JsonTokenType; - -namespace DragoAnt.System.Text.Json.Observer; - -/// -/// Default policies for values no rule matches, for observers without a context. Pass one as the default policy of a factory or a rule. -/// -public static class JsonObserverValuePolicies -{ - /// - /// The default policy, . - /// - public static readonly JsonObserverValueDelegate Default - = JsonObserverValuePolicies.Default; - - /// - /// Writes every value unchanged: only values matched by a rule are masked. - /// - public static readonly JsonObserverValueDelegate BlockList - = JsonObserverValuePolicies.BlockList; - - /// - /// Writes every string, number and boolean as "***" and keeps null: only values allowed by a rule are shown. - /// - public static readonly JsonObserverValueDelegate AllowList - = JsonObserverValuePolicies.AllowList; - - /// - /// The 1.x allow list: strings become "#str#*****", numbers "#number#*****", booleans and null pass. - /// - [Obsolete(LegacyAllowListMessage)] - public static readonly JsonObserverValueDelegate LegacyAllowList - = JsonObserverValuePolicies.LegacyAllowList; - - internal const string LegacyAllowListMessage = - "The 1.x allow list keeps booleans and the type of masked values. Use AllowList, or JsonObserver.FromShape for a structure-aware allow list."; - - /// - /// Writes every value as null. - /// - public static readonly JsonObserverValueDelegate NullList - = JsonObserverValuePolicies.NullList; - - /// - /// A default policy with its own rules that match the end of a property's path at any depth, for example every - /// password or every card.number, wherever it is nested. - /// - /// Adds the rules, see . - /// Policy for values none of these rules match; when null. - /// A policy to pass where a default policy is expected. - public static JsonObserverValueDelegate Relative( - Action> init, - JsonObserverValueDelegate? defaultValuePolicy = null) - => JsonObserverValuePolicies.Relative(init, defaultValuePolicy); -} - -/// -/// Default policies for values no rule matches. Pass one as the default policy of a factory or a rule. -/// -public static class JsonObserverValuePolicies -{ - /// - /// The default policy, . - /// - public static readonly JsonObserverValueDelegate Default = AllowList; - - /// - /// A default policy with its own rules that match the end of a property's path at any depth, for example every - /// password or every card.number, wherever it is nested. - /// - /// Adds the rules, see . - /// Policy for values none of these rules match; when null. - /// A policy to pass where a default policy is expected. - public static JsonObserverValueDelegate Relative( - Action> init, - JsonObserverValueDelegate? defaultValuePolicy = null) - { - var builder = new JsonValuePolicyBuilder(true, defaultValuePolicy); - init(builder); - var relative = new RelativeValuePolicy( - JsonValuePolicyBuilder.Build(builder), - JsonValuePolicyBuilder.BuildItems(builder), - defaultValuePolicy ?? Default); - - return relative.Invoke; - } - - /// - /// Writes every value as null. - /// - public static void NullList(ref Utf8JsonReader reader, JsonWriter writer, TContext context, ref PropertyPath propPath) - { - switch (reader.TokenType) - { - case JsonTokenType.String: - case Number: - case True: - case False: - case Null: - writer.WriteNullValue(); - break; - case Comment: - case StartArray: - case StartObject: - case PropertyName: - case EndObject: - case EndArray: - case JsonTokenType.None: - default: - throw new JsonObserverException("Wrong path"); - } - } - - /// - /// Writes every value unchanged: only values matched by a rule are masked. - /// - public static void BlockList(ref Utf8JsonReader reader, JsonWriter writer, TContext context, ref PropertyPath propPath) - { - switch (reader.TokenType) - { - case JsonTokenType.String: - writer.CopyStringValue(ref reader); - break; - case Number: - writer.CopyRawValue(ref reader); - break; - case True: - writer.WriteBooleanValue(true); - break; - case False: - writer.WriteBooleanValue(false); - break; - case Null: - writer.WriteNullValue(); - break; - case Comment: - case StartArray: - case StartObject: - case PropertyName: - case EndObject: - case EndArray: - case JsonTokenType.None: - default: - throw new JsonObserverException("Wrong path"); - } - } - - /// - /// Allow (or white) list policy approach: every value no rule allows is written as "***", whatever its type; - /// null stays null. - /// - public static void AllowList(ref Utf8JsonReader reader, JsonWriter writer, TContext context, ref PropertyPath propPath) - { - switch (reader.TokenType) - { - case JsonTokenType.String: - case Number: - case True: - case False: - writer.MaskOutput = true; - writer.WriteStringValue("***"u8); - writer.MaskOutput = false; - break; - case Null: - writer.WriteNullValue(); - break; - default: - throw new JsonObserverException("Wrong path"); - } - } - - /// - /// The 1.x allow list: strings become "#str#*****", numbers "#number#*****", booleans and null pass. - /// - [Obsolete(JsonObserverValuePolicies.LegacyAllowListMessage)] - public static void LegacyAllowList(ref Utf8JsonReader reader, JsonWriter writer, TContext context, ref PropertyPath propPath) - { - switch (reader.TokenType) - { - case JsonTokenType.String: - writer.MaskOutput = true; - writer.WriteStringValue("#str#*****"); - writer.MaskOutput = false; - break; - case Number: - writer.MaskOutput = true; - writer.WriteStringValue("#number#*****"); - writer.MaskOutput = false; - break; - case True: - writer.WriteBooleanValue(true); - break; - case False: - writer.WriteBooleanValue(false); - break; - case Null: - writer.WriteNullValue(); - break; - case Comment: - case StartArray: - case StartObject: - case PropertyName: - case EndObject: - case EndArray: - case JsonTokenType.None: - default: - throw new JsonObserverException("Wrong path"); - } - } -} - diff --git a/DragoAnt.System.Text.Json.Observer/JsonObserveringEmptyContext.cs b/DragoAnt.System.Text.Json.Observer/JsonObserveringEmptyContext.cs deleted file mode 100644 index d538f23..0000000 --- a/DragoAnt.System.Text.Json.Observer/JsonObserveringEmptyContext.cs +++ /dev/null @@ -1,16 +0,0 @@ -namespace DragoAnt.System.Text.Json.Observer; - -/// -/// Context of an observer that only masks and extracts nothing, such as every . -/// -public sealed class JsonObserveringEmptyContext -{ - /// - /// The only instance. - /// - public static readonly JsonObserveringEmptyContext Instance = new(); - - private JsonObserveringEmptyContext() - { - } -} diff --git a/DragoAnt.System.Text.Json.Observer/JsonPathExplanation.cs b/DragoAnt.System.Text.Json.Observer/JsonPathExplanation.cs deleted file mode 100644 index aee7b27..0000000 --- a/DragoAnt.System.Text.Json.Observer/JsonPathExplanation.cs +++ /dev/null @@ -1,48 +0,0 @@ -namespace DragoAnt.System.Text.Json.Observer; - -/// -/// What an observer does with a value, as reported by . -/// -public enum JsonPathOutcome -{ - /// - /// The value is written as it is. - /// - Unchanged, - - /// - /// The value is replaced: masked, hashed, written as null, or an object or array masked whole. - /// - Masked, - - /// - /// The value is handed to the context by a read rule and written as it is. - /// - Read, - - /// - /// A custom rule or policy decides; the observer cannot tell what it writes. - /// - Custom, - - /// - /// A payload with this structure is not masked at all: its status is . - /// - Invalid, -} - -/// -/// Which rule or policy of an observer handles a JSON path, and what it does with the value there. -/// -/// The path explained, normalized, for example lines[0].qty. -/// What happens to the value. -/// The rule or policy that decides, for example Match("qty") or default policy AllowList. -/// What it does, for example MaskAny("***") or writes "***". -/// How the observer gets there, one entry per level of the path. -public sealed record JsonPathExplanation(string Path, JsonPathOutcome Outcome, string Rule, string Action, IReadOnlyList Steps) -{ - /// - /// One line, for example lines[0].qty: Masked by Match("qty") → MaskAny("***"). - /// - public override string ToString() => $"{Path}: {Outcome} by {Rule} → {Action}"; -} diff --git a/DragoAnt.System.Text.Json.Observer/JsonShape.cs b/DragoAnt.System.Text.Json.Observer/JsonShape.cs index 522883f..18680d1 100644 --- a/DragoAnt.System.Text.Json.Observer/JsonShape.cs +++ b/DragoAnt.System.Text.Json.Observer/JsonShape.cs @@ -2,7 +2,6 @@ using System.Text; using System.Text.Json.Nodes; using System.Text.Json.Serialization.Metadata; -using DragoAnt.System.Text.Json.Observer.Strategies; namespace DragoAnt.System.Text.Json.Observer; @@ -111,6 +110,8 @@ private JsonShape(JsonShapeKind kind, MaskTag tag = default, JsonShape? item = n /// public static JsonShape Opaque { get; } = new(JsonShapeKind.Opaque); + internal static JsonShape UnknownMaskWhole { get; } = new(JsonShapeKind.Opaque); + internal static JsonShape UnknownDescend { get; } = new(JsonShapeKind.Opaque); internal static JsonShape UnknownPassThrough { get; } = new(JsonShapeKind.Opaque); @@ -198,7 +199,7 @@ public static JsonShape FromTypeInfo(JsonTypeInfo typeInfo, FuncUnescaped UTF-8 name. /// Compare names ignoring case, as by default. /// The property, or null when the shape does not know it or is not an object. - public JsonShapeProperty? FindMember(ReadOnlySpan utf8Name, bool propertyNameCaseInsensitive = true) + internal JsonShapeProperty? FindMember(ReadOnlySpan utf8Name, bool propertyNameCaseInsensitive = true) { if (!_sealed) { diff --git a/DragoAnt.System.Text.Json.Observer/JsonShapeOptions.cs b/DragoAnt.System.Text.Json.Observer/JsonShapeOptions.cs index 73ace67..0cb11fc 100644 --- a/DragoAnt.System.Text.Json.Observer/JsonShapeOptions.cs +++ b/DragoAnt.System.Text.Json.Observer/JsonShapeOptions.cs @@ -1,59 +1,47 @@ namespace DragoAnt.System.Text.Json.Observer; /// -/// What a shape-driven observer does with a property its shape does not know. +/// Settings of an observer built from a . /// -public enum UnknownMemberPolicy +public sealed record JsonShapeOptions { /// - /// The value is masked whole, as "***", whatever its JSON type; its content is never read. + /// Defaults: unknown properties masked whole as "***", null kept, names matched as the call says. /// - MaskWhole, + public static JsonShapeOptions Default { get; } = new(); /// - /// Objects and arrays are descended so that their property names stay visible; every value inside is masked. + /// What happens to a property the shape does not know; by default. /// - Descend, + public UnknownMemberPolicy Unknown { get; init; } = UnknownMemberPolicy.MaskWhole; /// - /// The value is written as is. Only the properties the shape marks as sensitive are masked. + /// How the values of unknown properties are masked, with and + /// ; ("***") by default. /// - PassThrough, -} + public MaskTag UnknownTag { get; init; } = MaskTag.Full; -/// -/// Settings of an observer built from a . -/// -/// What happens to a property the shape does not know. -/// Write null as null where a value would be masked; otherwise it is masked too. -/// -/// Match property names ignoring case; null follows -/// of the call, which ignores case by default. A property whose name does not match is unknown, so with -/// a differently cased sensitive property is written unchanged. -/// -public sealed record JsonShapeOptions( - UnknownMemberPolicy Unknown = UnknownMemberPolicy.MaskWhole, - bool KeepNulls = true, - bool? PropertyNameCaseInsensitive = null) -{ /// - /// Defaults: unknown properties masked whole, null kept, names matched as the call says. + /// Write null as null where a value would be masked; otherwise it is masked too. On by default. /// - public static JsonShapeOptions Default { get; } = new(); + public bool KeepNulls { get; init; } = true; + + /// + /// Match property names ignoring case; null follows of + /// the call, which ignores case by default. A property whose name does not match is unknown, so with + /// a differently cased sensitive property is written unchanged. + /// + public bool? NameCaseInsensitive { get; init; } /// /// Options that match property names the way the serializer does, typically those the shape was built from /// (JsonTypeInfo.Options). /// /// Options whose is used. - /// What happens to a property the shape does not know. - /// Write null as null where a value would be masked. - public static JsonShapeOptions FromSerializerOptions( - JsonSerializerOptions serializerOptions, - UnknownMemberPolicy unknown = UnknownMemberPolicy.MaskWhole, - bool keepNulls = true) + /// is null. + public static JsonShapeOptions FromSerializerOptions(JsonSerializerOptions serializerOptions) { ArgumentNullException.ThrowIfNull(serializerOptions); - return new JsonShapeOptions(unknown, keepNulls, serializerOptions.PropertyNameCaseInsensitive); + return new JsonShapeOptions { NameCaseInsensitive = serializerOptions.PropertyNameCaseInsensitive }; } } diff --git a/DragoAnt.System.Text.Json.Observer/JsonValueContext.cs b/DragoAnt.System.Text.Json.Observer/JsonValueContext.cs new file mode 100644 index 0000000..36cfac3 --- /dev/null +++ b/DragoAnt.System.Text.Json.Observer/JsonValueContext.cs @@ -0,0 +1,94 @@ +using System.Diagnostics.CodeAnalysis; + +namespace DragoAnt.System.Text.Json.Observer; + +/// +/// Everything a works with: the reader on the value, the writer, the context and +/// the path. Valid only during the call it is passed to; new members are added without breaking existing rules. +/// +/// Type that read rules write extracted values to. +public ref struct JsonValueContext +{ + private Utf8JsonReader _reader; + private JsonWalk _walk; + private readonly ObserveRule? _fallback; + private readonly ValueRule _defaultValue; + private readonly int _depth; + + internal JsonValueContext( + scoped ref Utf8JsonReader reader, + JsonWriter writer, + TContext context, + int depth, + scoped ref JsonWalk walk, + ValueRule defaultValue, + ObserveRule? fallback) + { + _reader = reader; + _walk = walk; + Writer = writer; + Context = context; + _depth = depth; + _defaultValue = defaultValue; + _fallback = fallback; + } + + /// + /// The reader, positioned on the value: a scalar, or the start of an object or array. A rule that moves past a + /// container must leave it on the container's end. + /// + [UnscopedRef] + public ref Utf8JsonReader Reader => ref _reader; + + /// + /// JSON type of the value the reader stands on. + /// + public readonly JsonTokenType TokenType => _reader.TokenType; + + /// + /// Receives exactly one value. + /// + public JsonWriter Writer { get; } + + /// + /// The context of the call, where read rules put extracted values. + /// + public TContext Context { get; } + + /// + /// Path of the value from the root, array items included. + /// + public readonly DataPath Path => _walk.Path; + + /// + /// Options of the call. + /// + public readonly JsonObserverOptions Options => _walk.Options; + + /// + /// Writes the value the way the enclosing default policy would: a string, number, boolean or null through the + /// policy, an object or array descended with it. Use it for the values a custom rule does not handle itself. + /// + public void WriteDefault() + { + if (_fallback is not null) + { + _fallback(ref _reader, Writer, Context, _depth, ref _walk, _defaultValue); + return; + } + + if (_reader.TokenType is JsonTokenType.StartObject or JsonTokenType.StartArray) + { + BuiltInPolicies.MaskContainer(ref _reader, Writer, ref _walk); + return; + } + + _defaultValue(ref _reader, Writer, Context, ref _walk); + } + + internal readonly void CopyBack(ref Utf8JsonReader reader, ref JsonWalk walk) + { + reader = _reader; + walk = _walk; + } +} diff --git a/DragoAnt.System.Text.Json.Observer/JsonValuePolicy.cs b/DragoAnt.System.Text.Json.Observer/JsonValuePolicy.cs new file mode 100644 index 0000000..0593848 --- /dev/null +++ b/DragoAnt.System.Text.Json.Observer/JsonValuePolicy.cs @@ -0,0 +1,234 @@ +using DragoAnt.System.Text.Json.Observer.Builders; +using static System.Text.Json.JsonTokenType; + +namespace DragoAnt.System.Text.Json.Observer; + +/// +/// What happens to values no rule matches: one of the built-in values (which convert to it), +/// a set of +/// rules, or a custom rule. +/// +/// Type that read rules write extracted values to. +public sealed class JsonValuePolicy +{ + private JsonValuePolicy(ValueRule rule, string name, ValuePolicy? builtIn, RelativeValuePolicy? relative) + { + Rule = rule; + Name = name; + BuiltIn = builtIn; + Relative = relative; + } + + internal static JsonValuePolicy Default { get; } = From(ValuePolicy.AllowList); + + internal ValueRule Rule { get; } + + internal string Name { get; } + + internal ValuePolicy? BuiltIn { get; } + + internal RelativeValuePolicy? Relative { get; } + + /// + /// A custom policy: is called for every string, number, boolean or null value no + /// rule matches and writes exactly one value. Objects and arrays are descended with the same policy. + /// + /// The policy. + /// is null. + public static JsonValuePolicy Custom(JsonValueRule rule) + { + ArgumentNullException.ThrowIfNull(rule); + return new JsonValuePolicy( + (ref Utf8JsonReader reader, JsonWriter writer, TContext context, ref JsonWalk walk) => + { + var value = new JsonValueContext(ref reader, writer, context, 0, ref walk, BuiltInPolicies.AllowList, null); + rule(ref value); + value.CopyBack(ref reader, ref walk); + }, + "custom default policy", + null, + null); + } + + internal static JsonValuePolicy AnyDepth(RelativeValuePolicy relative) => + new(relative.Invoke, "AnyDepth", null, relative); + + /// + /// The built-in policy . + /// + /// A built-in policy such as . + public static implicit operator JsonValuePolicy(ValuePolicy policy) => From(policy); + + /// + /// The policy's name, for example AllowList, AnyDepth or custom default policy. + /// + public override string ToString() => Name; + + private static JsonValuePolicy From(ValuePolicy policy) + { + ArgumentNullException.ThrowIfNull(policy); + return policy.Kind switch + { + ValuePolicyKind.BlockList => Cache.BlockList, + ValuePolicyKind.AllowList => Cache.AllowList, + ValuePolicyKind.NullList => Cache.NullList, + _ => new JsonValuePolicy(BuiltInPolicies.Tagged(policy.Tag), policy.ToString(), policy, null), + }; + } + + private static class Cache + { + public static readonly JsonValuePolicy BlockList = + new(BuiltInPolicies.BlockList, nameof(ValuePolicy.BlockList), ValuePolicy.BlockList, null); + + public static readonly JsonValuePolicy AllowList = + new(BuiltInPolicies.AllowList, nameof(ValuePolicy.AllowList), ValuePolicy.AllowList, null); + + public static readonly JsonValuePolicy NullList = + new(BuiltInPolicies.NullList, nameof(ValuePolicy.NullList), ValuePolicy.NullList, null); + } +} + +/// +/// Policies with rules of their own, and custom policies. +/// +public static class JsonValuePolicy +{ + /// + /// A default policy with its own rules that match the end of a property's path at any depth, for example every + /// password or every card.number, wherever it is nested. + /// + /// Adds the rules, see . + /// Policy for values none of these rules match; when null. + /// A policy to pass where a default policy is expected. + public static JsonValuePolicy AnyDepth( + Action> init, + JsonValuePolicy? fallback = null) + => AnyDepth(init, fallback); + + /// + /// A default policy with its own rules, for an observer with a context, that match the end of a property's path at + /// any depth. + /// + /// Adds the rules, see . + /// Policy for values none of these rules match; when null. + /// Type that read rules write extracted values to. + /// A policy to pass where a default policy is expected. + public static JsonValuePolicy AnyDepth( + Action> init, + JsonValuePolicy? fallback = null) + { + ArgumentNullException.ThrowIfNull(init); + var effectiveFallback = fallback ?? JsonValuePolicy.Default; + var builder = new JsonAnyDepthBuilder(effectiveFallback); + init(builder); + var relative = new RelativeValuePolicy( + JsonAnyDepthBuilder.Build(builder), + JsonAnyDepthBuilder.BuildItems(builder), + effectiveFallback); + + return JsonValuePolicy.AnyDepth(relative); + } + + /// + /// A custom policy for an observer without a context. + /// + /// Called for every string, number, boolean or null value no rule matches; writes exactly one value. + public static JsonValuePolicy Custom(JsonValueRule rule) => JsonValuePolicy.Custom(rule); +} + +/// +/// The engine side of the built-in policies. +/// +internal static class BuiltInPolicies +{ + public static readonly ValueRule BlockList = WriteUnchanged; + public static readonly ValueRule AllowList = WriteStars; + public static readonly ValueRule NullList = WriteNull; + + public static ValueRule Tagged(MaskTag tag) => + (ref Utf8JsonReader reader, JsonWriter writer, TContext _, ref JsonWalk walk) => + { + if (reader.TokenType is Null) + { + writer.WriteNullValue(); + return; + } + + TagMasking.Mask(ref reader, writer, tag, ref walk); + }; + + /// + /// Writes "***" for an object or array and moves past it unread. + /// + public static void MaskContainer(ref Utf8JsonReader reader, JsonWriter writer, ref JsonWalk walk) + { + writer.MaskOutput = true; + writer.WriteStringValue("***"u8); + writer.MaskOutput = false; + if (!reader.TrySkip()) + { + walk.Stop(); + } + } + + private static void WriteNull(ref Utf8JsonReader reader, JsonWriter writer, TContext context, ref JsonWalk walk) + { + switch (reader.TokenType) + { + case JsonTokenType.String: + case Number: + case True: + case False: + case Null: + writer.WriteNullValue(); + break; + default: + throw new JsonObserverException("Wrong path"); + } + } + + private static void WriteUnchanged(ref Utf8JsonReader reader, JsonWriter writer, TContext context, ref JsonWalk walk) + { + switch (reader.TokenType) + { + case JsonTokenType.String: + writer.CopyStringValue(ref reader); + break; + case Number: + writer.CopyRawValue(ref reader); + break; + case True: + writer.WriteBooleanValue(true); + break; + case False: + writer.WriteBooleanValue(false); + break; + case Null: + writer.WriteNullValue(); + break; + default: + throw new JsonObserverException("Wrong path"); + } + } + + private static void WriteStars(ref Utf8JsonReader reader, JsonWriter writer, TContext context, ref JsonWalk walk) + { + switch (reader.TokenType) + { + case JsonTokenType.String: + case Number: + case True: + case False: + writer.MaskOutput = true; + writer.WriteStringValue("***"u8); + writer.MaskOutput = false; + break; + case Null: + writer.WriteNullValue(); + break; + default: + throw new JsonObserverException("Wrong path"); + } + } +} diff --git a/DragoAnt.System.Text.Json.Observer/JsonWalk.cs b/DragoAnt.System.Text.Json.Observer/JsonWalk.cs new file mode 100644 index 0000000..4e836db --- /dev/null +++ b/DragoAnt.System.Text.Json.Observer/JsonWalk.cs @@ -0,0 +1,61 @@ +namespace DragoAnt.System.Text.Json.Observer; + +/// +/// State of one observer pass: the path, whether reading must stop, and the counters rules need. +/// +internal ref struct JsonWalk +{ + public DataPath Path; + private readonly bool _hasInput; + + public JsonWalk(int capacity, ReadOnlySpan input, JsonObserverOptions options) + { + Path = new DataPath(input, capacity, options.NameCaseInsensitive); + Options = options; + _hasInput = !input.IsEmpty; + } + + public JsonObserverOptions Options { get; } + + /// + /// The input ended inside a value, or a rule asked to stop: every rule must stop reading. + /// + public bool Stopped { get; private set; } + + /// + /// Position of the next value handed to a strategy. + /// + public long ValueIndex { get; set; } + + public readonly int CurrentDepth => Path.Length - 1; + + public readonly int MaxLength => Path.MaxLength; + + public readonly ReadOnlySpan CurrentUtf8 => Path.LastName; + + public void Stop() => Stopped = true; + + /// + /// Adds the property name the reader stands on, pointing into the input when it is one span and unescaped. + /// + public void AddPropertyName(ref Utf8JsonReader reader) + { + if (!reader.HasValueSequence && !reader.ValueIsEscaped && _hasInput) + { + Path.PushInputName(checked((int)reader.TokenStartIndex + 1), reader.ValueSpan.Length); + return; + } + + var maxLength = reader.HasValueSequence ? checked((int)reader.ValueSequence.Length) : reader.ValueSpan.Length; + var written = reader.CopyString(Path.ReserveName(maxLength)); + Path.PushReservedName(written); + } + + public void AddPropertyName(ReadOnlySpan utf8Name) => Path.PushName(utf8Name); + + public void AddArrayItem(int index) => Path.PushItem(index); + + public void RemovePropertyName() => Path.Pop(); + + public void Dispose() => Path.Dispose(); +} diff --git a/DragoAnt.System.Text.Json.Observer/JsonWriter.cs b/DragoAnt.System.Text.Json.Observer/JsonWriter.cs index ddaf480..f55e18f 100644 --- a/DragoAnt.System.Text.Json.Observer/JsonWriter.cs +++ b/DragoAnt.System.Text.Json.Observer/JsonWriter.cs @@ -115,7 +115,7 @@ private protected JsonWriter() public abstract void WriteNumberValue(double value); /// - /// Options of the current call; rules with a read their strategy and hash key here. + /// Options of the current call; rules with a read their strategy and hash key here. /// internal virtual JsonObserverOptions Options => JsonObserverOptions.Default; @@ -125,7 +125,7 @@ private protected JsonWriter() internal virtual bool Stopped => false; /// - /// The strings written now replace a value, so does not cut them. + /// The strings written now replace a value, so does not cut them. /// internal virtual bool MaskOutput { diff --git a/DragoAnt.System.Text.Json.Observer/MaskResult.cs b/DragoAnt.System.Text.Json.Observer/MaskResult.cs deleted file mode 100644 index eb2a0b0..0000000 --- a/DragoAnt.System.Text.Json.Observer/MaskResult.cs +++ /dev/null @@ -1,42 +0,0 @@ -namespace DragoAnt.System.Text.Json.Observer; - -/// -/// Outcome of masking or reading a JSON payload. Whatever the status, the output never holds a value a rule masks. -/// -public enum MaskStatus -{ - /// - /// The whole payload was masked and every value was written in full. - /// - Masked, - - /// - /// Part of the data is missing from the output, which is still valid JSON: - /// the payload ended inside the document or the output reached - /// (the output holds the masked part with every open object and array closed, and - /// is where reading stopped), or a string longer than was cut - /// (the whole document was read and is -1). - /// - Truncated, - - /// - /// The payload is empty or its root is not an object or an array. Nothing was written. - /// - NotJson, - - /// - /// The payload is not valid JSON, or a rule failed. The output holds the masked part read before the failure, - /// with every open object and array closed; nothing after the failure is written. - /// - Invalid, -} - -/// -/// Result of masking or reading a JSON payload. -/// -/// What happened. -/// UTF-8 bytes written to the output; 0 when only reading. -/// -/// UTF-8 offset in the input where reading stopped; -1 when the whole payload was read, 0 for . -/// -public readonly record struct MaskResult(MaskStatus Status, int BytesWritten, long FailedAtByte); diff --git a/DragoAnt.System.Text.Json.Observer/PropertyPathMatch.cs b/DragoAnt.System.Text.Json.Observer/NamePathMatch.cs similarity index 52% rename from DragoAnt.System.Text.Json.Observer/PropertyPathMatch.cs rename to DragoAnt.System.Text.Json.Observer/NamePathMatch.cs index 22d682c..78219f2 100644 --- a/DragoAnt.System.Text.Json.Observer/PropertyPathMatch.cs +++ b/DragoAnt.System.Text.Json.Observer/NamePathMatch.cs @@ -1,21 +1,14 @@ -using DragoAnt.System.Text.Json.Observer.Strategies; - namespace DragoAnt.System.Text.Json.Observer; /// -/// Property path matching class. +/// A rule's name tests, one per level: from the enclosing object down (absolute) or at the end of the path (any depth). /// -internal sealed class PropertyPathMatch +internal sealed class NamePathMatch { - public const StringComparison DefaultComparison = StringComparison.OrdinalIgnoreCase; - private readonly NameMatcher[] _matches; - - public PropertyPathMatch(PropMatchingStrategy[] matches) - : this(matches.Select(m => m.Matcher).ToArray()) - { - } + private readonly NameMatch[] _matches; + private readonly bool _isPath; - private PropertyPathMatch(NameMatcher[] matches) + public NamePathMatch(NameMatch[] matches, bool isPath) { if (matches.Length == 0) { @@ -23,18 +16,19 @@ private PropertyPathMatch(NameMatcher[] matches) } _matches = matches; + _isPath = isPath; } - public string Describe() => $"Match({string.Join(", ", _matches.Select(m => m.Describe()))})"; + public string Describe() => $"{(_isPath ? "Path" : "Match")}({string.Join(", ", _matches.Select(m => m.ToString()))})"; - public (bool success, int depth) RelativeMatch(int depth, ref PropertyPath propPath) + public (bool success, int depth) RelativeMatch(int depth, ref JsonWalk walk) { - var last = propPath.CurrentDepth; + var last = walk.CurrentDepth; var j = -1; for (var i = _matches.Length - 1; i >= 0; i--) { j++; - if (!_matches[i].Match(ref propPath, last - j)) + if (!_matches[i].IsMatch(in walk.Path, last - j)) { return (false, _matches.Length); } @@ -43,11 +37,11 @@ private PropertyPathMatch(NameMatcher[] matches) return (true, _matches.Length); } - public (bool success, int depth) AbsoluteMatch(int depth, ref PropertyPath propPath) + public (bool success, int depth) AbsoluteMatch(int depth, ref JsonWalk walk) { for (var i = 0; i < _matches.Length; i++) { - if (!_matches[i].Match(ref propPath, i + depth)) + if (!_matches[i].IsMatch(in walk.Path, i + depth)) { return (false, _matches.Length); } diff --git a/DragoAnt.System.Text.Json.Observer/PathExplainer.cs b/DragoAnt.System.Text.Json.Observer/PathExplainer.cs index b5a9197..c412717 100644 --- a/DragoAnt.System.Text.Json.Observer/PathExplainer.cs +++ b/DragoAnt.System.Text.Json.Observer/PathExplainer.cs @@ -16,28 +16,33 @@ internal readonly record struct PathSegment(string? Name, int Index) /// internal abstract class PathExplainer { - public JsonPathExplanation Explain(string path, JsonTokenType valueKind, bool propertyNameCaseInsensitive) + public PathExplanation Explain(string path, ValueKind valueKind, bool propertyNameCaseInsensitive) { ArgumentNullException.ThrowIfNull(path); - if (valueKind is not (JsonTokenType.String or JsonTokenType.Number or JsonTokenType.True or JsonTokenType.False - or JsonTokenType.Null or JsonTokenType.StartObject or JsonTokenType.StartArray)) + var token = valueKind switch { - throw new ArgumentOutOfRangeException(nameof(valueKind), valueKind, "Expected a value type, StartObject or StartArray."); - } + ValueKind.String => JsonTokenType.String, + ValueKind.Number => JsonTokenType.Number, + ValueKind.Boolean => JsonTokenType.True, + ValueKind.Null => JsonTokenType.Null, + ValueKind.Object => JsonTokenType.StartObject, + ValueKind.Array => JsonTokenType.StartArray, + _ => throw new ArgumentOutOfRangeException(nameof(valueKind), valueKind, "Expected a defined value kind."), + }; var segments = Parse(path); var normalized = Format(segments); if (segments.Count == 0) { - return new JsonPathExplanation("$", JsonPathOutcome.Unchanged, "root", "the root's rules apply to its members", []); + return new PathExplanation { Path = "$", Outcome = PathOutcome.Unchanged, Rule = "root", Action = "the root's rules apply to its members" }; } var steps = new List(); - var (outcome, rule, action) = Explain(segments, valueKind, propertyNameCaseInsensitive, steps); - return new JsonPathExplanation(normalized, outcome, rule, action, steps); + var (outcome, rule, action) = Explain(segments, token, propertyNameCaseInsensitive, steps); + return new PathExplanation { Path = normalized, Outcome = outcome, Rule = rule, Action = action, Steps = steps }; } - protected abstract (JsonPathOutcome Outcome, string Rule, string Action) Explain( + protected abstract (PathOutcome Outcome, string Rule, string Action) Explain( IReadOnlyList segments, JsonTokenType valueKind, bool propertyNameCaseInsensitive, @@ -48,10 +53,10 @@ protected static JsonTokenType TokenAt(IReadOnlyList segments, int : segments[index + 1].IsIndex ? JsonTokenType.StartArray : JsonTokenType.StartObject; - protected static PropertyPath PathOf(IReadOnlyList segments, bool propertyNameCaseInsensitive) => - new(segments.Count, default) { PropertyNameCaseInsensitive = propertyNameCaseInsensitive }; + protected static JsonWalk PathOf(IReadOnlyList segments, bool propertyNameCaseInsensitive) => + new(segments.Count, default, JsonObserverOptions.Default with { NameCaseInsensitive = propertyNameCaseInsensitive }); - protected static void Push(ref PropertyPath path, PathSegment segment) + protected static void Push(ref JsonWalk path, PathSegment segment) { if (segment.IsIndex) { @@ -76,13 +81,29 @@ protected static string Format(IReadOnlyList segments, int count = } else { - PropertyPath.AppendName(text, segments[i].Name!, first: i == 0); + AppendName(text, segments[i].Name!, first: i == 0); } } return text.ToString(); } + private static void AppendName(StringBuilder text, string name, bool first) + { + if (name.Length == 0 || name.AsSpan().IndexOfAny(".[]'") >= 0) + { + text.Append("['").Append(name.Replace("'", "\\'", StringComparison.Ordinal)).Append("']"); + return; + } + + if (!first) + { + text.Append('.'); + } + + text.Append(name); + } + /// /// Parses $.a.b[2]['c.d']; the leading $ and the first dot are optional. /// diff --git a/DragoAnt.System.Text.Json.Observer/PropertyPath.cs b/DragoAnt.System.Text.Json.Observer/PropertyPath.cs deleted file mode 100644 index f07fbeb..0000000 --- a/DragoAnt.System.Text.Json.Observer/PropertyPath.cs +++ /dev/null @@ -1,309 +0,0 @@ -using System.Buffers; -using System.Text; - -namespace DragoAnt.System.Text.Json.Observer; - -/// -/// Path of the value a rule is called for: one level per enclosing property or array item, from the root down. -/// -/// -/// Valid only during the call it is passed to. Names are kept as UTF-8 and decoded only when asked for; an array item -/// keeps its index, so renders items[2].sku. -/// -public ref struct PropertyPath -{ - private readonly ReadOnlySpan _input; - private Segment[] _segments; - private byte[]? _scratch; - private int _scratchUsed; - - internal PropertyPath(int capacity, ReadOnlySpan input) - { - _input = input; - _segments = ArrayPool.Shared.Rent(Math.Max(capacity, 1)); - } - - /// - /// Current property path depth. - /// - private int Depth { get; set; } = -1; - - /// - /// Capacity of internal array. - /// - internal int MaxLength { get; private set; } - - internal readonly int CurrentDepth => Depth; - - /// - /// Number of levels in the path. - /// - public readonly int Length => Depth + 1; - - /// - /// Whether names are matched ignoring case in this call, see . - /// - public bool PropertyNameCaseInsensitive { readonly get; internal set; } = true; - - /// - /// The input ended inside a value: every rule must stop reading. - /// - internal bool Stopped { get; private set; } - - internal void Stop() => Stopped = true; - - /// - /// Adds the property name the reader stands on. - /// - internal void AddPropertyName(ref Utf8JsonReader reader) - { - ref var segment = ref Push(); - if (!reader.HasValueSequence && !reader.ValueIsEscaped && !_input.IsEmpty) - { - segment = new Segment(SegmentKind.Input, checked((int)reader.TokenStartIndex + 1), reader.ValueSpan.Length); - return; - } - - var maxLength = reader.HasValueSequence ? checked((int)reader.ValueSequence.Length) : reader.ValueSpan.Length; - var start = Reserve(maxLength); - var written = reader.CopyString(_scratch.AsSpan(start, maxLength)); - _scratchUsed = start + written; - segment = new Segment(SegmentKind.Scratch, start, written); - } - - /// - /// Adds an unescaped UTF-8 property name. - /// - internal void AddPropertyName(ReadOnlySpan utf8Name) - { - ref var segment = ref Push(); - var start = Reserve(utf8Name.Length); - utf8Name.CopyTo(_scratch.AsSpan(start)); - _scratchUsed = start + utf8Name.Length; - segment = new Segment(SegmentKind.Scratch, start, utf8Name.Length); - } - - /// - /// Adds the array item at . - /// - internal void AddArrayItem(int index) - { - ref var segment = ref Push(); - segment = new Segment(SegmentKind.ArrayItem, index, 0); - } - - internal void RemovePropertyName() - { - if (Depth <= -1) - { - return; - } - - ref var segment = ref _segments[Depth]; - if (segment.Kind == SegmentKind.Scratch) - { - _scratchUsed = segment.Start; - } - - segment = default; - Depth--; - } - - /// - /// Gets the unescaped UTF-8 name of a level without decoding it. - /// - /// Level, from 0 to - 1. - /// The name; valid only during the call. - /// false for an array item or an index out of range. - public readonly bool TryGetPropertyNameUtf8(int index, out ReadOnlySpan utf8Name) - { - if (index < 0 || index > Depth) - { - utf8Name = default; - return false; - } - - var segment = _segments[index]; - switch (segment.Kind) - { - case SegmentKind.Input: - utf8Name = _input.Slice(segment.Start, segment.Length); - return true; - case SegmentKind.Scratch: - utf8Name = _scratch.AsSpan(segment.Start, segment.Length); - return true; - default: - utf8Name = default; - return false; - } - } - - /// - /// Whether the level at is an array item. - /// - /// Level, from 0 to - 1. - public readonly bool IsArrayItem(int index) => index >= 0 && index <= Depth && _segments[index].Kind == SegmentKind.ArrayItem; - - /// - /// Gets the zero-based position of an array item level within its array. - /// - /// Level, from 0 to - 1. - /// Position of the item; -1 when the level is not an array item. - /// true when the level is an array item. - public readonly bool TryGetArrayIndex(int index, out int arrayIndex) - { - if (IsArrayItem(index)) - { - arrayIndex = _segments[index].Start; - return true; - } - - arrayIndex = -1; - return false; - } - - /// - /// UTF-8 name of the deepest level. - /// - internal readonly ReadOnlySpan CurrentUtf8 => TryGetPropertyNameUtf8(Depth, out var name) ? name : default; - - /// - /// Name of the level at , 0 being the root's property. - /// - /// Level, from 0 to - 1. - /// The decoded name; null for an array item or an index out of range. - public string? GetPropertyName(int index) - { - if (index < 0 || index > Depth) - { - return null; - } - - ref var segment = ref _segments[index]; - if (segment.Decoded is not null || segment.Kind == SegmentKind.ArrayItem) - { - return segment.Decoded; - } - - TryGetPropertyNameUtf8(index, out var utf8); - return segment.Decoded = Encoding.UTF8.GetString(utf8); - } - - /// - /// Name of a level counted from the deepest one. - /// - /// 0 for the value's own name, 1 for its parent, and so on. - /// The decoded name; null for an array item or an index out of range. - public string? GetPropertyNameReverse(int reversedIndex) => GetPropertyName(Depth - reversedIndex); - - /// - /// The path from the root down: names joined with dots and array items as [index], for example - /// items[2].sku; a name that is empty or holds ., [, ] or ' is written as ['name']. - /// - public override string ToString() - { - var text = new StringBuilder(); - for (var i = 0; i <= Depth; i++) - { - if (TryGetArrayIndex(i, out var arrayIndex)) - { - text.Append('[').Append(arrayIndex).Append(']'); - continue; - } - - AppendName(text, GetPropertyName(i)!, first: i == 0); - } - - return text.ToString(); - } - - internal static void AppendName(StringBuilder text, string name, bool first) - { - if (name.Length == 0 || name.AsSpan().IndexOfAny(".[]'") >= 0) - { - text.Append("['").Append(name.Replace("'", "\\'", StringComparison.Ordinal)).Append("']"); - return; - } - - if (!first) - { - text.Append('.'); - } - - text.Append(name); - } - - /// - /// Returns the pooled buffers. The path must not be used afterwards. - /// - internal void Dispose() - { - var segments = _segments; - _segments = []; - if (segments.Length > 0) - { - ArrayPool.Shared.Return(segments, clearArray: true); - } - - var scratch = _scratch; - _scratch = null; - if (scratch is not null) - { - ArrayPool.Shared.Return(scratch); - } - } - - private ref Segment Push() - { - Depth++; - MaxLength = Math.Max(MaxLength, Depth + 1); - - if (Depth >= _segments.Length) - { - var grown = ArrayPool.Shared.Rent(Math.Max(MaxLength, _segments.Length * 2)); - _segments.AsSpan().CopyTo(grown); - if (_segments.Length > 0) - { - ArrayPool.Shared.Return(_segments, clearArray: true); - } - - _segments = grown; - } - - return ref _segments[Depth]; - } - - private int Reserve(int length) - { - var start = _scratchUsed; - var required = start + length; - if (_scratch is null || required > _scratch.Length) - { - var grown = ArrayPool.Shared.Rent(Math.Max(required, 256)); - if (_scratch is not null) - { - _scratch.AsSpan(0, start).CopyTo(grown); - ArrayPool.Shared.Return(_scratch); - } - - _scratch = grown; - } - - return start; - } - - internal enum SegmentKind : byte - { - ArrayItem, - Input, - Scratch, - } - - internal struct Segment(SegmentKind kind, int start, int length) - { - public readonly SegmentKind Kind = kind; - public readonly int Start = start; - public readonly int Length = length; - public string? Decoded; - } -} diff --git a/DragoAnt.System.Text.Json.Observer/RelativeValuePolicy.cs b/DragoAnt.System.Text.Json.Observer/RelativeValuePolicy.cs index cfadd8e..ed0e70b 100644 --- a/DragoAnt.System.Text.Json.Observer/RelativeValuePolicy.cs +++ b/DragoAnt.System.Text.Json.Observer/RelativeValuePolicy.cs @@ -1,29 +1,29 @@ namespace DragoAnt.System.Text.Json.Observer; /// -/// A relative value policy; kept as the delegate target so that its whole-value rules can be offered containers too. +/// An any-depth value policy; kept as the delegate target so that its whole-value rules can be offered containers too. /// internal sealed class RelativeValuePolicy( - JsonObserverDelegate policy, + ObserveRule policy, JsonObserverItem[] items, - JsonObserverValueDelegate defaultValuePolicy) + JsonValuePolicy fallback) { public JsonObserverItem[] Items => items; - public JsonObserverValueDelegate DefaultValuePolicy => defaultValuePolicy; + public JsonValuePolicy Fallback => fallback; - public void Invoke(ref Utf8JsonReader reader, JsonWriter writer, TContext context, ref PropertyPath propPath) - => policy(ref reader, writer, context, 0, ref propPath, defaultValuePolicy); + public void Invoke(ref Utf8JsonReader reader, JsonWriter writer, TContext context, ref JsonWalk walk) + => policy(ref reader, writer, context, 0, ref walk, fallback.Rule); - public bool TryApplyContainer(ref Utf8JsonReader reader, JsonWriter writer, TContext context, ref PropertyPath propPath) + public bool TryApplyContainer(ref Utf8JsonReader reader, JsonWriter writer, TContext context, ref JsonWalk walk) { - var (match, depth) = JsonObserverItem.MatchPolicy(items, 0, ref propPath, reader.TokenType); + var (match, depth) = JsonObserverItem.MatchPolicy(items, 0, ref walk, reader.TokenType); if (match is null) { return false; } - match.Apply(ref reader, writer, context, depth, ref propPath, defaultValuePolicy); + match.Apply(ref reader, writer, context, depth, ref walk, fallback.Rule); return true; } } diff --git a/DragoAnt.System.Text.Json.Observer/RuleExplainer.cs b/DragoAnt.System.Text.Json.Observer/RuleExplainer.cs index 519266b..b892f25 100644 --- a/DragoAnt.System.Text.Json.Observer/RuleExplainer.cs +++ b/DragoAnt.System.Text.Json.Observer/RuleExplainer.cs @@ -1,3 +1,5 @@ +using DragoAnt.System.Text.Json.Observer.Builders; + namespace DragoAnt.System.Text.Json.Observer; /// @@ -6,7 +8,7 @@ namespace DragoAnt.System.Text.Json.Observer; /// internal sealed class RuleExplainer(RuleSet? obj, RuleSet? array) : PathExplainer { - protected override (JsonPathOutcome Outcome, string Rule, string Action) Explain( + protected override (PathOutcome Outcome, string Rule, string Action) Explain( IReadOnlyList segments, JsonTokenType valueKind, bool propertyNameCaseInsensitive, @@ -17,13 +19,13 @@ protected override (JsonPathOutcome Outcome, string Rule, string Action) Explain { var root = segments[0].IsIndex ? "array" : "object"; steps.Add($"$: the observer does not accept a root {root}"); - return (JsonPathOutcome.Invalid, "root", $"a root {root} makes the payload Invalid"); + return (PathOutcome.Invalid, "root", $"a root {root} makes the payload Invalid"); } var path = PathOf(segments, propertyNameCaseInsensitive); try { - var effective = set.ValuePolicy ?? JsonObserverValuePolicies.Default; + var effective = set.ValuePolicy ?? JsonValuePolicy.Default; var items = set.Items; var depth = 0; var chain = new List(); @@ -60,14 +62,14 @@ protected override (JsonPathOutcome Outcome, string Rule, string Action) Explain if (!last) { - if (effective.Target is RelativeValuePolicy relative) + if (effective.Relative is { } relative) { var (relativeItem, _) = JsonObserverItem.MatchPolicy(relative.Items, 0, ref path, token); if (relativeItem is not null) { var info = relativeItem.Info; - steps.Add($"{at}: relative {info.Match} → {info.Action} on the whole {Container(token)}"); - return (info.Outcome, $"relative {info.Match}", $"{info.Action} on the whole {Container(token)}"); + steps.Add($"{at}: AnyDepth {info.Match} → {info.Action} on the whole {Container(token)}"); + return (info.Outcome, $"AnyDepth {info.Match}", $"{info.Action} on the whole {Container(token)}"); } } @@ -86,56 +88,55 @@ protected override (JsonPathOutcome Outcome, string Rule, string Action) Explain } } - private static (JsonPathOutcome, string, string) DefaultPolicy( - JsonObserverValueDelegate policy, - ref PropertyPath path, + private static (PathOutcome, string, string) DefaultPolicy( + JsonValuePolicy policy, + ref JsonWalk path, JsonTokenType token, string at, List steps) { - if (policy.Target is RelativeValuePolicy relative) + if (policy.Relative is { } relative) { var reads = ReadsAt(relative.Items, 0, ref path, token); foreach (var read in reads) { - steps.Add($"{at}: relative {read.Match} → {read.Action}"); + steps.Add($"{at}: AnyDepth {read.Match} → {read.Action}"); } var (item, _) = JsonObserverItem.MatchPolicy(relative.Items, 0, ref path, token); if (item is not null) { var (relativeOutcome, relativeAction) = Resolve(item.Info, token); - steps.Add($"{at}: relative {item.Info.Match} → {relativeAction}"); - return WithReads((relativeOutcome, $"relative {item.Info.Match}", relativeAction), ["relative"], reads); + steps.Add($"{at}: AnyDepth {item.Info.Match} → {relativeAction}"); + return WithReads((relativeOutcome, $"AnyDepth {item.Info.Match}", relativeAction), ["AnyDepth"], reads); } - steps.Add($"{at}: no relative rule"); - return WithReads(DefaultPolicy(relative.DefaultValuePolicy, ref path, token, at, steps), ["relative"], reads); + steps.Add($"{at}: no AnyDepth rule"); + return WithReads(DefaultPolicy(relative.Fallback, ref path, token, at, steps), ["AnyDepth"], reads); } - var name = KnownPolicyName(policy); - var rule = name is null ? "custom default policy" : $"default policy {name}"; + var builtIn = policy.BuiltIn; + var rule = builtIn is null ? policy.Name : $"default policy {builtIn}"; var (outcome, action) = token is JsonTokenType.Null - ? (JsonPathOutcome.Unchanged, "keeps null") - : name switch + ? (PathOutcome.Unchanged, "keeps null") + : builtIn?.Kind switch { - nameof(JsonObserverValuePolicies.AllowList) => (JsonPathOutcome.Masked, "writes \"***\""), - nameof(JsonObserverValuePolicies.BlockList) => (JsonPathOutcome.Unchanged, "writes the value as is"), - nameof(JsonObserverValuePolicies.NullList) => (JsonPathOutcome.Masked, "writes null"), - "LegacyAllowList" when token is JsonTokenType.True or JsonTokenType.False => (JsonPathOutcome.Unchanged, "writes the boolean as is"), - "LegacyAllowList" => (JsonPathOutcome.Masked, token is JsonTokenType.String ? "writes \"#str#*****\"" : "writes \"#number#*****\""), - _ => (JsonPathOutcome.Custom, "custom default policy decides"), + ValuePolicyKind.AllowList => (PathOutcome.Masked, "writes \"***\""), + ValuePolicyKind.BlockList => (PathOutcome.Unchanged, "writes the value as is"), + ValuePolicyKind.NullList => (PathOutcome.Masked, "writes null"), + ValuePolicyKind.Tagged => (PathOutcome.Masked, RuleText.Tag(builtIn.Tag)), + _ => (PathOutcome.Custom, "custom default policy decides"), }; if (token is JsonTokenType.StartObject or JsonTokenType.StartArray) { - (outcome, action) = (JsonPathOutcome.Unchanged, $"the {Container(token)} is descended with the same rules"); + (outcome, action) = (PathOutcome.Unchanged, $"the {Container(token)} is descended with the same rules"); } steps.Add($"{at}: {rule} → {action}"); return (outcome, rule, action); } - private static List> ReadsAt(JsonObserverItem[] items, int depth, ref PropertyPath path, JsonTokenType token) + private static List> ReadsAt(JsonObserverItem[] items, int depth, ref JsonWalk path, JsonTokenType token) { List> reads = []; foreach (var item in items) @@ -152,8 +153,8 @@ private static List> ReadsAt(JsonObserverItem[] ite /// /// A value that read rules hand to the context is still written by the rule or policy that decided the result. /// - private static (JsonPathOutcome, string, string) WithReads( - (JsonPathOutcome Outcome, string Rule, string Action) written, + private static (PathOutcome, string, string) WithReads( + (PathOutcome Outcome, string Rule, string Action) written, List chain, List> reads) { @@ -164,21 +165,13 @@ private static (JsonPathOutcome, string, string) WithReads( var readRules = string.Join(" + ", reads.Select(r => string.Join(" > ", chain.Append(r.Match)))); var readActions = string.Join("; ", reads.Select(r => r.Action)); - var outcome = written.Outcome is JsonPathOutcome.Unchanged ? JsonPathOutcome.Read : written.Outcome; + var outcome = written.Outcome is PathOutcome.Unchanged ? PathOutcome.Read : written.Outcome; return (outcome, $"{readRules} + {written.Rule}", $"{readActions}; {written.Action}"); } - private static string? KnownPolicyName(JsonObserverValueDelegate policy) - { - var declaring = policy.Method.DeclaringType; - return declaring is { IsGenericType: true } && declaring.GetGenericTypeDefinition() == typeof(JsonObserverValuePolicies<>) - ? policy.Method.Name - : null; - } - - private static (JsonPathOutcome Outcome, string Action) Resolve(RuleInfo info, JsonTokenType token) => - token is JsonTokenType.Null && info.Action.StartsWith("MaskAny(", StringComparison.Ordinal) - ? (JsonPathOutcome.Unchanged, $"{info.Action} keeps null") + private static (PathOutcome Outcome, string Action) Resolve(RuleInfo info, JsonTokenType token) => + token is JsonTokenType.Null && info.KeepsNull + ? (PathOutcome.Unchanged, $"{info.Action} keeps null") : (info.Outcome, info.Action); private static string Container(JsonTokenType token) => token is JsonTokenType.StartArray ? "array" : "object"; diff --git a/DragoAnt.System.Text.Json.Observer/RuleSet.cs b/DragoAnt.System.Text.Json.Observer/RuleSet.cs index 90d6a05..d8bc8e3 100644 --- a/DragoAnt.System.Text.Json.Observer/RuleSet.cs +++ b/DragoAnt.System.Text.Json.Observer/RuleSet.cs @@ -3,16 +3,17 @@ namespace DragoAnt.System.Text.Json.Observer; /// /// The rules of one object or array, kept beside the delegate built from them so that a path can be explained. /// -internal sealed record RuleSet(bool IsArray, JsonObserverItem[] Items, JsonObserverValueDelegate? ValuePolicy); +internal sealed record RuleSet(bool IsArray, JsonObserverItem[] Items, JsonValuePolicy? ValuePolicy); /// -/// What a rule tests and what it does, for . +/// What a rule tests and what it does, for . /// -/// The test, for example Match("card", "number"). -/// The action, for example MaskAny("***"). +/// The test, for example Path("card", "number"). +/// The action, for example Mask("***"). /// What the action does to the value. /// Rules for the matched object or array; null for a value rule or a custom container rule. -internal sealed record RuleInfo(string Match, string Action, JsonPathOutcome Outcome, RuleSet? Child = null) +/// The action writes null for a null value without masking it. +internal sealed record RuleInfo(string Match, string Action, PathOutcome Outcome, RuleSet? Child = null, bool KeepsNull = false) { - public static RuleInfo Unknown { get; } = new("rule", "custom rule", JsonPathOutcome.Custom); + public static RuleInfo Unknown { get; } = new("rule", "custom rule", PathOutcome.Custom); } diff --git a/DragoAnt.System.Text.Json.Observer/ShapeWalker.cs b/DragoAnt.System.Text.Json.Observer/ShapeWalker.cs index 3e09a58..d6ccbb7 100644 --- a/DragoAnt.System.Text.Json.Observer/ShapeWalker.cs +++ b/DragoAnt.System.Text.Json.Observer/ShapeWalker.cs @@ -1,6 +1,5 @@ using System.Runtime.CompilerServices; using System.Text; -using DragoAnt.System.Text.Json.Observer.Strategies; using static System.Text.Json.JsonTokenType; namespace DragoAnt.System.Text.Json.Observer; @@ -14,31 +13,33 @@ internal sealed class ShapeWalker : PathExplainer private readonly JsonShape _unknown; private readonly bool _keepNulls; private readonly bool? _ignoreCase; + private readonly MaskTag _unknownTag; public ShapeWalker(JsonShape root, JsonShapeOptions options) { root.Freeze(); _root = root; _keepNulls = options.KeepNulls; - _ignoreCase = options.PropertyNameCaseInsensitive; + _ignoreCase = options.NameCaseInsensitive; + _unknownTag = options.UnknownTag; _unknown = options.Unknown switch { UnknownMemberPolicy.Descend => JsonShape.UnknownDescend, UnknownMemberPolicy.PassThrough => JsonShape.UnknownPassThrough, - _ => JsonShape.Opaque, + _ => JsonShape.UnknownMaskWhole, }; } public void Invoke( ref Utf8JsonReader reader, JsonWriter writer, - JsonObserveringEmptyContext context, + NoContext context, int depth, - ref PropertyPath propPath, - JsonObserverValueDelegate defaultValue) + ref JsonWalk propPath, + ValueRule defaultValue) => Write(ref reader, writer, ref propPath, _root); - private void Write(ref Utf8JsonReader reader, JsonWriter writer, ref PropertyPath propPath, JsonShape shape) + private void Write(ref Utf8JsonReader reader, JsonWriter writer, ref JsonWalk propPath, JsonShape shape) { var token = reader.TokenType; if (ReferenceEquals(shape, JsonShape.UnknownPassThrough)) @@ -53,6 +54,12 @@ private void Write(ref Utf8JsonReader reader, JsonWriter writer, ref PropertyPat return; } + if (ReferenceEquals(shape, JsonShape.UnknownMaskWhole)) + { + MaskWhole(ref reader, writer, ref propPath, _unknownTag); + return; + } + switch (shape.Kind) { case JsonShapeKind.Scalar when token is not (StartObject or StartArray): @@ -79,7 +86,7 @@ private void Write(ref Utf8JsonReader reader, JsonWriter writer, ref PropertyPat } } - private void WriteObject(ref Utf8JsonReader reader, JsonWriter writer, ref PropertyPath propPath, JsonShape? shape, JsonShape? values) + private void WriteObject(ref Utf8JsonReader reader, JsonWriter writer, ref JsonWalk propPath, JsonShape? shape, JsonShape? values) { RuntimeHelpers.EnsureSufficientExecutionStack(); writer.WriteStartObject(); @@ -99,7 +106,7 @@ private void WriteObject(ref Utf8JsonReader reader, JsonWriter writer, ref Prope case PropertyName: propPath.AddPropertyName(ref reader); var name = propPath.CurrentUtf8; - var child = values ?? shape!.Find(name, _ignoreCase ?? propPath.PropertyNameCaseInsensitive) ?? _unknown; + var child = values ?? shape!.Find(name, _ignoreCase ?? propPath.Options.NameCaseInsensitive) ?? _unknown; if (!reader.Read()) { propPath.RemovePropertyName(); @@ -115,7 +122,7 @@ private void WriteObject(ref Utf8JsonReader reader, JsonWriter writer, ref Prope } } - private void WriteArray(ref Utf8JsonReader reader, JsonWriter writer, ref PropertyPath propPath, JsonShape item) + private void WriteArray(ref Utf8JsonReader reader, JsonWriter writer, ref JsonWalk propPath, JsonShape item) { RuntimeHelpers.EnsureSufficientExecutionStack(); writer.WriteStartArray(); @@ -147,7 +154,7 @@ private void WriteArray(ref Utf8JsonReader reader, JsonWriter writer, ref Proper /// /// Writes an unknown value: containers are descended with the same treatment, scalars are masked or copied. /// - private void Copy(ref Utf8JsonReader reader, JsonWriter writer, ref PropertyPath propPath, JsonShape mode, bool maskScalars) + private void Copy(ref Utf8JsonReader reader, JsonWriter writer, ref JsonWalk propPath, JsonShape mode, bool maskScalars) { switch (reader.TokenType) { @@ -160,7 +167,7 @@ private void Copy(ref Utf8JsonReader reader, JsonWriter writer, ref PropertyPath default: if (maskScalars) { - MaskWhole(ref reader, writer, ref propPath, MaskTag.Full); + MaskWhole(ref reader, writer, ref propPath, _unknownTag); } else { @@ -171,7 +178,7 @@ private void Copy(ref Utf8JsonReader reader, JsonWriter writer, ref PropertyPath } } - private void MaskWhole(ref Utf8JsonReader reader, JsonWriter writer, ref PropertyPath propPath, MaskTag tag) + private void MaskWhole(ref Utf8JsonReader reader, JsonWriter writer, ref JsonWalk propPath, MaskTag tag) { if (reader.TokenType is Null && _keepNulls) { @@ -182,7 +189,7 @@ private void MaskWhole(ref Utf8JsonReader reader, JsonWriter writer, ref Propert TagMasking.Mask(ref reader, writer, tag, ref propPath); } - protected override (JsonPathOutcome Outcome, string Rule, string Action) Explain( + protected override (PathOutcome Outcome, string Rule, string Action) Explain( IReadOnlyList segments, JsonTokenType valueKind, bool propertyNameCaseInsensitive, @@ -237,24 +244,26 @@ protected override (JsonPathOutcome Outcome, string Rule, string Action) Explain return Final(current, valueKind, steps, Format(segments), unknownMember); } + private string UnknownAction => _unknownTag == MaskTag.Full ? "writes \"***\"" : $"MaskTag.{_unknownTag.Kind}"; + private string Unknown => ReferenceEquals(_unknown, JsonShape.UnknownDescend) ? "Descend" : ReferenceEquals(_unknown, JsonShape.UnknownPassThrough) ? "PassThrough" : "MaskWhole"; - private (JsonPathOutcome, string, string) Final(JsonShape shape, JsonTokenType token, List steps, string at, bool unknownMember) + private (PathOutcome, string, string) Final(JsonShape shape, JsonTokenType token, List steps, string at, bool unknownMember) { var isContainer = token is StartObject or StartArray; var (outcome, rule, action) = shape switch { - _ when ReferenceEquals(shape, JsonShape.UnknownPassThrough) => (JsonPathOutcome.Unchanged, "unknown member (PassThrough)", "writes the value as is"), - _ when ReferenceEquals(shape, JsonShape.UnknownDescend) && isContainer => (JsonPathOutcome.Unchanged, "unknown member (Descend)", "shows the names, masks every value inside"), - _ when ReferenceEquals(shape, JsonShape.UnknownDescend) => KeepNull(token, "unknown member (Descend)", "writes \"***\""), - { Kind: JsonShapeKind.Scalar } when !isContainer => (JsonPathOutcome.Unchanged, "shape Scalar", "writes the value as is"), + _ when ReferenceEquals(shape, JsonShape.UnknownPassThrough) => (PathOutcome.Unchanged, "unknown member (PassThrough)", "writes the value as is"), + _ when ReferenceEquals(shape, JsonShape.UnknownDescend) && isContainer => (PathOutcome.Unchanged, "unknown member (Descend)", "shows the names, masks every value inside"), + _ when ReferenceEquals(shape, JsonShape.UnknownDescend) => KeepNull(token, "unknown member (Descend)", UnknownAction), + { Kind: JsonShapeKind.Scalar } when !isContainer => (PathOutcome.Unchanged, "shape Scalar", "writes the value as is"), { Kind: JsonShapeKind.Masked } => KeepNull(token, $"shape Masked({shape.Tag.Kind})", $"MaskTag.{shape.Tag.Kind}"), - { Kind: JsonShapeKind.Object or JsonShapeKind.Map } when token is StartObject => (JsonPathOutcome.Unchanged, $"shape {shape.Kind}", "applies the shape to the members"), - { Kind: JsonShapeKind.Array } when token is StartArray => (JsonPathOutcome.Unchanged, "shape Array", "applies the item shape to every item"), - { Kind: JsonShapeKind.Object or JsonShapeKind.Map or JsonShapeKind.Array } when token is Null => (JsonPathOutcome.Unchanged, $"shape {shape.Kind}", "keeps null"), - _ when unknownMember => KeepNull(token, "unknown member (MaskWhole)", "writes \"***\" for the whole value"), + { Kind: JsonShapeKind.Object or JsonShapeKind.Map } when token is StartObject => (PathOutcome.Unchanged, $"shape {shape.Kind}", "applies the shape to the members"), + { Kind: JsonShapeKind.Array } when token is StartArray => (PathOutcome.Unchanged, "shape Array", "applies the item shape to every item"), + { Kind: JsonShapeKind.Object or JsonShapeKind.Map or JsonShapeKind.Array } when token is Null => (PathOutcome.Unchanged, $"shape {shape.Kind}", "keeps null"), + _ when unknownMember => KeepNull(token, "unknown member (MaskWhole)", $"{UnknownAction} for the whole value"), { Kind: JsonShapeKind.Opaque } => KeepNull(token, "shape Opaque", "writes \"***\" for the whole value"), _ => KeepNull(token, $"shape {shape.Kind} does not fit a {token} value", "writes \"***\" for the whole value"), }; @@ -263,13 +272,13 @@ _ when ReferenceEquals(shape, JsonShape.UnknownDescend) => KeepNull(token, "unkn return (outcome, rule, action); } - private (JsonPathOutcome, string, string) KeepNull(JsonTokenType token, string rule, string action) => - token is Null && _keepNulls ? (JsonPathOutcome.Unchanged, rule, "keeps null") : (JsonPathOutcome.Masked, rule, action); + private (PathOutcome, string, string) KeepNull(JsonTokenType token, string rule, string action) => + token is Null && _keepNulls ? (PathOutcome.Unchanged, rule, "keeps null") : (PathOutcome.Masked, rule, action); - private static (JsonPathOutcome, string, string) Masked(List steps, string at, string rule, string action) + private static (PathOutcome, string, string) Masked(List steps, string at, string rule, string action) { steps.Add($"{at}: {rule} → {action}"); - return (JsonPathOutcome.Masked, rule, action); + return (PathOutcome.Masked, rule, action); } private static void CopyScalar(ref Utf8JsonReader reader, JsonWriter writer) diff --git a/DragoAnt.System.Text.Json.Observer/Strategies/MaskTag.cs b/DragoAnt.System.Text.Json.Observer/Strategies/MaskTag.cs deleted file mode 100644 index e5337d3..0000000 --- a/DragoAnt.System.Text.Json.Observer/Strategies/MaskTag.cs +++ /dev/null @@ -1,103 +0,0 @@ -namespace DragoAnt.System.Text.Json.Observer.Strategies; - -/// -/// How a sensitive value is masked. -/// -public enum MaskKind -{ - /// - /// Replaced by "***". - /// - Full, - - /// - /// Only the last four characters are kept, as "***1234"; values shorter than eight characters are masked fully. - /// - Last4, - - /// - /// Replaced by a keyed hash, as "hash:0f3a…", so that equal values can be correlated without being shown. - /// - Hash, - - /// - /// Replaced by null. - /// - Omit, - - /// - /// Masked the way a custom decides from ; - /// the built-in strategy replaces it by "***". - /// - Custom, -} - -/// -/// Tag a rule passes to the , so that one strategy serves every kind of masking. -/// -/// -/// How the value is masked. A strategy that does not know falls back to it, so a tag such as -/// new MaskTag(MaskKind.Hash, classification) is still hashed by the built-in strategy. -/// -/// -/// Optional classification of the value, for example a data classification of a compliance taxonomy or a redactor -/// name, that a custom strategy maps to its own masking. It is compared with , -/// so prefer immutable keys with value equality. -/// -public readonly record struct MaskTag(MaskKind Kind, object? Key = null) -{ - /// - /// A tag only a custom strategy interprets, by ; the built-in strategy writes "***". - /// - /// Classification the strategy maps to its masking. - /// is null. - public static MaskTag Custom(object key) - { - ArgumentNullException.ThrowIfNull(key); - return new MaskTag(MaskKind.Custom, key); - } - - /// - /// Gets when it is a . - /// - /// The key; default when it is absent or of another type. - /// Expected key type. - /// true when the key is a . - public bool TryGetKey(out T? key) - { - if (Key is T typed) - { - key = typed; - return true; - } - - key = default; - return false; - } - - /// - /// . - /// - public static MaskTag Full => new(MaskKind.Full); - - /// - /// . - /// - public static MaskTag Last4 => new(MaskKind.Last4); - - /// - /// . - /// - public static MaskTag Hash => new(MaskKind.Hash); - - /// - /// . - /// - public static MaskTag Omit => new(MaskKind.Omit); - - /// - /// Lets a stand for its tag. - /// - /// How the value is masked. - public static implicit operator MaskTag(MaskKind kind) => new(kind); -} diff --git a/DragoAnt.System.Text.Json.Observer/Strategies/NameMatcher.cs b/DragoAnt.System.Text.Json.Observer/Strategies/NameMatcher.cs deleted file mode 100644 index bc945c7..0000000 --- a/DragoAnt.System.Text.Json.Observer/Strategies/NameMatcher.cs +++ /dev/null @@ -1,179 +0,0 @@ -using System.Text; -using System.Text.RegularExpressions; - -namespace DragoAnt.System.Text.Json.Observer.Strategies; - -/// -/// Property name test that works on the UTF-8 name and decodes it only when it has to. -/// -internal abstract class NameMatcher -{ - public static readonly NameMatcher Never = new FuncNameMatcher(_ => false); - - public abstract string Describe(); - - public bool MatchString(string? name) => MatchString(name, PropertyPathMatch.DefaultComparison); - - public abstract bool MatchString(string? name, StringComparison comparison); - - public virtual bool Match(ref PropertyPath path, int index) => MatchString(path.GetPropertyName(index), ComparisonOf(ref path)); - - private protected static StringComparison ComparisonOf(ref PropertyPath path) => - path.PropertyNameCaseInsensitive ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal; - - public static NameMatcher Exact(string pattern) => new TextNameMatcher(TextNameMatcher.Mode.Equals, pattern); - - public static NameMatcher StartsWith(string pattern) => new TextNameMatcher(TextNameMatcher.Mode.StartsWith, pattern); - - public static NameMatcher EndsWith(string pattern) => new TextNameMatcher(TextNameMatcher.Mode.EndsWith, pattern); - - public static NameMatcher Contains(string pattern) => new TextNameMatcher(TextNameMatcher.Mode.Contains, pattern); - - public static NameMatcher OneOf(string[] names) => new OneOfNameMatcher(names); - - internal sealed class FuncNameMatcher(Func match, string? description = null) : NameMatcher - { - public FuncNameMatcher(Func match, string? description = null) - : this((name, _) => match(name), description) - { - } - - public override string Describe() => description ?? "custom name test"; - - public override bool MatchString(string? name, StringComparison comparison) => match(name, comparison); - } - - /// - /// A regular expression; a case-insensitive call also matches names that differ in case only. - /// - internal sealed class RegexNameMatcher(Regex regex) : NameMatcher - { - private readonly Regex _ignoreCase = (regex.Options & RegexOptions.IgnoreCase) != 0 - ? regex - : new Regex(regex.ToString(), regex.Options | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant, regex.MatchTimeout); - - public override string Describe() => $"Regex(/{regex}/)"; - - public override bool MatchString(string? name, StringComparison comparison) => - name is not null && (comparison == StringComparison.Ordinal ? regex : _ignoreCase).IsMatch(name); - } - - /// - /// Pattern compared byte by byte when both it and the name are ASCII; any other name falls back to the string comparison. - /// - private sealed class TextNameMatcher(TextNameMatcher.Mode mode, string pattern) : NameMatcher - { - private readonly byte[]? _ascii = Ascii.IsValid(pattern) ? Encoding.ASCII.GetBytes(pattern) : null; - - public override string Describe() => mode switch - { - Mode.Equals => $"\"{pattern}\"", - _ => $"{mode}(\"{pattern}\")", - }; - - public override bool MatchString(string? name, StringComparison comparison) => name is not null && mode switch - { - Mode.Equals => string.Equals(name, pattern, comparison), - Mode.StartsWith => name.StartsWith(pattern, comparison), - Mode.EndsWith => name.EndsWith(pattern, comparison), - _ => name.Contains(pattern, comparison), - }; - - public override bool Match(ref PropertyPath path, int index) - { - if (!path.TryGetPropertyNameUtf8(index, out var name)) - { - return false; - } - - if (_ascii is null || !Ascii.IsValid(name)) - { - return MatchString(path.GetPropertyName(index), ComparisonOf(ref path)); - } - - ReadOnlySpan utf8 = _ascii; - var ignoreCase = path.PropertyNameCaseInsensitive; - return mode switch - { - Mode.Equals => name.Length == utf8.Length && SameText(name, utf8, ignoreCase), - Mode.StartsWith => name.Length >= utf8.Length && SameText(name[..utf8.Length], utf8, ignoreCase), - Mode.EndsWith => name.Length >= utf8.Length && SameText(name[^utf8.Length..], utf8, ignoreCase), - _ => ContainsText(name, utf8, ignoreCase), - }; - } - - private static bool ContainsText(ReadOnlySpan name, ReadOnlySpan value, bool ignoreCase) - { - if (!ignoreCase) - { - return name.IndexOf(value) >= 0; - } - - for (var i = 0; i + value.Length <= name.Length; i++) - { - if (Ascii.EqualsIgnoreCase(name.Slice(i, value.Length), value)) - { - return true; - } - } - - return false; - } - - internal enum Mode : byte - { - Equals, - StartsWith, - EndsWith, - Contains, - } - } - - private static bool SameText(ReadOnlySpan left, ReadOnlySpan right, bool ignoreCase) => - ignoreCase ? Ascii.EqualsIgnoreCase(left, right) : left.SequenceEqual(right); - - private sealed class OneOfNameMatcher : NameMatcher - { - private readonly string[] _names; - private readonly HashSet _ignoreCase; - private readonly HashSet _exact; - private readonly byte[][]? _asciiNames; - - public OneOfNameMatcher(string[] names) - { - _names = names; - _ignoreCase = new HashSet(names, StringComparer.OrdinalIgnoreCase); - _exact = new HashSet(names, StringComparer.Ordinal); - _asciiNames = names.All(n => Ascii.IsValid(n)) ? names.Select(n => Encoding.ASCII.GetBytes(n)).ToArray() : null; - } - - public override string Describe() => $"OneOf({string.Join(", ", _names.Select(n => $"\"{n}\""))})"; - - public override bool MatchString(string? name, StringComparison comparison) => - name is not null && (comparison == StringComparison.Ordinal ? _exact : _ignoreCase).Contains(name); - - public override bool Match(ref PropertyPath path, int index) - { - if (!path.TryGetPropertyNameUtf8(index, out var name)) - { - return false; - } - - if (_asciiNames is null || !Ascii.IsValid(name)) - { - return MatchString(path.GetPropertyName(index), ComparisonOf(ref path)); - } - - var ignoreCase = path.PropertyNameCaseInsensitive; - foreach (var candidate in _asciiNames) - { - if (candidate.Length == name.Length && SameText(name, candidate, ignoreCase)) - { - return true; - } - } - - return false; - } - } -} diff --git a/DragoAnt.System.Text.Json.Observer/Strategies/PropMatches.cs b/DragoAnt.System.Text.Json.Observer/Strategies/PropMatches.cs deleted file mode 100644 index ea5b4fb..0000000 --- a/DragoAnt.System.Text.Json.Observer/Strategies/PropMatches.cs +++ /dev/null @@ -1,41 +0,0 @@ -using System.Text.RegularExpressions; - -namespace DragoAnt.System.Text.Json.Observer.Strategies; - -/// -/// Property matching strategies. -/// -public static class PropMatches -{ - /// - /// Matches property name by start value. - /// - /// Property name start value. - public static PropMatchingStrategy StartsWith(string value) => new(NameMatcher.StartsWith(value)); - - /// - /// Matches property name by ending. - /// - /// Property name ending value. - public static PropMatchingStrategy EndsWith(string value) => new(NameMatcher.EndsWith(value)); - - /// - /// Matches property name by containing value. - /// - /// Property name containing value. - public static PropMatchingStrategy Contains(string value) => new(NameMatcher.Contains(value)); - - /// - /// Matches property name by regular expression. Like the other tests it follows - /// : by default a name that differs in case only also matches. - /// A regular expression built with ignores case under either option. - /// - /// Property name regular expression. - public static PropMatchingStrategy Regex(Regex regex) => new(new NameMatcher.RegexNameMatcher(regex)); - - /// - /// Matches property by full name equality. - /// - /// Property names. - public static PropMatchingStrategy OneOf(params string[] propNames) => new(NameMatcher.OneOf(propNames)); -} diff --git a/DragoAnt.System.Text.Json.Observer/Strategies/PropMatchingStrategy.cs b/DragoAnt.System.Text.Json.Observer/Strategies/PropMatchingStrategy.cs deleted file mode 100644 index c6d7812..0000000 --- a/DragoAnt.System.Text.Json.Observer/Strategies/PropMatchingStrategy.cs +++ /dev/null @@ -1,55 +0,0 @@ -namespace DragoAnt.System.Text.Json.Observer.Strategies; - -/// -/// A property name test. A string converts to an exact, case-insensitive match; has the others. -/// -public readonly struct PropMatchingStrategy -{ - private readonly NameMatcher? _matcher; - - /// - /// Matches property names with a custom test; the name is decoded to a for it. The test decides - /// case on its own; the overload that takes a follows . - /// - /// Name test; receives null for an array item. - public PropMatchingStrategy(Func strategy) - { - _matcher = new NameMatcher.FuncNameMatcher(strategy); - } - - /// - /// Matches property names with a custom test that honours ; - /// the name is decoded to a for it. - /// - /// - /// Name test; receives null for an array item, and or - /// as the call's case option. - /// - public PropMatchingStrategy(Func strategy) - { - _matcher = new NameMatcher.FuncNameMatcher(strategy); - } - - internal PropMatchingStrategy(NameMatcher matcher) - { - _matcher = matcher; - } - - internal NameMatcher Matcher => _matcher ?? NameMatcher.Never; - - /// - /// The name test as a function of the decoded name. - /// - /// Name test. - public static implicit operator Func(PropMatchingStrategy strategy) => strategy.Matcher.MatchString; - /// - /// A custom name test; the name is decoded to a for it. - /// - /// Name test; receives null for an array item. - public static implicit operator PropMatchingStrategy(Func strategy) => new(strategy); - /// - /// Matches the exact name, case-insensitively. - /// - /// Property name. - public static implicit operator PropMatchingStrategy(string pattern) => new(NameMatcher.Exact(pattern)); -} diff --git a/DragoAnt.System.Text.Json.Observer/Strategies/Utf8MaskContext.cs b/DragoAnt.System.Text.Json.Observer/Strategies/Utf8MaskContext.cs deleted file mode 100644 index 0c7224e..0000000 --- a/DragoAnt.System.Text.Json.Observer/Strategies/Utf8MaskContext.cs +++ /dev/null @@ -1,54 +0,0 @@ -namespace DragoAnt.System.Text.Json.Observer.Strategies; - -/// -/// What a knows about the value it masks. Valid only during the call it is passed to; -/// reading it allocates nothing. -/// -public readonly ref struct Utf8MaskContext -{ - private readonly PropertyPath _path; - - internal Utf8MaskContext(ReadOnlySpan value, JsonTokenType tokenType, MaskTag tag, JsonObserverOptions options, PropertyPath path) - { - Value = value; - TokenType = tokenType; - Tag = tag; - Options = options; - _path = path; - } - - /// - /// The unescaped text of a string, the literal of a number or boolean, or empty for null, an object or an array. - /// - public ReadOnlySpan Value { get; } - - /// - /// JSON type of the value; or for a container. - /// - public JsonTokenType TokenType { get; } - - /// - /// How the rule asks for the value to be masked. - /// - public MaskTag Tag { get; } - - /// - /// Options of the current call. - /// - public JsonObserverOptions Options { get; } - - /// - /// Path of the value from the root, array indices included. - /// - public PropertyPath Path => _path; - - /// - /// Unescaped UTF-8 name of the property that holds the value; empty for an array item. - /// - public ReadOnlySpan PropertyName => _path.TryGetPropertyNameUtf8(_path.Length - 1, out var name) ? name : default; - - /// - /// The value is an item of an array rather than the value of a property. - /// - public bool IsArrayItem => _path.IsArrayItem(_path.Length - 1); -} diff --git a/DragoAnt.System.Text.Json.Observer/Strategies/Utf8MaskStrategy.cs b/DragoAnt.System.Text.Json.Observer/Strategies/Utf8MaskStrategy.cs deleted file mode 100644 index 5694f19..0000000 --- a/DragoAnt.System.Text.Json.Observer/Strategies/Utf8MaskStrategy.cs +++ /dev/null @@ -1,123 +0,0 @@ -using System.Security.Cryptography; - -namespace DragoAnt.System.Text.Json.Observer.Strategies; - -/// -/// Masks a sensitive value given as UTF-8. One instance serves every rule: the rule's says how. -/// -/// -/// Override to also see the property name and path of the value, -/// or the shorter overload when the value and the tag are enough. A strategy that overrides neither masks like -/// . -/// -public abstract class Utf8MaskStrategy -{ - /// - /// Built-in strategy: , , - /// (HMAC-SHA256 with ), ; anything else, - /// included, becomes "***". - /// - public static Utf8MaskStrategy Default { get; } = new DefaultUtf8MaskStrategy(); - - /// - /// Writes the masked replacement of one value, knowing where it is. This is the method the observer calls; by - /// default it forwards to . - /// - /// The value, its JSON type, the rule's tag, the call's options and the value's path. - /// Receives exactly one value. - public virtual void Mask(in Utf8MaskContext context, JsonWriter writer) => - Mask(context.Value, context.TokenType, context.Tag, writer, context.Options); - - /// - /// Writes the masked replacement of one value. By default it masks like . - /// - /// - /// The unescaped text of a string, the literal of a number or boolean, or empty for an object or array. - /// - /// JSON type of the value; or for a container. - /// How the rule asks for the value to be masked. - /// Receives exactly one value. - /// Options of the current call. - public virtual void Mask(ReadOnlySpan value, JsonTokenType tokenType, MaskTag tag, JsonWriter writer, JsonObserverOptions options) => - Default.Mask(value, tokenType, tag, writer, options); - - private sealed class DefaultUtf8MaskStrategy : Utf8MaskStrategy - { - private const int Last4MinLength = 8; - private const int HashHexLength = 16; - private static readonly byte[] ProcessKey = RandomNumberGenerator.GetBytes(32); - - private static ReadOnlySpan Stars => "***"u8; - private static ReadOnlySpan HashPrefix => "hash:"u8; - private static ReadOnlySpan Hex => "0123456789abcdef"u8; - - public override void Mask(ReadOnlySpan value, JsonTokenType tokenType, MaskTag tag, JsonWriter writer, JsonObserverOptions options) - { - var scalar = tokenType is JsonTokenType.String or JsonTokenType.Number; - switch (tag.Kind) - { - case MaskKind.Omit: - writer.WriteNullValue(); - break; - case MaskKind.Last4 when scalar: - WriteLast4(value, writer); - break; - case MaskKind.Hash when scalar: - WriteHash(value, options.HashKey.IsEmpty ? ProcessKey : options.HashKey.Span, writer); - break; - default: - writer.WriteStringValue(Stars); - break; - } - } - - private static void WriteLast4(ReadOnlySpan value, JsonWriter writer) - { - var start = value.Length; - var chars = 0; - var total = 0; - for (var i = value.Length - 1; i >= 0; i--) - { - if ((value[i] & 0xC0) == 0x80) - { - continue; - } - - total++; - if (chars < 4) - { - chars++; - start = i; - } - } - - if (total < Last4MinLength) - { - writer.WriteStringValue(Stars); - return; - } - - var tail = value[start..]; - Span masked = stackalloc byte[Stars.Length + tail.Length]; - Stars.CopyTo(masked); - tail.CopyTo(masked[Stars.Length..]); - writer.WriteStringValue(masked); - } - - private static void WriteHash(ReadOnlySpan value, ReadOnlySpan key, JsonWriter writer) - { - Span hash = stackalloc byte[32]; - HMACSHA256.HashData(key, value, hash); - - Span text = stackalloc byte[HashPrefix.Length + HashHexLength]; - HashPrefix.CopyTo(text); - for (var i = 0; i < HashHexLength / 2; i++) - { - text[HashPrefix.Length + 2 * i] = Hex[hash[i] >> 4]; - text[HashPrefix.Length + 2 * i + 1] = Hex[hash[i] & 0xF]; - } - - writer.WriteStringValue(text); - } - } -} diff --git a/DragoAnt.System.Text.Json.Observer/Strategies/StringMaskingStrategy.cs b/DragoAnt.System.Text.Json.Observer/StringMaskingStrategy.cs similarity index 91% rename from DragoAnt.System.Text.Json.Observer/Strategies/StringMaskingStrategy.cs rename to DragoAnt.System.Text.Json.Observer/StringMaskingStrategy.cs index a8478af..2def5b2 100644 --- a/DragoAnt.System.Text.Json.Observer/Strategies/StringMaskingStrategy.cs +++ b/DragoAnt.System.Text.Json.Observer/StringMaskingStrategy.cs @@ -1,6 +1,6 @@ using System.Text.RegularExpressions; -namespace DragoAnt.System.Text.Json.Observer.Strategies; +namespace DragoAnt.System.Text.Json.Observer; /// /// Replacement of a masked value: a constant string, a regular expression whose matches become *, or a function. @@ -24,6 +24,12 @@ private StringMaskingStrategy(string constant) private Func Strategy { get; } + internal static StringMaskingStrategy From(Func strategy) + { + ArgumentNullException.ThrowIfNull(strategy); + return new StringMaskingStrategy((value, _) => strategy(value)); + } + internal string? Constant { get; } /// diff --git a/DragoAnt.System.Text.Json.Observer/TagMasking.cs b/DragoAnt.System.Text.Json.Observer/TagMasking.cs index e8fd816..44ad023 100644 --- a/DragoAnt.System.Text.Json.Observer/TagMasking.cs +++ b/DragoAnt.System.Text.Json.Observer/TagMasking.cs @@ -1,5 +1,5 @@ using System.Buffers; -using DragoAnt.System.Text.Json.Observer.Strategies; +using static System.Text.Json.JsonTokenType; namespace DragoAnt.System.Text.Json.Observer; @@ -8,39 +8,60 @@ internal static class TagMasking private const int StackallocThreshold = 256; /// - /// Writes the strategy's replacement for the current value and moves past it; a container is never read. + /// Writes the call's strategy's replacement for the current value and moves past it; a container is not read + /// unless the strategy keeps it. /// - public static void Mask(ref Utf8JsonReader reader, JsonWriter writer, MaskTag tag, ref PropertyPath propPath) + public static void Mask(ref Utf8JsonReader reader, JsonWriter writer, MaskTag tag, ref JsonWalk walk) => + Mask(ref reader, writer, tag, walk.Options.Strategy ?? ValueMaskStrategy.Default, ref walk); + + /// + /// Writes 's replacement for the current value and moves past it. + /// + public static void Mask(ref Utf8JsonReader reader, JsonWriter writer, MaskTag tag, ValueMaskStrategy strategy, ref JsonWalk walk) { + var output = JsonMaskValueWriter.Rent(writer); + bool keep; + bool failed; writer.MaskOutput = true; try { - MaskValue(ref reader, writer, tag, ref propPath); + MaskValue(ref reader, output, tag, strategy, ref walk); } finally { writer.MaskOutput = false; + keep = output.KeepRequested; + failed = output.Failed; + output.Return(); + } + + if (failed) + { + throw new JsonObserverException("The strategy wrote an invalid number."); + } + + if (keep) + { + JsonCopy.CopyValue(ref reader, writer, ref walk); + } + else if (reader.TokenType is StartObject or StartArray && !reader.TrySkip()) + { + walk.Stop(); } } - private static void MaskValue(ref Utf8JsonReader reader, JsonWriter writer, MaskTag tag, ref PropertyPath propPath) + private static void MaskValue(ref Utf8JsonReader reader, JsonMaskValueWriter output, MaskTag tag, ValueMaskStrategy strategy, ref JsonWalk walk) { - var options = writer.Options; - var strategy = options.MaskStrategy ?? Utf8MaskStrategy.Default; + var options = walk.Options; var tokenType = reader.TokenType; + var kind = KindOf(tokenType); + var index = walk.ValueIndex++; switch (tokenType) { - case JsonTokenType.StartObject: - case JsonTokenType.StartArray: - strategy.Mask(new Utf8MaskContext(default, tokenType, tag, options, propPath), writer); - if (!reader.TrySkip()) - { - propPath.Stop(); - } - - return; - case JsonTokenType.Null: - strategy.Mask(new Utf8MaskContext(default, tokenType, tag, options, propPath), writer); + case StartObject: + case StartArray: + case Null: + strategy.Mask(new MaskContext(default, kind, tag, options, in walk.Path, index), output); return; case JsonTokenType.String when reader.HasValueSequence || reader.ValueIsEscaped: { @@ -49,7 +70,7 @@ private static void MaskValue(ref Utf8JsonReader reader, JsonWriter writer, Mask try { var written = reader.CopyString(buffer); - strategy.Mask(new Utf8MaskContext(buffer.AsSpan(0, written), tokenType, tag, options, propPath), writer); + strategy.Mask(new MaskContext(buffer.AsSpan(0, written), kind, tag, options, in walk.Path, index), output); } finally { @@ -66,7 +87,7 @@ private static void MaskValue(ref Utf8JsonReader reader, JsonWriter writer, Mask try { reader.ValueSequence.CopyTo(buffer); - strategy.Mask(new Utf8MaskContext(buffer[..length], tokenType, tag, options, propPath), writer); + strategy.Mask(new MaskContext(buffer[..length], kind, tag, options, in walk.Path, index), output); } finally { @@ -79,8 +100,142 @@ private static void MaskValue(ref Utf8JsonReader reader, JsonWriter writer, Mask return; } default: - strategy.Mask(new Utf8MaskContext(reader.ValueSpan, tokenType, tag, options, propPath), writer); + strategy.Mask(new MaskContext(reader.ValueSpan, kind, tag, options, in walk.Path, index), output); + return; + } + } + + public static ValueKind KindOf(JsonTokenType tokenType) => tokenType switch + { + JsonTokenType.String => ValueKind.String, + Number => ValueKind.Number, + True or False => ValueKind.Boolean, + StartObject => ValueKind.Object, + StartArray => ValueKind.Array, + _ => ValueKind.Null, + }; +} + +/// +/// The JSON : writes a strategy's replacement through a . +/// One instance per thread, reused across calls. +/// +internal sealed class JsonMaskValueWriter : MaskValueWriter +{ + [ThreadStatic] + private static JsonMaskValueWriter? t_cached; + + private JsonWriter _writer = JsonWriter.Empty; + + public bool KeepRequested { get; private set; } + + public bool Failed { get; private set; } + + public static JsonMaskValueWriter Rent(JsonWriter writer) + { + var output = t_cached ?? new JsonMaskValueWriter(); + t_cached = null; + output._writer = writer; + output.KeepRequested = false; + output.Failed = false; + return output; + } + + public void Return() + { + _writer = JsonWriter.Empty; + t_cached = this; + } + + public override void String(ReadOnlySpan utf8) => _writer.WriteStringValue(utf8); + + public override void String(ReadOnlySpan chars) => _writer.WriteStringValue(chars); + + public override void Boolean(bool value) => _writer.WriteBooleanValue(value); + + public override void Null() => _writer.WriteNullValue(); + + public override void Keep() => KeepRequested = true; + + protected override void WriteNumber(ReadOnlySpan utf8Literal) => _writer.WriteRawValue(utf8Literal); + + protected override void InvalidNumber() => Failed = true; +} + +/// +/// Copies a value unchanged, an object or array with everything in it. +/// +internal static class JsonCopy +{ + public static void CopyValue(ref Utf8JsonReader reader, JsonWriter writer, ref JsonWalk walk) + { + switch (reader.TokenType) + { + case StartObject: + case StartArray: + CopyContainer(ref reader, writer, ref walk); + return; + default: + BuiltInPolicies.BlockList(ref reader, writer, null, ref walk); + return; + } + } + + private static void CopyContainer(ref Utf8JsonReader reader, JsonWriter writer, ref JsonWalk walk) + { + var depth = reader.CurrentDepth; + if (reader.TokenType is StartObject) + { + writer.WriteStartObject(); + } + else + { + writer.WriteStartArray(); + } + + while (true) + { + if (writer.Stopped || !reader.Read()) + { + walk.Stop(); return; + } + + switch (reader.TokenType) + { + case EndObject: + writer.WriteEndObject(); + if (reader.CurrentDepth == depth) + { + return; + } + + break; + case EndArray: + writer.WriteEndArray(); + if (reader.CurrentDepth == depth) + { + return; + } + + break; + case StartObject: + writer.WriteStartObject(); + break; + case StartArray: + writer.WriteStartArray(); + break; + case PropertyName: + walk.AddPropertyName(ref reader); + writer.WritePropertyName(walk.CurrentUtf8); + walk.RemovePropertyName(); + break; + case Comment: + break; + default: + BuiltInPolicies.BlockList(ref reader, writer, null, ref walk); + break; + } } } } diff --git a/DragoAnt.System.Text.Json.Observer/buildTransitive/DragoAnt.System.Text.Json.Observer.props b/DragoAnt.System.Text.Json.Observer/buildTransitive/DragoAnt.System.Text.Json.Observer.props new file mode 100644 index 0000000..2579b33 --- /dev/null +++ b/DragoAnt.System.Text.Json.Observer/buildTransitive/DragoAnt.System.Text.Json.Observer.props @@ -0,0 +1,7 @@ + + + + + + diff --git a/DragoAnt.System.Text.Json.Observer/package.readme.md b/DragoAnt.System.Text.Json.Observer/package.readme.md index d4edf1a..29222bd 100644 --- a/DragoAnt.System.Text.Json.Observer/package.readme.md +++ b/DragoAnt.System.Text.Json.Observer/package.readme.md @@ -2,18 +2,18 @@ Mask or extract JSON values by property-path rules in a single streaming pass from `Utf8JsonReader` to `Utf8JsonWriter` — no deserialization, no DOM. Built for logging: it never throws, even on cut-off or invalid JSON, and never writes a masked value in clear. -Targets `net8.0`, `net9.0` and `net10.0`, with no dependencies beyond the .NET base class library. +Targets `net8.0` and `net10.0`; depends only on [DragoAnt.Observer.Core](https://www.nuget.org/packages/DragoAnt.Observer.Core), whose `DragoAnt.Observer` namespace (`MaskTag`, `MaskResult`, `ValuePolicy`, …) the package imports into every C# file (`false` opts out). ## Quick start ```csharp using DragoAnt.System.Text.Json.Observer; -using DragoAnt.System.Text.Json.Observer.Strategies; -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; -var masker = JsonObserver.Obj(Relative(rules => rules - .Match("password").MaskAny("***") - .Match("card", "number").MaskAny(MaskTag.Last4), +var masker = JsonObserver.Obj(AnyDepth(rules => rules + .Match("password").Mask("***") + .Path("card", "number").Mask(MaskTag.Last4), BlockList)); Console.WriteLine(masker.Mask("""{"user":"alice","password":"s3cret","card":{"number":"4111111111111111"}}""")); @@ -21,7 +21,7 @@ Console.WriteLine(masker.Mask("""{"user":"alice","password":"s3cret","card":{"nu // {"user":"alice","password":"***","card":{"number":"***1111"}} ``` -Build an observer once and share it: it is thread-safe. `Relative` rules match the end of a property's path at any depth; rules passed straight to `JsonObserver.Obj(root => …)` follow the path from the root. Values no rule names get the default policy: `AllowList` (the default) writes every string, number and boolean as `"***"`, `BlockList` writes them unchanged, `NullList` as `null`. +Build an observer once and share it: it is thread-safe. `AnyDepth` rules match the end of a property's path at any depth; rules passed straight to `JsonObserver.Obj(root => …)` follow the path from the root. Values no rule names get the default policy: `AllowList` (the default) writes every string, number and boolean as `"***"`, `BlockList` writes them unchanged, `NullList` as `null`. Every `Mask*` rule masks the whole value whatever its JSON type; an object or array under a mask rule is skipped unread. @@ -33,8 +33,8 @@ using DragoAnt.System.Text.Json.Observer; var observer = JsonObserver.Obj( rules => rules .Match("id").ReadInt((id, order) => order.Id = id) - .Match("card").MaskAny("***"), - JsonObserverValuePolicies.BlockList); + .Match("card").Mask("***"), + ValuePolicy.BlockList); var order = new Order(); Console.WriteLine(observer.Mask("""{"id":42,"card":"4111111111111111"}""", order)); @@ -51,15 +51,16 @@ sealed class Order ## Cut-off JSON and the UTF-8 API -`MaskResult.Status` is `Masked`, `Truncated` (the payload ended early, hit `MaxOutputBytes`, or a string was cut to `MaxValueBytes`), `Invalid` or `NotJson`. Whatever the status, the output is valid JSON holding only masked values. The string and the UTF-8 API produce the same output. +`MaskResult.Status` is `Masked`, `Truncated` (the payload ended early, hit `MaxOutputBytes`, or a string was cut to `MaxValueBytes`), `Invalid` or `Unrecognized`. Whatever the status, the output is valid JSON holding only masked values. The string and the UTF-8 API produce the same output. ```csharp using System.Buffers; using System.Text; using DragoAnt.System.Text.Json.Observer; -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; -var masker = JsonObserver.Obj(Relative(rules => rules.Match("password").MaskAny("***"), BlockList)); +var masker = JsonObserver.Obj(AnyDepth(rules => rules.Match("password").Mask("***"), BlockList)); var output = new ArrayBufferWriter(1024); MaskResult result = masker.Mask("""{"user":"alice","password":"secret","roles":["admin","de"""u8, output); diff --git a/README.md b/README.md index 17910be..a911e13 100644 --- a/README.md +++ b/README.md @@ -6,24 +6,24 @@ Mask or extract JSON values by property-path rules in a single streaming pass fr [![NuGet](https://img.shields.io/nuget/v/DragoAnt.System.Text.Json.Observer)](https://www.nuget.org/packages/DragoAnt.System.Text.Json.Observer) [![Downloads](https://img.shields.io/nuget/dt/DragoAnt.System.Text.Json.Observer)](https://www.nuget.org/packages/DragoAnt.System.Text.Json.Observer) [![License](https://img.shields.io/github/license/DragoAnt/Extensions.System.Text.Json)](https://github.com/DragoAnt/Extensions.System.Text.Json/blob/main/LICENSE) -![.NET](https://img.shields.io/badge/.NET-8.0%20%7C%209.0%20%7C%2010.0-512BD4) +![.NET](https://img.shields.io/badge/.NET-8.0%20%7C%2010.0-512BD4) ## In short - **What:** you describe which properties are sensitive (`password`, `card.number`, anything ending in `token`); the observer rewrites the JSON with those values masked, and can hand selected values to a context object on the way. It reads and writes UTF-8 tokens directly, so a body is never turned into objects or a DOM. - **Built for logging:** it never throws, even on cut-off or invalid JSON, and never writes a masked value in clear — a cut-off body gives its masked prefix, closed into valid JSON. -- **Install:** `dotnet add package DragoAnt.System.Text.Json.Observer` (`net8.0`, `net9.0`, `net10.0`; no dependencies beyond the base class library). +- **Install:** `dotnet add package DragoAnt.System.Text.Json.Observer` (`net8.0`, `net10.0`; it brings [DragoAnt.Observer.Core](https://www.nuget.org/packages/DragoAnt.Observer.Core), the format-neutral types, and imports the `DragoAnt.Observer` namespace into every C# file — set `false` to opt out). Builds with any .NET 8+ SDK. ### Quick start ```csharp using DragoAnt.System.Text.Json.Observer; -using DragoAnt.System.Text.Json.Observer.Strategies; -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; -var masker = JsonObserver.Obj(Relative(rules => rules - .Match("password").MaskAny("***") - .Match("card", "number").MaskAny(MaskTag.Last4), +var masker = JsonObserver.Obj(AnyDepth(rules => rules + .Match("password").Mask("***") + .Path("card", "number").Mask(MaskTag.Last4), BlockList)); Console.WriteLine(masker.Mask("""{"user":"alice","password":"s3cret","card":{"number":"4111111111111111"}}""")); @@ -35,7 +35,7 @@ Build an observer once (a `static readonly` field) and share it: it is thread-sa ### Mask or extract -**Mask** rules (`MaskAny`, `MaskStr`, …) replace a value. **Read** rules (`ReadStr`, `ReadInt`, …) hand a value to a context object; they do not decide what is written, so the default policy writes the value unless `.Unmasked()` or a mask method is chained on the read (`Match("ssn").ReadStr(f).MaskAny(MaskTag.Last4)`). One observer can do both in the same pass: +**Mask** rules (`Mask`, `MaskInt`, …) replace a value. **Read** rules (`ReadStr`, `ReadInt`, …) hand a value to a context object; they do not decide what is written, so the default policy writes the value unless `.Unmasked()` or a mask method is chained on the read (`Match("ssn").ReadStr(f).Mask(MaskTag.Last4)`). One observer can do both in the same pass: ```csharp using DragoAnt.System.Text.Json.Observer; @@ -43,8 +43,8 @@ using DragoAnt.System.Text.Json.Observer; var observer = JsonObserver.Obj( rules => rules .Match("id").ReadInt((id, order) => order.Id = id) - .Match("card").MaskAny("***"), - JsonObserverValuePolicies.BlockList); + .Match("card").Mask("***"), + ValuePolicy.BlockList); var order = new Order(); Console.WriteLine(observer.Mask("""{"id":42,"card":"4111111111111111","total":9.5}""", order)); @@ -71,18 +71,19 @@ Ready-made agent skills for masking, HTTP body logging and testing — and how t ### Default policy — what happens to values no rule names -The last argument of a factory or of `Relative(...)` decides. Booleans are values like any other. +The last argument of a factory or of `AnyDepth(...)` decides: a `ValuePolicy` (it works for observers of any context type), an `AnyDepth(...)` rule set, or `JsonValuePolicy.Custom(rule)`. Booleans are values like any other. | Policy | `{"s":"x","n":1,"b":true,"z":null}` becomes | | --- | --- | | `AllowList` (the default) | `{"s":"***","n":"***","b":"***","z":null}` | | `BlockList` | unchanged: only values a rule names are masked | | `NullList` | `{"s":null,"n":null,"b":null,"z":null}` | -| `LegacyAllowList` (obsolete, the 1.x default) | `{"s":"#str#*****","n":"#number#*****","b":true,"z":null}` | +| `Tagged(tag)` | every string, number and boolean masked by the call's strategy with the tag, for example hashed | ```csharp using DragoAnt.System.Text.Json.Observer; -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; const string json = """{"s":"x","n":1,"b":true,"z":null}"""; Console.WriteLine(JsonObserver.Obj(AllowList).Mask(json)); @@ -92,23 +93,23 @@ Console.WriteLine(JsonObserver.Obj(NullList).Mask(json)); // {"s":null,"n":null,"b":null,"z":null} ``` -### Absolute and relative rules +### Absolute and any-depth rules -**Absolute** rules follow the path from the root, one `Match` per level or several names in one `Match`. **Relative** rules (inside `Relative(...)`) match the end of a property's path at any depth. Names match case-insensitively; `PropMatches.StartsWith`, `EndsWith`, `Contains`, `OneOf` and `Regex` test a name differently and follow the same case option. The first rule that writes a value wins; read rules run on every match. +**Absolute** rules follow the path from the root: `Match(name)` for one level, `Path(a, b, …)` for several. **Any-depth** rules (inside `AnyDepth(...)`) match a property's name, or with `Path` the end of its path, wherever it is nested. Names match case-insensitively; `Names.StartsWith`, `EndsWith`, `Contains`, `OneOf` and `Regex` test a name differently and follow the same case option. The first rule that writes a value wins; read rules run on every match. ```csharp using DragoAnt.System.Text.Json.Observer; -using DragoAnt.System.Text.Json.Observer.Strategies; -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; var absolute = JsonObserver.Obj(root => root .Match("order").Obj(order => order .Match("id").Unmasked() .Match("status").Unmasked())); -var relative = JsonObserver.Obj(Relative(rules => rules - .Match(PropMatches.EndsWith("card"), "saved", "id").MaskStr("***") - .Match(PropMatches.Contains("email")).MaskStr("***"), +var relative = JsonObserver.Obj(AnyDepth(rules => rules + .Path(Names.EndsWith("card"), "saved", "id").Mask("***") + .Match(Names.Contains("email")).Mask("***"), BlockList)); Console.WriteLine(absolute.Mask("""{"order":{"id":42,"status":"paid","customer":"Alice","paid":true},"note":"x"}""")); @@ -124,53 +125,52 @@ Every `Mask*` rule masks the **whole value whatever its JSON type** — a sensit | Rule | What the masking function receives | `null` value | | --- | --- | --- | -| `MaskAny(strategy)` | a string decoded; a number or boolean as its literal (`"12.50"`, `"true"`); `null` for an object or array | stays `null`, the function is not called | -| `MaskStr(strategy)` | the same as `MaskAny` | the function receives `null` | -| `MaskRawValue(strategy)` | the same, but a string as its raw JSON text, escapes kept | the function receives `null` | +| `Mask(replacement)` | a string decoded; a number or boolean as its literal (`"12.50"`, `"true"`); `null` for an object or array | stays `null`, the function is not called; `Mask(replacement, MaskNulls.Mask)` passes it | | `MaskInt` / `MaskLong` / `MaskDecimal` | the number when it fits the type, otherwise `null` | the function receives `null` | | `MaskBool` | `true` / `false`, otherwise `null` | the function receives `null` | -| `MaskAny(MaskTag)` | — the call's `Utf8MaskStrategy` writes `Full` `"***"`, `Last4` `"***1234"` (shorter than 8 characters: `"***"`), `Hash` `"hash:…"` or `Omit` `null` | stays `null` | +| `Mask(MaskTag)` | — the call's `ValueMaskStrategy` writes `Full` `"***"`, `Last4` `"***1234"` (shorter than 8 characters: `"***"`), `Hash` the keyed hash (the same text as Microsoft's `HmacRedactor`) or `Null` `null` | stays `null` | +| `Mask(strategy, tag)` | — this rule's own `ValueMaskStrategy` instead of the call's | stays `null` | | `Unmasked()` | — writes a string, number, boolean or `null` unchanged | | | `ReadStr` / `ReadInt` / `ReadLong` / `ReadDecimal` / `ReadBool` / `ReadRaw` | hands the value to the context; the default policy writes it unless `.Unmasked()` or a mask method follows on the same match; a number that does not fit arrives as `null` | | -The table holds for absolute and relative rules alike. A strategy is a constant string, a `Regex` whose matches become `*`, or a function of the value and the context; a `null` result writes `null`. The function receives the whole value, and what it returns is never cut by `MaxValueBytes`, which limits values written unmasked only. `Hash` uses `JsonObserverOptions.HashKey`, or a random key per process when it is empty. +The table holds for absolute and any-depth rules alike. A replacement is a constant string, a `Regex` whose matches become `*`, or a function of the value (and the context); a `null` result writes `null`. The function receives the whole value, and what it returns is never cut by `MaxValueBytes`, which limits values written unmasked only. `Hash` is HMAC-SHA256 with `HashKey` (a random key per process when it is empty), written exactly as Microsoft's `HmacRedactor` writes it: 16 bytes in base64, after `":"` when a key id is set. ### Custom mask strategies -A `MaskTag` can carry a `Key` — a data classification, a redactor name — that only your `Utf8MaskStrategy` interprets; the built-in strategy falls back to the tag's kind (`MaskTag.Custom(key)` becomes `"***"`). Override `Mask(in Utf8MaskContext, JsonWriter)` to also see the property name and the path of the value, without allocations. +A `MaskTag` can carry a `Key` — a data classification, a redactor name — that only your `ValueMaskStrategy` interprets; the built-in strategy falls back to the tag's kind (`MaskTag.Custom(key)` becomes `"***"`). `Mask(in MaskContext, MaskValueWriter)` sees the value, its kind, the tag, the property name and the path of the value, without allocations. ```csharp using System.Text; using DragoAnt.System.Text.Json.Observer; -using DragoAnt.System.Text.Json.Observer.Strategies; -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; -var masker = JsonObserver.Obj(Relative(rules => rules - .Match("email").MaskAny(MaskTag.Custom("pii")) - .Match("password").MaskAny(MaskTag.Full), +var masker = JsonObserver.Obj(AnyDepth(rules => rules + .Match("email").Mask(MaskTag.Custom("pii")) + .Match("password").Mask(MaskTag.Full), BlockList)); -var options = new JsonObserverOptions(MaskStrategy: new LabelStrategy()); +var options = new JsonObserverOptions { Strategy = new LabelStrategy() }; Console.WriteLine(masker.Mask("""{"user":{"email":"a@b.c","password":"s3cret"},"items":[{"email":"x@y.z"}]}""", options)); // Output: // {"user":{"email":"","password":"***"},"items":[{"email":""}]} -sealed class LabelStrategy : Utf8MaskStrategy +sealed class LabelStrategy : ValueMaskStrategy { - public override void Mask(in Utf8MaskContext context, JsonWriter writer) + public override void Mask(in MaskContext context, MaskValueWriter output) { if (context.Tag.TryGetKey(out var label)) { - writer.WriteStringValue($"<{label} at {context.Path.ToString()}>"); + output.String($"<{label} at {context.Path.ToString()}>"); return; } - Default.Mask(context, writer); + Default.Mask(context, output); } } ``` -`Utf8MaskContext` has `Value`, `TokenType`, `Tag`, `Options`, `PropertyName` (UTF-8), `IsArrayItem` and `Path`; `JsonWriter` takes `ReadOnlySpan` values too, so a char-based redactor writes its result without an intermediate string. +`MaskContext` has `Value`, `Kind`, `Tag`, `Options`, `Name` (UTF-8), `IsArrayItem`, `Path` and `ValueIndex`; `MaskValueWriter` takes `ReadOnlySpan` strings too, so a char-based redactor writes its result without an intermediate string, and it validates numbers. The strategy is the same for every format. ### Explain a path @@ -179,19 +179,20 @@ sealed class LabelStrategy : Utf8MaskStrategy ```csharp using System.Text.Json; using DragoAnt.System.Text.Json.Observer; -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; var masker = JsonObserver.Obj( root => root.Match("lines").Array(lines => lines.Obj(line => line.Match("sku").Unmasked())), - Relative(rules => rules.Match("password").MaskAny("***"), AllowList)); + AnyDepth(rules => rules.Match("password").Mask("***"), AllowList)); Console.WriteLine(masker.Explain("lines[2].sku")); -Console.WriteLine(masker.Explain("lines[2].qty", JsonTokenType.Number)); +Console.WriteLine(masker.Explain("lines[2].qty", ValueKind.Number)); Console.WriteLine(masker.Explain("user.password")); // Output: // lines[2].sku: Unchanged by Match("lines") > object item > Match("sku") → Unmasked() // lines[2].qty: Masked by default policy AllowList → writes "***" -// user.password: Masked by relative Match("password") → MaskAny("***") +// user.password: Masked by AnyDepth Match("password") → Mask("***") ``` The result also lists one step per level (`Steps`) and the `Outcome`: `Unchanged`, `Masked`, `Read`, `Custom` or `Invalid`. Observers built from a `JsonShape` explain against the shape. @@ -204,7 +205,6 @@ The result also lists one step per level (`Steps`) and the `Outcome`: `Unchanged using System.Text.Json; using System.Text.Json.Serialization.Metadata; using DragoAnt.System.Text.Json.Observer; -using DragoAnt.System.Text.Json.Observer.Strategies; var options = new JsonSerializerOptions(JsonSerializerDefaults.Web) { TypeInfoResolver = new DefaultJsonTypeInfoResolver() }; var shape = JsonShape.FromTypeInfo(options.GetTypeInfo(typeof(Customer)), property => property.Name == "card" ? MaskTag.Last4 : null); @@ -217,7 +217,7 @@ Console.WriteLine(masker.Mask("""{"name":"Alice","card":"4111111111111111","adde sealed record Customer(string Name, string Card); ``` -`JsonShapeOptions` choose what happens to unknown members (`MaskWhole`, `Descend`, `PassThrough`) and whether `null` stays; `JsonShapeOptions.FromSerializerOptions(options)` also matches names with the serializer's `PropertyNameCaseInsensitive`. On .NET 8, source-generated metadata carries no attributes, so classify by name there. +`JsonShapeOptions` choose what happens to unknown members (`MaskWhole`, `Descend`, `PassThrough`) and whether `null` stays; `JsonShapeOptions.FromSerializerOptions(options)` also matches names with the serializer's `PropertyNameCaseInsensitive`; `UnknownTag` masks unknown values with another tag. On .NET 8, source-generated metadata carries no attributes, so classify by name there. Every node and member keeps its metadata for integrations: `JsonShape.Members` lists `JsonShapeProperty` items with the `JsonPropertyInfo`, the CLR member, `PropertyType`, `IsRequired`, `IsNullable` and `GetCustomAttributes()`, and nodes and members carry `Annotations` that an integration fills — for example from the `annotate` callback of `FromTypeInfo`. @@ -229,16 +229,17 @@ Both APIs never throw. `MaskResult` says what happened: | `MaskStatus` | Meaning | Output | | --- | --- | --- | -| `Masked` | the whole payload was masked | the masked JSON | -| `Truncated` | the payload ended early or hit `MaxOutputBytes` (`FailedAtByte` is where reading stopped), or a string was cut to `MaxValueBytes` (`FailedAtByte` is `-1`) | valid JSON: the masked part, every open object and array closed | +| `Masked` | the whole payload was masked; `Flags` is empty | the masked JSON | +| `Truncated` | the payload ended early or hit `MaxOutputBytes` (`FailedAtByte` is where reading stopped), or a string was cut to `MaxValueBytes` (`FailedAtByte` is `-1`); `Flags` says which | valid JSON: the masked part, every open object and array closed | | `Invalid` | the payload is not valid JSON, or a rule failed | valid JSON: the masked part read before the failure | -| `NotJson` | empty, or the root is not an object or an array | empty | +| `Unrecognized` | empty, or the root is not an object or an array | empty | ```csharp using DragoAnt.System.Text.Json.Observer; -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; -var masker = JsonObserver.Obj(Relative(rules => rules.Match("password").MaskAny("***"), BlockList)); +var masker = JsonObserver.Obj(AnyDepth(rules => rules.Match("password").Mask("***"), BlockList)); var output = masker.Mask("""{"user":{"login":"alice","password":"s3cret","roles":["admin","dev""", out var result); Console.WriteLine(output); @@ -256,9 +257,10 @@ Console.WriteLine(result.Status); using System.Buffers; using System.Text; using DragoAnt.System.Text.Json.Observer; -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; -var masker = JsonObserver.Obj(Relative(rules => rules.Match("password").MaskAny("***"), BlockList)); +var masker = JsonObserver.Obj(AnyDepth(rules => rules.Match("password").Mask("***"), BlockList)); var output = new ArrayBufferWriter(1024); MaskResult result = masker.Mask("""{"user":"alice","password":"secret"}"""u8, output); @@ -269,7 +271,7 @@ Console.WriteLine($"{result.Status} {Encoding.UTF8.GetString(output.WrittenSpan) ### Options -`JsonObserverOptions` apply to both APIs: +`JsonObserverOptions` apply to both APIs; every option but `RelaxedEscaping` and `Indented` comes from the format-neutral `ObserverOptions`: | Option | Default | Effect | | --- | --- | --- | @@ -277,10 +279,10 @@ Console.WriteLine($"{result.Status} {Encoding.UTF8.GetString(output.WrittenSpan) | `MaxValueBytes` | unlimited | longest string written unmasked; a longer one is cut, ends with `…`, and the status is `Truncated`; mask output is never cut | | `MaxDepth` | 64 | deeper nesting is `Invalid` | | `RelaxedEscaping` | `true` | non-ASCII and HTML characters are written unescaped | -| `HashKey`, `MaskStrategy` | random per process, built-in | used by `MaskTag` rules | +| `HashKey`, `HashKeyId`, `Strategy` | random per process, none, built-in | used by `MaskTag` rules; `WithBase64HashKey(key)` takes a Microsoft `HmacRedactorOptions.Key` | | `IgnoreNulls` | `false` | drops `null` properties and items, and objects and arrays left empty by that | | `Indented` | `false` | indented output | -| `PropertyNameCaseInsensitive` | `true` | match rule names, `PropMatches` tests (`Regex` included) and shapes ignoring case; pass the serializer's setting to match names as deserialization does | +| `NameCaseInsensitive` | `true` | match rule names, `Names` tests (`Regex` included) and shapes ignoring case; pass the serializer's setting to match names as deserialization does | Input may contain comments and trailing commas; a UTF-8 byte order mark is skipped. Comments are not written. @@ -319,7 +321,7 @@ using var response = await httpClient.SendAsync(request, cancellationToken); public sealed class PaymentMaskers : IJsonBodyMaskerProvider { - private static readonly JsonObserver Charge = JsonObserver.Obj(rules => rules.Match("cardNumber").MaskStr("****")); + private static readonly JsonObserver Charge = JsonObserver.Obj(rules => rules.Match("cardNumber").Mask("****")); // null logs the body as "[body withheld]". public JsonObserver? GetMasker(Type? modelType, string clientName) => @@ -335,9 +337,9 @@ Without an `IJsonBodyMaskerProvider` every value of an object or array body is m - **Not logged** — the query string (`Path` is the path only). `IncludeSensitive = true` logs bodies unmasked, marks the entry `BodyUnmasked` and logs a warning; use it for local debugging only. - **Options** are named per client and read on every call, so a reload applies to the next call; `services.ConfigureAll(...)` changes every client. Add the handler with `AddJsonBodyLogging`, not `AddHttpMessageHandler()`. -## Upgrading from 1.x +## Upgrading -See the breaking changes in the [changelog](./CHANGELOG.md). +See [Migrating to 2.0](./docs/migrating-to-2.0.md) and the breaking changes in the [changelog](./CHANGELOG.md). ## Contributing diff --git a/skills/json-observer-http-logging/SKILL.md b/skills/json-observer-http-logging/SKILL.md index d338d79..58a2b6a 100644 --- a/skills/json-observer-http-logging/SKILL.md +++ b/skills/json-observer-http-logging/SKILL.md @@ -7,7 +7,7 @@ description: Log HttpClient request and response JSON bodies with secrets masked `JsonBodyLoggingHandler` sits in an `HttpClient` pipeline and turns a call into one `JsonBodyLogEntry` — method, path, status, outcome, elapsed time, and the request and response bodies **masked** by a `JsonObserver` you choose per body model type. The caller is never affected: the response streams as usual, its body stays readable in full, and a failure inside logging never reaches the caller. -Package: `DragoAnt.System.Text.Json.Observer.Http` (net8.0, net9.0, net10.0), namespace `DragoAnt.System.Text.Json.Observer.Http`. Masking rules come from `DragoAnt.System.Text.Json.Observer` — see the `json-observer-masking` skill. +Package: `DragoAnt.System.Text.Json.Observer.Http` (net8.0, net10.0), namespace `DragoAnt.System.Text.Json.Observer.Http`. Masking rules come from `DragoAnt.System.Text.Json.Observer` — see the `json-observer-masking` skill. ## Quick start @@ -17,9 +17,10 @@ using System.Net.Http.Json; using System.Text; using DragoAnt.System.Text.Json.Observer; using DragoAnt.System.Text.Json.Observer.Http; -using DragoAnt.System.Text.Json.Observer.Strategies; +using DragoAnt.Observer; using Microsoft.Extensions.DependencyInjection; -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; var sink = new ConsoleSink(); var services = new ServiceCollection(); @@ -50,8 +51,8 @@ public sealed record ChargeResponse(string Id, string? Error); public sealed class PaymentMaskers : IJsonBodyMaskerProvider { - private static readonly JsonObserver Charge = JsonObserver.Obj(Relative(rules => rules - .Match("cardNumber").MaskAny(MaskTag.Last4), + private static readonly JsonObserver Charge = JsonObserver.Obj(AnyDepth(rules => rules + .Match("cardNumber").Mask(MaskTag.Last4), BlockList)); public JsonObserver? GetMasker(Type? modelType, string clientName) => diff --git a/skills/json-observer-http-logging/examples.md b/skills/json-observer-http-logging/examples.md index 4a93f76..bb2258a 100644 --- a/skills/json-observer-http-logging/examples.md +++ b/skills/json-observer-http-logging/examples.md @@ -13,7 +13,8 @@ using System.Text; using DragoAnt.System.Text.Json.Observer; using DragoAnt.System.Text.Json.Observer.Http; using Microsoft.Extensions.DependencyInjection; -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; var sink = new CaptureSink(); var services = new ServiceCollection(); @@ -50,9 +51,9 @@ public sealed class UsersClient(HttpClient http) public sealed class UserMaskers : IJsonBodyMaskerProvider { - private static readonly JsonObserver Users = JsonObserver.Obj(Relative(rules => rules - .Match("password").MaskAny("***") - .Match("apiToken").MaskAny("***"), + private static readonly JsonObserver Users = JsonObserver.Obj(AnyDepth(rules => rules + .Match("password").Mask("***") + .Match("apiToken").Mask("***"), BlockList)); public JsonObserver? GetMasker(Type? modelType, string clientName) => @@ -87,7 +88,8 @@ using System.Net; using System.Text; using DragoAnt.System.Text.Json.Observer; using DragoAnt.System.Text.Json.Observer.Http; -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; var sink = new ListSink(); var options = new JsonBodyLoggingOptions @@ -111,7 +113,7 @@ Console.WriteLine($"[{entry.ClientName}] {entry.Method} {entry.Path} {entry.Requ public sealed class OneMasker : IJsonBodyMaskerProvider { - private static readonly JsonObserver Masker = JsonObserver.Obj(Relative(rules => rules.Match("token").MaskAny("***"), BlockList)); + private static readonly JsonObserver Masker = JsonObserver.Obj(AnyDepth(rules => rules.Match("token").Mask("***"), BlockList)); public JsonObserver? GetMasker(Type? modelType, string clientName) => Masker; } @@ -194,10 +196,11 @@ Under `OnFailure`, a call that throws is logged without a response, then the exc using System.Text; using DragoAnt.System.Text.Json.Observer; using DragoAnt.System.Text.Json.Observer.Http; -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; var sink = new PrintSink(); -var masker = new FixedMasker(JsonObserver.Obj(Relative(rules => rules.Match("pin").MaskAny("***"), BlockList))); +var masker = new FixedMasker(JsonObserver.Obj(AnyDepth(rules => rules.Match("pin").Mask("***"), BlockList))); var handler = new JsonBodyLoggingHandler(new JsonBodyLoggingOptions(), masker, sink) { InnerHandler = new Unreachable() }; using var client = new HttpClient(handler) { BaseAddress = new Uri("https://bank.example.com/") }; diff --git a/skills/json-observer-http-logging/pitfalls.md b/skills/json-observer-http-logging/pitfalls.md index 69c8084..654af2f 100644 --- a/skills/json-observer-http-logging/pitfalls.md +++ b/skills/json-observer-http-logging/pitfalls.md @@ -19,7 +19,8 @@ using System.Net; using System.Text; using DragoAnt.System.Text.Json.Observer; using DragoAnt.System.Text.Json.Observer.Http; -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; foreach (var attachTypes in new[] { false, true }) { @@ -50,7 +51,7 @@ public sealed record Login(string User, string Password); public sealed class LoginMaskers : IJsonBodyMaskerProvider { - private static readonly JsonObserver Login = JsonObserver.Obj(Relative(rules => rules.Match("password").MaskAny("***"), BlockList)); + private static readonly JsonObserver Login = JsonObserver.Obj(AnyDepth(rules => rules.Match("password").Mask("***"), BlockList)); public JsonObserver? GetMasker(Type? modelType, string clientName) => modelType == typeof(Login) ? Login : null; } diff --git a/skills/json-observer-http-logging/recipes.md b/skills/json-observer-http-logging/recipes.md index 5266661..07cd4dc 100644 --- a/skills/json-observer-http-logging/recipes.md +++ b/skills/json-observer-http-logging/recipes.md @@ -15,7 +15,7 @@ using System.Text.Json; using System.Text.Json.Serialization.Metadata; using DragoAnt.System.Text.Json.Observer; using DragoAnt.System.Text.Json.Observer.Http; -using DragoAnt.System.Text.Json.Observer.Strategies; +using DragoAnt.Observer; var sink = new CaptureSink(); var handler = new JsonBodyLoggingHandler(new JsonBodyLoggingOptions { When = JsonBodyLogWhen.Always }, new ModelMaskers(), sink) @@ -236,7 +236,8 @@ using System.Net; using System.Text; using DragoAnt.System.Text.Json.Observer; using DragoAnt.System.Text.Json.Observer.Http; -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; var sink = new CaptureSink(); var options = new JsonBodyLoggingOptions { When = JsonBodyLogWhen.Always, MaxBodyBytes = 40 }; diff --git a/skills/json-observer-masking/SKILL.md b/skills/json-observer-masking/SKILL.md index 7f8b870..d4cebda 100644 --- a/skills/json-observer-masking/SKILL.md +++ b/skills/json-observer-masking/SKILL.md @@ -7,18 +7,18 @@ description: Mask or extract values in JSON with DragoAnt.System.Text.Json.Obser `JsonObserver` rewrites a JSON payload token by token: values a rule names are masked (or handed to a context object), everything else follows a **default policy**. It never builds objects or a DOM, and it **never throws** — a cut-off or invalid payload yields its masked prefix, closed into valid JSON. -Package: `DragoAnt.System.Text.Json.Observer` (net8.0, net9.0, net10.0). Namespaces: `DragoAnt.System.Text.Json.Observer`, `.Strategies` (`MaskTag`, `PropMatches`) and `using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies;` (`BlockList`, `AllowList`, `NullList`, `Relative`). +Package: `DragoAnt.System.Text.Json.Observer` (net8.0, net10.0). Namespaces: `DragoAnt.System.Text.Json.Observer`, and `DragoAnt.Observer` (`MaskTag`, `Names`, `MaskResult`, `ValuePolicy`; the package imports it into every C# file). Add `using static DragoAnt.Observer.ValuePolicy;` (`BlockList`, `AllowList`, `NullList`, `Tagged`) and `using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy;` (`AnyDepth`). ## Quick start ```csharp using DragoAnt.System.Text.Json.Observer; -using DragoAnt.System.Text.Json.Observer.Strategies; -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; -var masker = JsonObserver.Obj(Relative(rules => rules - .Match("password").MaskAny("***") - .Match("card", "number").MaskAny(MaskTag.Last4), +var masker = JsonObserver.Obj(AnyDepth(rules => rules + .Match("password").Mask("***") + .Path("card", "number").Mask(MaskTag.Last4), BlockList)); Console.WriteLine(masker.Mask("""{"user":"alice","password":"s3cret","card":{"number":"4111111111111111"},"active":true}""")); @@ -29,7 +29,7 @@ Console.WriteLine(masker.Mask("""{"user":"alice","password":"s3cret","card":{"nu ## Decision path 1. **Do you have the DTO the JSON comes from?** Build an allow-list from it: `JsonShape.FromTypeInfo(typeInfo, classify)` + `JsonObserver.FromShape(shape)`. Known fields pass, the ones you classify are masked by tag, anything new is masked. Safest choice for third-party payloads → [examples.md#allow-list-from-a-type](./examples.md#allow-list-from-a-type). -2. **You only know which names are sensitive** ("mask every `password`, wherever it is") → **block-list**: `JsonObserver.Obj(Relative(rules => …, BlockList))`. +2. **You only know which names are sensitive** ("mask every `password`, wherever it is") → **block-list**: `JsonObserver.Obj(AnyDepth(rules => …, BlockList))`. 3. **You know which fields are safe to show** and want everything else hidden → **allow-list rules**: absolute rules with `.Unmasked()` under the default `AllowList`. 4. **You also need values out** (an order id for a log scope) → `JsonObserver.Obj(…)` with `Read*` rules; `Read(json, ctx)` extracts without writing → [examples.md#extract-values-while-masking](./examples.md#extract-values-while-masking). 5. **Hot path** (every request body) → the UTF-8 API `Mask(ReadOnlySpan, IBufferWriter)` with a reused `ArrayBufferWriter` → [recipes.md#hot-path-utf-8-api](./recipes.md#hot-path-utf-8-api). @@ -38,18 +38,18 @@ Console.WriteLine(masker.Mask("""{"user":"alice","password":"s3cret","card":{"nu ## Rules that matter 1. **Build once, share everywhere.** An observer is immutable and thread-safe; keep it in a `static readonly` field. Building one per call costs far more than masking. -2. **The default policy is `AllowList`.** A factory without a policy, and `Relative(rules)` without its second argument, write every string, number **and boolean** no rule names as `"***"` (`null` stays). Pass `BlockList` to keep unnamed values. -3. **Absolute vs relative.** Rules on a builder (`Obj(root => root.Match("order").Obj(…))`) follow the path from the root. Rules inside `Relative(…)` match the **end** of a path at any depth: `Match("card", "number")` hits every `…card.number`. An **array item is one path level**, so reach `{"lines":[{"qty":…}]}` with `Match("lines", anyItem, "qty")` where `anyItem = new PropMatchingStrategy(_ => true)`. The first rule that matches wins. -4. **Names match exactly and case-insensitively.** Use `PropMatches.EndsWith/StartsWith/Contains/OneOf/Regex` for anything else; they follow the same case option. -5. **Prefer `MaskAny` for secrets.** Every `Mask*` rule masks the whole value whatever its JSON type (a number, a boolean, an object), but `MaskAny` keeps `null` as `null` without calling your function, while `MaskStr` passes `null` to it. -6. **Tags for standard masks:** `MaskAny(MaskTag.Full)` → `"***"`, `Last4` → `"***1111"` (shorter than 8 characters → `"***"`), `Hash` → `"hash:<16 hex>"`, `Omit` → `null`. Set `JsonObserverOptions.HashKey` for hashes that correlate across processes; the default key is random per process. -7. **Never throws, always safe.** Both APIs return the masked prefix of cut-off or invalid input, closed into valid JSON, and never write a masked value in clear. `MaskStatus` is `Masked`, `Truncated`, `Invalid` or `NotJson` (empty output: empty input, or a root that is not an object or array). +2. **The default policy is `AllowList`.** A factory without a policy, and `AnyDepth(rules)` without its second argument, write every string, number **and boolean** no rule names as `"***"` (`null` stays). Pass `BlockList` to keep unnamed values. +3. **Absolute vs relative.** Rules on a builder (`Obj(root => root.Match("order").Obj(…))`) follow the path from the root. Rules inside `AnyDepth(…)` match the **end** of a path at any depth: `Match("card", "number")` hits every `…card.number`. An **array item is one path level**, so reach `{"lines":[{"qty":…}]}` with `Match("lines", anyItem, "qty")` where `anyItem = new NameMatch(_ => true)`. The first rule that matches wins. +4. **Names match exactly and case-insensitively.** Use `Names.EndsWith/StartsWith/Contains/OneOf/Regex` for anything else; they follow the same case option. +5. **Prefer `Mask` for secrets.** Every `Mask*` rule masks the whole value whatever its JSON type (a number, a boolean, an object), and `Mask` keeps `null` as `null` without calling your function; `Mask(f, MaskNulls.Mask)` passes `null` to it. +6. **Tags for standard masks:** `Mask(MaskTag.Full)` → `"***"`, `Last4` → `"***1111"` (shorter than 8 characters → `"***"`), `Hash` → 24 base64 characters, the same text as Microsoft's `HmacRedactor`, `Null` → `null`. Set `JsonObserverOptions.HashKey` for hashes that correlate across processes; the default key is random per process. +7. **Never throws, always safe.** Both APIs return the masked prefix of cut-off or invalid input, closed into valid JSON, and never write a masked value in clear. `MaskStatus` is `Masked`, `Truncated`, `Invalid` or `Unrecognized` (empty output: empty input, or a root that is not an object or array). 8. **Measure allocations, never assume zero.** With constant or tag rules, the UTF-8 API allocates a small constant amount per call; a masking function receives a `string`, which allocates. ## Pitfalls (details in [pitfalls.md](./pitfalls.md)) -- "Everything became `***`" → you used the default `AllowList`; pass `BlockList` (to the factory, or as `Relative`'s second argument). -- "The secret is still visible" → the rule is absolute but the field is nested, or the name differs (`Password` vs `passwd`); use `Relative` and a `PropMatches`. +- "Everything became `***`" → you used the default `AllowList`; pass `BlockList` (to the factory, or as `AnyDepth`'s second argument). +- "The secret is still visible" → the rule is absolute but the field is nested, or the name differs (`Password` vs `passwd`); use `AnyDepth` and a `Names`. - A rule written before an `Obj(...)` rule for the same name wins and masks the whole object. - `JsonObserver.Obj(...)` on a root array returns `Invalid`; use `JsonObserver.Any(...)` when the root can be either. - Comments in the input are accepted and never written; a UTF-8 BOM is skipped. diff --git a/skills/json-observer-masking/examples.md b/skills/json-observer-masking/examples.md index e445e32..b6021a6 100644 --- a/skills/json-observer-masking/examples.md +++ b/skills/json-observer-masking/examples.md @@ -4,7 +4,7 @@ Each block is a complete program: create a console project, reference `DragoAnt. ## Default policies -The default policy decides what happens to a value no rule names. It is the last argument of `JsonObserver.Obj/Array/Any`, of `Relative(...)`, and of nested `Obj(...)`/`Array(...)` rules (a nested rule inherits the enclosing one when it is omitted). +The default policy decides what happens to a value no rule names. It is the last argument of `JsonObserver.Obj/Array/Any`, of `AnyDepth(...)`, and of nested `Obj(...)`/`Array(...)` rules (a nested rule inherits the enclosing one when it is omitted). | Policy | `{"s":"x","n":1,"b":true,"z":null}` becomes | | --- | --- | @@ -14,7 +14,8 @@ The default policy decides what happens to a value no rule names. It is the last ```csharp using DragoAnt.System.Text.Json.Observer; -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; const string json = """{"s":"x","n":1,"b":true,"z":null}"""; @@ -41,7 +42,7 @@ var masker = JsonObserver.Obj(root => root .Match("order").Obj(order => order .Match("status").Unmasked() .Match("total").Unmasked()) - .Match("customer", "country").Unmasked()); + .Path("customer", "country").Unmasked()); Console.WriteLine(masker.Mask(""" {"id":7,"order":{"status":"paid","total":9.5,"note":"leave at door"},"customer":{"name":"Alice","country":"NL"}} @@ -50,22 +51,22 @@ Console.WriteLine(masker.Mask(""" // {"id":7,"order":{"status":"paid","total":9.5,"note":"***"},"customer":{"name":"***","country":"NL"}} ``` -## Relative rules and name matchers +## Any-depth rules and name matchers -`Relative(rules, defaultPolicy)` is a policy whose rules match the **end** of a property path at any depth. A plain string is an exact, case-insensitive name; `PropMatches` tests names differently. The first matching rule wins. +`AnyDepth(rules, defaultPolicy)` is a policy whose rules match the **end** of a property path at any depth. A plain string is an exact, case-insensitive name; `Names` tests names differently. The first matching rule wins. ```csharp using System.Text.RegularExpressions; using DragoAnt.System.Text.Json.Observer; -using DragoAnt.System.Text.Json.Observer.Strategies; -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; - -var masker = JsonObserver.Obj(Relative(rules => rules - .Match(PropMatches.EndsWith("token")).MaskAny("***") - .Match(PropMatches.Contains("email")).MaskAny("***") - .Match(PropMatches.OneOf("pwd", "passwd", "password")).MaskAny("***") - .Match(PropMatches.Regex(new Regex("^x-api-", RegexOptions.IgnoreCase))).MaskAny("***") - .Match("card", "cvv").MaskAny("***"), +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; + +var masker = JsonObserver.Obj(AnyDepth(rules => rules + .Match(Names.EndsWith("token")).Mask("***") + .Match(Names.Contains("email")).Mask("***") + .Match(Names.OneOf("pwd", "passwd", "password")).Mask("***") + .Match(Names.Regex(new Regex("^x-api-", RegexOptions.IgnoreCase))).Mask("***") + .Path("card", "cvv").Mask("***"), BlockList)); Console.WriteLine(masker.Mask(""" @@ -75,15 +76,16 @@ Console.WriteLine(masker.Mask(""" // {"auth":{"AccessToken":"***","refresh_token":"***"},"user":{"WorkEmail":"***","PWD":"***"},"headers":{"X-Api-Key":"***"},"payment":{"card":{"cvv":"***","brand":"visa"}}} ``` -Absolute and relative rules combine: absolute rules first, then a `Relative(...)` policy for everything they do not name. +Absolute and relative rules combine: absolute rules first, then a `AnyDepth(...)` policy for everything they do not name. ```csharp using DragoAnt.System.Text.Json.Observer; -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; var masker = JsonObserver.Obj( - root => root.Match("debug").MaskAny("[removed]"), - Relative(rules => rules.Match("password").MaskAny("***"), BlockList)); + root => root.Match("debug").Mask("[removed]"), + AnyDepth(rules => rules.Match("password").Mask("***"), BlockList)); Console.WriteLine(masker.Mask("""{"debug":{"trace":"…"},"login":{"user":"bob","password":"p"}}""")); // Output: @@ -96,28 +98,28 @@ Every `Mask*` rule masks the whole value, whatever its JSON type; an object or a | Rule | The function receives | A `null` value | | --- | --- | --- | -| `MaskAny("***")` / `MaskAny((value, ctx) => …)` | a string decoded; a number or boolean as its literal (`"12.50"`, `"true"`); `null` for an object or array | stays `null`; the function is not called | -| `MaskStr(...)` | the same as `MaskAny` | may reach the function as `null` | -| `MaskRawValue(...)` | the same, but a string as raw JSON text, escapes kept | may reach the function as `null` | +| `Mask("***")` / `Mask((value, ctx) => …)` | a string decoded; a number or boolean as its literal (`"12.50"`, `"true"`); `null` for an object or array | stays `null`; the function is not called | +| `Mask(..., MaskNulls.Mask)` | the same as `Mask` | the function is called with `null` | | `MaskInt` / `MaskLong` / `MaskDecimal` | the number when it fits, otherwise `null` | may reach the function as `null` | | `MaskBool` | `true`/`false`, otherwise `null` | may reach the function as `null` | -| `MaskAny(MaskTag)` | — written by the tag strategy | stays `null` | +| `Mask(MaskTag)` | — written by the tag strategy | stays `null` | | `Unmasked()` | — a string, number, boolean or `null` written unchanged | stays `null` | -A strategy is a constant string, a `Regex` whose matches become `*`, or a function; a function returning `null` writes `null`. Write functions so that a `null` input returns `null` (or a constant): whether a JSON `null` reaches a `MaskStr`-family function differs between absolute and relative rules. +A strategy is a constant string, a `Regex` whose matches become `*`, or a function; a function returning `null` writes `null`. Write functions so that a `null` input returns `null` (or a constant) when you pass `MaskNulls.Mask`. ```csharp using System.Text.RegularExpressions; using DragoAnt.System.Text.Json.Observer; -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; -var masker = JsonObserver.Obj(Relative(rules => rules - .Match("pin").MaskAny("***") - .Match("phone").MaskStr(new Regex("[0-9](?=[0-9]{2})")) - .Match("amount").MaskAny((value, _) => value is null ? null : $"<{value.Length} chars>") +var masker = JsonObserver.Obj(AnyDepth(rules => rules + .Match("pin").Mask("***") + .Match("phone").Mask(new Regex("[0-9](?=[0-9]{2})"), MaskNulls.Mask) + .Match("amount").Mask((value, _) => value is null ? null : $"<{value.Length} chars>") .Match("age").MaskInt((age, _) => age >= 18 ? "adult" : "minor") - .Match("address").MaskAny("***") - .Match("note").MaskAny("***"), + .Match("address").Mask("***") + .Match("note").Mask("***"), BlockList)); Console.WriteLine(masker.Mask(""" @@ -131,23 +133,23 @@ Console.WriteLine(masker.Mask(""" `JsonObserver.Obj(...)` expects a root object and `JsonObserver.Array(...)` a root array; the other root is `Invalid`. `JsonObserver.Any(obj, array, policy)` accepts both. In an array builder every rule applies to every item; `Obj(...)` handles the items that are objects. -**An array item is one level of a property path.** A multi-name `Match` crosses one level per name, so `Match("lines", "sku")` never reaches `{"lines":[{"sku":…}]}`; put a match-anything test where the item is: `Match("lines", AnyItem, "sku")` with `AnyItem = new PropMatchingStrategy(_ => true)`. +**An array item is one level of a property path.** A multi-name `Match` crosses one level per name, so `Match("lines", "sku")` never reaches `{"lines":[{"sku":…}]}`; put a match-anything test where the item is: `Match("lines", AnyItem, "sku")` with `AnyItem = new NameMatch(_ => true)`. ```csharp using DragoAnt.System.Text.Json.Observer; -using DragoAnt.System.Text.Json.Observer.Strategies; -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; -var anyItem = new PropMatchingStrategy(_ => true); +var anyItem = new NameMatch(_ => true); var rootArray = JsonObserver.Array(items => items.Obj(item => item.Match("sku").Unmasked())); var scalarItems = JsonObserver.Obj(root => root.Match("tags").Array(tags => tags.Unmasked())); var objectItems = JsonObserver.Obj(root => root - .Match("lines", anyItem, "sku").Unmasked() - .Match("lines", anyItem, "qty").Unmasked()); + .Path("lines", anyItem, "sku").Unmasked() + .Path("lines", anyItem, "qty").Unmasked()); -var either = JsonObserver.Any(_ => { }, _ => { }, Relative(rules => rules.Match("password").MaskAny("***"), BlockList)); +var either = JsonObserver.Any(_ => { }, _ => { }, AnyDepth(rules => rules.Match("password").Mask("***"), BlockList)); Console.WriteLine(rootArray.Mask("""[{"sku":"A1","price":3},{"sku":"B2","price":4}]""")); Console.WriteLine(scalarItems.Mask("""{"tags":["vip",3],"customer":"Alice"}""")); @@ -168,31 +170,31 @@ Console.WriteLine(result.Status); ## Tags -`MaskAny(MaskTag.X)` masks with the call's `Utf8MaskStrategy` (the built-in one unless `JsonObserverOptions.MaskStrategy` sets another). `Hash` is an HMAC-SHA256 keyed by `JsonObserverOptions.HashKey`; with no key, a random key is used for the lifetime of the process. +`Mask(MaskTag.X)` masks with the call's `ValueMaskStrategy` (the built-in one unless `JsonObserverOptions.Strategy` sets another). `Hash` is an HMAC-SHA256 keyed by `JsonObserverOptions.HashKey`, in the output format of Microsoft's `HmacRedactor` (16 bytes in base64, `":"` first when a key id is set); with no key, a random key is used for the lifetime of the process. ```csharp using System.Text; using System.Text.RegularExpressions; using DragoAnt.System.Text.Json.Observer; -using DragoAnt.System.Text.Json.Observer.Strategies; -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; - -var masker = JsonObserver.Obj(Relative(rules => rules - .Match("full").MaskAny(MaskTag.Full) - .Match("card").MaskAny(MaskTag.Last4) - .Match("short").MaskAny(MaskTag.Last4) - .Match("email").MaskAny(MaskTag.Hash) - .Match("ssn").MaskAny(MaskTag.Omit), +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; + +var masker = JsonObserver.Obj(AnyDepth(rules => rules + .Match("full").Mask(MaskTag.Full) + .Match("card").Mask(MaskTag.Last4) + .Match("short").Mask(MaskTag.Last4) + .Match("email").Mask(MaskTag.Hash) + .Match("ssn").Mask(MaskTag.Null), BlockList)); -var options = new JsonObserverOptions(HashKey: Encoding.UTF8.GetBytes("a key shared by every instance")); +var options = new JsonObserverOptions { HashKey = Encoding.UTF8.GetBytes("a key shared by every instance") }; var masked = masker.Mask("""{"full":true,"card":"4111111111111111","short":"1234567","email":"a@b.c","ssn":"123-45-6789"}""", options)!; -var hash = Regex.Match(masked, "hash:[0-9a-f]{16}").Value; +var hash = Regex.Match(masked, "[A-Za-z0-9+/]{22}==").Value; -Console.WriteLine(masked.Replace(hash, "hash:…")); +Console.WriteLine(masked.Replace(hash, "")); Console.WriteLine(masker.Mask("""{"email":"a@b.c"}""", options) == $$"""{"email":"{{hash}}"}"""); // Output: -// {"full":"***","card":"***1111","short":"***","email":"hash:…","ssn":null} +// {"full":"***","card":"***1111","short":"***","email":"","ssn":null} // True ``` @@ -211,7 +213,6 @@ using System.Text.Json; using System.Text.Json.Serialization; using System.Text.Json.Serialization.Metadata; using DragoAnt.System.Text.Json.Observer; -using DragoAnt.System.Text.Json.Observer.Strategies; var jsonOptions = new JsonSerializerOptions(JsonSerializerDefaults.Web) { TypeInfoResolver = new DefaultJsonTypeInfoResolver() }; var shape = JsonShape.FromTypeInfo( @@ -219,7 +220,7 @@ var shape = JsonShape.FromTypeInfo( property => property.AttributeProvider?.IsDefined(typeof(SensitiveAttribute), inherit: true) == true ? MaskTag.Last4 : null); var maskWhole = JsonObserver.FromShape(shape); -var descend = JsonObserver.FromShape(shape, new JsonShapeOptions(UnknownMemberPolicy.Descend)); +var descend = JsonObserver.FromShape(shape, new JsonShapeOptions { Unknown = UnknownMemberPolicy.Descend }); const string json = """{"name":"Alice","card_no":"4111111111111111","tags":["vip"],"extra":{"risk":"high"}}"""; Console.WriteLine(maskWhole.Mask(json)); @@ -247,7 +248,6 @@ On .NET 8, metadata from a source-generated `JsonSerializerContext` carries no a ```csharp using DragoAnt.System.Text.Json.Observer; -using DragoAnt.System.Text.Json.Observer.Strategies; var shape = JsonShape.Object( ("id", JsonShape.Scalar), @@ -264,7 +264,7 @@ Console.WriteLine(JsonObserver.FromShape(shape).Mask(""" ## Extract values while masking -`JsonObserver.Obj(...)` adds `Read*` rules that hand a value to a context object. A read rule does not decide what is written: the default policy writes the value (here `BlockList`, so unchanged) unless `.Unmasked()` or a mask method is chained on the read, as in `Match("ssn").ReadStr(f).MaskAny(MaskTag.Last4)`. The context-aware policies live in `JsonObserverValuePolicies`. `Mask(json, context)` masks and extracts in one pass; `Read(json, context)` only extracts and returns a `MaskResult`. +`JsonObserver.Obj(...)` adds `Read*` rules that hand a value to a context object. A read rule does not decide what is written: the default policy writes the value (here `BlockList`, so unchanged) unless `.Unmasked()` or a mask method is chained on the read, as in `Match("ssn").ReadStr(f).Mask(MaskTag.Last4)`. The context-aware policies live in `JsonObserverValuePolicies`. `Mask(json, context)` masks and extracts in one pass; `Read(json, context)` only extracts and returns a `MaskResult`. ```csharp using DragoAnt.System.Text.Json.Observer; @@ -273,8 +273,8 @@ var observer = JsonObserver.Obj( root => root .Match("orderId").ReadLong((id, info) => info.OrderId = id) .Match("total").ReadDecimal((total, info) => info.Total = total) - .Match("customer", "email").MaskAny("***"), - JsonObserverValuePolicies.BlockList); + .Path("customer", "email").Mask("***"), + ValuePolicy.BlockList); const string json = """{"orderId":1001,"total":19.90,"customer":{"email":"a@b.c","tier":"gold"}}"""; diff --git a/skills/json-observer-masking/migrating-from-1x.md b/skills/json-observer-masking/migrating-from-1x.md index 250ec42..78ce39d 100644 --- a/skills/json-observer-masking/migrating-from-1x.md +++ b/skills/json-observer-masking/migrating-from-1x.md @@ -1,10 +1,10 @@ # Migrating from 1.x to 2.0 — json-observer-masking -2.0 keeps the builder API (`JsonObserver.Obj/Array/Any`, `Match`, `Relative`, `Mask*`, `Read*`) and changes what the defaults produce. Work through this list; the "before" blocks are 1.x code and do not compile against 2.0. +2.0 keeps the shape of the builder API (`JsonObserver.Obj/Array/Any`, `Match`, `Mask*`, `Read*`), renames parts of it (`Relative` → `AnyDepth`, multi-name `Match` → `Path`, `MaskAny`/`MaskStr`/`MaskRawValue` → `Mask`), moves the format-neutral types to the `DragoAnt.Observer` namespace of [DragoAnt.Observer.Core](https://www.nuget.org/packages/DragoAnt.Observer.Core), and changes what the defaults produce. The full rename table is in the repository's `docs/migrating-to-2.0.md`. Work through this list; the "before" blocks are 1.x code and do not compile against 2.0. ## 1. The default policy masks booleans and uses one token -`AllowList` (still the default) now writes every string, number **and boolean** as `"***"`. 1.x wrote `"#str#*****"` / `"#number#*****"` and kept booleans; that output survives as the obsolete `LegacyAllowList`. Update golden strings in tests, and log parsers that looked for `#str#`. +`AllowList` (still the default) now writes every string, number **and boolean** as `"***"`. 1.x wrote `"#str#*****"` / `"#number#*****"` and kept booleans; that output is gone (`LegacyAllowList` was removed). Update golden strings in tests, and log parsers that looked for `#str#`. ```csharp using DragoAnt.System.Text.Json.Observer; @@ -33,10 +33,11 @@ catch (JsonException) ```csharp using DragoAnt.System.Text.Json.Observer; -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; var observer = JsonObserver.Obj(BlockList); -var masked = observer.Mask("""{"a":1,"b":null,}""", out var result, new JsonObserverOptions(IgnoreNulls: true)); +var masked = observer.Mask("""{"a":1,"b":null,}""", out var result, new JsonObserverOptions { IgnoreNulls = true }); Console.WriteLine($"{result.Status} {masked}"); // Output: // Masked {"a":1} @@ -65,13 +66,14 @@ sealed class Holder ## 4. Every `Mask*` rule masks the whole value, whatever its type -`MaskStr`, `MaskRawValue`, `MaskInt`, `MaskLong`, `MaskDecimal` and `MaskBool` used to hand a value of another type to the default policy (so under `BlockList` a numeric `cvv` under `MaskStr` stayed visible) and descended into objects. Now they mask any value and skip containers unread; `MaskStr` receives a number or boolean as its literal. Because they also match containers, a mask rule placed before an `Obj(...)`/`Array(...)` rule for the same name now wins over it — reorder such rules. +1.x `MaskStr`, `MaskRawValue`, `MaskInt`, `MaskLong`, `MaskDecimal` and `MaskBool` used to hand a value of another type to the default policy (so under `BlockList` a numeric `cvv` under `MaskStr` stayed visible) and descended into objects. Now every mask rule masks any value and skips containers unread; `Mask` receives a number or boolean as its literal. Because they also match containers, a mask rule placed before an `Obj(...)`/`Array(...)` rule for the same name now wins over it — reorder such rules. ```csharp using DragoAnt.System.Text.Json.Observer; -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; -var observer = JsonObserver.Obj(Relative(rules => rules.Match("cvv").MaskStr("***").Match("address").MaskStr("***"), BlockList)); +var observer = JsonObserver.Obj(AnyDepth(rules => rules.Match("cvv").Mask("***", MaskNulls.Mask).Match("address").Mask("***", MaskNulls.Mask), BlockList)); Console.WriteLine(observer.Mask("""{"cvv":123,"address":{"street":"Main 1"}}""")); // Output: // {"cvv":"***","address":"***"} @@ -85,9 +87,9 @@ It used to report success. `FailedAtByte` is -1 in that case (the whole document `ReadInt`, `ReadLong` and `ReadDecimal` receive `null` for a number that does not fit; the default policy writes the token. -## 7. `PropertyPath` is a `ref struct` +## 7. Custom rules take one context: `JsonValueRule` -Custom `MaskValue` rules that keep a `PropertyPath` beyond the call, construct one, or use `MaxLength`/`Dispose` must change: only `GetPropertyName`, `GetPropertyNameReverse`, `Length` and `ToString` remain. Rebuild custom rules against 2.0. +`MaskValue`, `Obj(rule)` and `Array(rule)` take `(ref JsonValueContext c) => …`: `c.Reader`, `c.Writer`, `c.Context`, `c.Path` (a `DataPath`, valid during the call only: `GetName`, `GetNameFromEnd`, `TryGetItemIndex`, `Length`, `ToString`) and `c.WriteDefault()`. A custom default policy is `JsonValuePolicy.Custom(rule)`. Rebuild custom rules against 2.0. ## 8. `JsonWriter` is sealed to the library @@ -95,7 +97,7 @@ It cannot be derived from outside; `JsonWriter.FromUtf8JsonWriter`, `JsonWriter. ## 9. Internal types -`JsonObserverException`, `PropertyPathMatch`, `JsonPropertyMatchDelegate`, `JsonPropertyPathMatchDelegate` and the builder constructors are internal. Start rules with `Match(...)` on the builder you are given. +`JsonObserverException`, the engine's delegates, `JsonShape.FindMember` and the builder constructors are internal. Start rules with `Match(...)` or `Path(...)` on the builder you are given. ## 10. A UTF-8 byte order mark is skipped @@ -105,11 +107,10 @@ A `Read*` rule used to write its value unchanged, even under `AllowList`. It now ```csharp using DragoAnt.System.Text.Json.Observer; -using DragoAnt.System.Text.Json.Observer.Strategies; var observer = JsonObserver.Obj(root => root .Match("id").ReadInt((id, p) => p.Id = id).Unmasked() - .Match("ssn").ReadStr((ssn, p) => p.Ssn = ssn).MaskAny(MaskTag.Last4)); + .Match("ssn").ReadStr((ssn, p) => p.Ssn = ssn).Mask(MaskTag.Last4)); var person = new Person(); Console.WriteLine(observer.Mask("""{"id":7,"ssn":"123-45-6789","name":"Kim"}""", person)); Console.WriteLine($"{person.Id} {person.Ssn}"); @@ -131,5 +132,5 @@ A masking function used to receive a value longer than `MaxValueBytes` cut to th ## New in 2.0, worth adopting while you migrate - The UTF-8 API with a reused `IBufferWriter` ([recipes.md](./recipes.md#hot-path-utf-8-api)). -- `MaskAny(MaskTag.Last4/Hash/Omit)` instead of hand-written masking functions ([examples.md](./examples.md#tags)). +- `Mask(MaskTag.Last4/Hash/Null)` instead of hand-written masking functions ([examples.md](./examples.md#tags)). - `JsonShape.FromTypeInfo` + `JsonObserver.FromShape` for structure-aware allow-lists ([examples.md](./examples.md#allow-list-from-a-type)). diff --git a/skills/json-observer-masking/pitfalls.md b/skills/json-observer-masking/pitfalls.md index 587580d..41285b8 100644 --- a/skills/json-observer-masking/pitfalls.md +++ b/skills/json-observer-masking/pitfalls.md @@ -4,18 +4,19 @@ Symptom first, then the cause and the fix. Each block is a complete program. ## Everything became `"***"` -**Cause:** the default policy is `AllowList`. `JsonObserver.Obj(rules)` without a second argument, and `Relative(rules)` without its second argument, mask every string, number and boolean no rule names. +**Cause:** the default policy is `AllowList`. `JsonObserver.Obj(rules)` without a second argument, and `AnyDepth(rules)` without its second argument, mask every string, number and boolean no rule names. **Fix:** pass `BlockList` when only the named values are sensitive. ```csharp using DragoAnt.System.Text.Json.Observer; -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; const string json = """{"user":"alice","password":"p","active":true}"""; -Console.WriteLine(JsonObserver.Obj(Relative(rules => rules.Match("password").MaskAny("x"))).Mask(json)); -Console.WriteLine(JsonObserver.Obj(Relative(rules => rules.Match("password").MaskAny("x"), BlockList)).Mask(json)); +Console.WriteLine(JsonObserver.Obj(AnyDepth(rules => rules.Match("password").Mask("x"))).Mask(json)); +Console.WriteLine(JsonObserver.Obj(AnyDepth(rules => rules.Match("password").Mask("x"), BlockList)).Mask(json)); // Output: // {"user":"***","password":"x","active":"***"} // {"user":"alice","password":"x","active":true} @@ -25,20 +26,20 @@ Console.WriteLine(JsonObserver.Obj(Relative(rules => rules.Match("password").Mas Check, in order: -1. **The rule is absolute, the field is nested.** `JsonObserver.Obj(root => root.Match("password")…)` only matches a top-level `password`. Use `Relative(...)` to match at any depth. -2. **The name differs.** Matching is exact (case-insensitive): `password` does not match `newPassword` or `passwd`. Use `PropMatches.Contains("password")` or `PropMatches.OneOf(...)`. -3. **The path crosses an array.** An array item is a path level: `Match("users", "password")` does not reach `{"users":[{"password":…}]}`. Use a single name in `Relative(...)`, or `Match("users", AnyItem, "password")` with `AnyItem = new PropMatchingStrategy(_ => true)`. -4. **The rule only reads.** A `Read*` rule leaves the writing to the default policy, which under `BlockList` writes the value unchanged; chain a mask method on the read (`ReadStr(f).MaskAny(MaskTag.Full)`) or use a `Mask*` rule. +1. **The rule is absolute, the field is nested.** `JsonObserver.Obj(root => root.Match("password")…)` only matches a top-level `password`. Use `AnyDepth(...)` to match at any depth. +2. **The name differs.** Matching is exact (case-insensitive): `password` does not match `newPassword` or `passwd`. Use `Names.Contains("password")` or `Names.OneOf(...)`. +3. **The path crosses an array.** An array item is a path level: `Match("users", "password")` does not reach `{"users":[{"password":…}]}`. Use a single name in `AnyDepth(...)`, or `Match("users", AnyItem, "password")` with `AnyItem = new NameMatch(_ => true)`. +4. **The rule only reads.** A `Read*` rule leaves the writing to the default policy, which under `BlockList` writes the value unchanged; chain a mask method on the read (`ReadStr(f).Mask(MaskTag.Full)`) or use a `Mask*` rule. ```csharp using DragoAnt.System.Text.Json.Observer; -using DragoAnt.System.Text.Json.Observer.Strategies; -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; const string json = """{"login":{"newPassword":"p1"},"users":[{"password":"p2"}]}"""; -var tooNarrow = JsonObserver.Obj(root => root.Match("password").MaskAny("***"), BlockList); -var fixedRules = JsonObserver.Obj(Relative(rules => rules.Match(PropMatches.Contains("password")).MaskAny("***"), BlockList)); +var tooNarrow = JsonObserver.Obj(root => root.Match("password").Mask("***"), BlockList); +var fixedRules = JsonObserver.Obj(AnyDepth(rules => rules.Match(Names.Contains("password")).Mask("***"), BlockList)); Console.WriteLine(tooNarrow.Mask(json)); Console.WriteLine(fixedRules.Mask(json)); @@ -55,17 +56,18 @@ Console.WriteLine(fixedRules.Mask(json)); ```csharp using DragoAnt.System.Text.Json.Observer; -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; const string json = """{"card":{"brand":"visa","number":"4111"}}"""; var maskFirst = JsonObserver.Obj(root => root - .Match("card").MaskAny("***") - .Match("card").Obj(card => card.Match("number").MaskAny("***")), BlockList); + .Match("card").Mask("***") + .Match("card").Obj(card => card.Match("number").Mask("***")), BlockList); var objFirst = JsonObserver.Obj(root => root - .Match("card").Obj(card => card.Match("number").MaskAny("***")) - .Match("card").MaskAny("***"), BlockList); + .Match("card").Obj(card => card.Match("number").Mask("***")) + .Match("card").Mask("***"), BlockList); Console.WriteLine(maskFirst.Mask(json)); Console.WriteLine(objFirst.Mask(json)); @@ -76,13 +78,14 @@ Console.WriteLine(objFirst.Mask(json)); ## The output is empty -**Cause:** `MaskStatus.NotJson` — the input is empty, or its root is a string or number. Or `Invalid` with nothing read: a root array given to `JsonObserver.Obj(...)` (or a root object to `JsonObserver.Array(...)`). +**Cause:** `MaskStatus.Unrecognized` — the input is empty, or its root is a string or number. Or `Invalid` with nothing read: a root array given to `JsonObserver.Obj(...)` (or a root object to `JsonObserver.Array(...)`). **Fix:** check `MaskResult.Status`; use `JsonObserver.Any(...)` when the root can be an object or an array. ```csharp using DragoAnt.System.Text.Json.Observer; -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; var objOnly = JsonObserver.Obj(BlockList); var any = JsonObserver.Any(_ => { }, _ => { }, BlockList); @@ -92,7 +95,7 @@ Console.WriteLine($"[{objOnly.Mask("\"text\"", out var r2)}] {r2.Status}"); Console.WriteLine($"[{any.Mask("[1]", out var r3)}] {r3.Status}"); // Output: // [] Invalid -// [] NotJson +// [] Unrecognized // [[1]] Masked ``` @@ -106,7 +109,7 @@ Console.WriteLine($"[{any.Mask("[1]", out var r3)}] {r3.Status}"); ## A masking function is slow on huge values -**Cause:** a masking function receives the whole value, decoded to a `string`, whatever `MaxValueBytes` says; the cap limits values written unmasked only. Prefer `MaskAny(MaskTag…)` or a constant for fields that can be huge: they never decode the value to a `string`. +**Cause:** a masking function receives the whole value, decoded to a `string`, whatever `MaxValueBytes` says; the cap limits values written unmasked only. Prefer `Mask(MaskTag…)` or a constant for fields that can be huge: they never decode the value to a `string`. ## Building an observer per call diff --git a/skills/json-observer-masking/recipes.md b/skills/json-observer-masking/recipes.md index 16b9692..317e085 100644 --- a/skills/json-observer-masking/recipes.md +++ b/skills/json-observer-masking/recipes.md @@ -8,8 +8,8 @@ A body headed for a log is usually size-capped, so it may be cut mid-document. M ```csharp using DragoAnt.System.Text.Json.Observer; -using DragoAnt.System.Text.Json.Observer.Strategies; -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; var logged = BodyLog.Describe("""{"user":"alice","password":"s3cret","card":{"number":"4111111111111111","cvv":123}}"""); Console.WriteLine(logged); @@ -18,13 +18,13 @@ Console.WriteLine(logged); static class BodyLog { - private static readonly JsonObserver Masker = JsonObserver.Obj(Relative(rules => rules - .Match(PropMatches.OneOf("password", "secret", "cvv")).MaskAny(MaskTag.Full) - .Match("card", "number").MaskAny(MaskTag.Last4) - .Match(PropMatches.EndsWith("token")).MaskAny(MaskTag.Full), + private static readonly JsonObserver Masker = JsonObserver.Obj(AnyDepth(rules => rules + .Match(Names.OneOf("password", "secret", "cvv")).Mask(MaskTag.Full) + .Path("card", "number").Mask(MaskTag.Last4) + .Match(Names.EndsWith("token")).Mask(MaskTag.Full), BlockList)); - private static readonly JsonObserverOptions Options = new(MaxOutputBytes: 4096, MaxValueBytes: 512); + private static readonly JsonObserverOptions Options = new() { MaxOutputBytes = 4096, MaxValueBytes = 512 }; public static string Describe(string body) { @@ -42,7 +42,8 @@ static class BodyLog using System.Buffers; using System.Text; using DragoAnt.System.Text.Json.Observer; -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; var first = Utf8Masking.Mask("""{"user":"alice","password":"p1"}"""u8); var second = Utf8Masking.Mask("""{"user":"bob","password":"p2"}"""u8); @@ -54,7 +55,7 @@ Console.WriteLine(second); static class Utf8Masking { - private static readonly JsonObserver Masker = JsonObserver.Obj(Relative(rules => rules.Match("password").MaskAny("***"), BlockList)); + private static readonly JsonObserver Masker = JsonObserver.Obj(AnyDepth(rules => rules.Match("password").Mask("***"), BlockList)); [ThreadStatic] private static ArrayBufferWriter? _output; @@ -80,13 +81,14 @@ Both APIs never throw. Branch on `MaskResult.Status`: | `Masked` | the whole payload was read and masked | the masked JSON | | `Truncated` | the payload ended inside the document, or the output reached `MaxOutputBytes` (`FailedAtByte` = where reading stopped), or a string was cut to `MaxValueBytes` (`FailedAtByte` = -1) | valid JSON: the masked part, open objects and arrays closed | | `Invalid` | not valid JSON (including plain text), deeper than `MaxDepth`, the root type the observer does not accept, or a rule threw | valid JSON: the masked part read before the failure (may be empty) | -| `NotJson` | empty input, or valid JSON whose root is not an object or array (`"text"`, `42`) | empty string, `BytesWritten` 0 | +| `Unrecognized` | empty input, or valid JSON whose root is not an object or array (`"text"`, `42`) | empty string, `BytesWritten` 0 | ```csharp using DragoAnt.System.Text.Json.Observer; -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; -var masker = JsonObserver.Obj(Relative(rules => rules.Match("password").MaskAny("***"), BlockList)); +var masker = JsonObserver.Obj(AnyDepth(rules => rules.Match("password").Mask("***"), BlockList)); foreach (var body in new[] { @@ -104,7 +106,7 @@ foreach (var body in new[] // Masked at= -1 {"user":"alice","password":"***"} // Truncated at= 61 {"user":{"login":"alice","password":"***","roles":["admin"]}} // Invalid at= 16 {"user":"alice"} -// NotJson at= 0 +// Unrecognized at= 0 // Invalid at= 0 ``` @@ -121,19 +123,20 @@ A value that is cut never leaks: a string being written when the input ends is d | `MaxDepth` | 64 | deeper nesting → `Invalid` | | `RelaxedEscaping` | `true` | non-ASCII and HTML characters written unescaped | | `HashKey` | random per process | key of `MaskTag.Hash` | -| `MaskStrategy` | built-in | writes `MaskTag` rules | +| `Strategy` | built-in | writes `MaskTag` rules | | `IgnoreNulls` | `false` | drops `null` properties and items, and objects and arrays left empty by that | | `Indented` | `false` | indented output | ```csharp using DragoAnt.System.Text.Json.Observer; -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; var masker = JsonObserver.Obj(BlockList); const string json = """{"note":"a very long free-text note","empty":null,"tags":[null],"city":"Zürich"}"""; -Console.WriteLine(masker.Mask(json, new JsonObserverOptions(MaxValueBytes: 10, IgnoreNulls: true))); -Console.WriteLine(masker.Mask(json, new JsonObserverOptions(RelaxedEscaping: false))); +Console.WriteLine(masker.Mask(json, new JsonObserverOptions { MaxValueBytes = 10, IgnoreNulls = true })); +Console.WriteLine(masker.Mask(json, new JsonObserverOptions { RelaxedEscaping = false })); // Output: // {"note":"a very lon…","city":"Zürich"} // {"note":"a very long free-text note","empty":null,"tags":[null],"city":"Z\u00FCrich"} @@ -141,19 +144,19 @@ Console.WriteLine(masker.Mask(json, new JsonObserverOptions(RelaxedEscaping: fal ## Correlate masked values across services -`MaskTag.Hash` writes `hash:` and 16 hex characters of an HMAC-SHA256. Give every instance the same `HashKey` (from configuration or a secret store, never from source) and the same value hashes the same everywhere, so you can follow one customer through logs without seeing their email. Without a key, hashes only correlate inside one process. +`MaskTag.Hash` writes an HMAC-SHA256 the way Microsoft's `HmacRedactor` does: 16 bytes in base64 (24 characters), after `":"` when `HashKeyId` is set, so a hash correlates with logs redacted by `Microsoft.Extensions.Compliance.Redaction` under the same key (`WithBase64HashKey(key)` takes its `HmacRedactorOptions.Key`). Give every instance the same `HashKey` (from configuration or a secret store, never from source) and the same value hashes the same everywhere, so you can follow one customer through logs without seeing their email. Without a key, hashes only correlate inside one process. ```csharp using System.Text; using DragoAnt.System.Text.Json.Observer; -using DragoAnt.System.Text.Json.Observer.Strategies; -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; -var masker = JsonObserver.Obj(Relative(rules => rules.Match("email").MaskAny(MaskTag.Hash), BlockList)); +var masker = JsonObserver.Obj(AnyDepth(rules => rules.Match("email").Mask(MaskTag.Hash), BlockList)); var keyFromConfiguration = Encoding.UTF8.GetBytes("load-me-from-configuration"); -var serviceA = new JsonObserverOptions(HashKey: keyFromConfiguration); -var serviceB = new JsonObserverOptions(HashKey: keyFromConfiguration.ToArray()); +var serviceA = new JsonObserverOptions { HashKey = keyFromConfiguration }; +var serviceB = new JsonObserverOptions { HashKey = keyFromConfiguration.ToArray() }; var a = masker.Mask("""{"email":"alice@example.com"}""", serviceA); var b = masker.Mask("""{"email":"alice@example.com"}""", serviceB); @@ -166,35 +169,34 @@ Console.WriteLine(a!.Contains("alice", StringComparison.Ordinal)); ## Custom mask strategy -`MaskTag` rules are written by a `Utf8MaskStrategy`. Replace it per call with `JsonObserverOptions.MaskStrategy` — one strategy serves every tag; delegate the kinds you do not change to `Utf8MaskStrategy.Default`. `value` is the unescaped string, the literal of a number or boolean, or empty for an object or array; write exactly one value. +`MaskTag` rules are written by a `ValueMaskStrategy`. Replace it per call with `JsonObserverOptions.Strategy` — one strategy serves every tag; delegate the kinds you do not change to `ValueMaskStrategy.Default`. `context.Value` is the unescaped string, the literal of a number or boolean, or empty for an object or array; write exactly one value to the `MaskValueWriter`. The strategy is format-neutral: the same class serves the CSV and YAML observers. ```csharp -using System.Text.Json; using DragoAnt.System.Text.Json.Observer; -using DragoAnt.System.Text.Json.Observer.Strategies; -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; -var masker = JsonObserver.Obj(Relative(rules => rules - .Match("password").MaskAny(MaskTag.Full) - .Match("card").MaskAny(MaskTag.Last4), +var masker = JsonObserver.Obj(AnyDepth(rules => rules + .Match("password").Mask(MaskTag.Full) + .Match("card").Mask(MaskTag.Last4), BlockList)); -var options = new JsonObserverOptions(MaskStrategy: new RedactedStrategy()); +var options = new JsonObserverOptions { Strategy = new RedactedStrategy() }; Console.WriteLine(masker.Mask("""{"password":"p","card":"4111111111111111"}""", options)); // Output: // {"password":"[redacted]","card":"***1111"} -sealed class RedactedStrategy : Utf8MaskStrategy +sealed class RedactedStrategy : ValueMaskStrategy { - public override void Mask(ReadOnlySpan value, JsonTokenType tokenType, MaskTag tag, JsonWriter writer, JsonObserverOptions options) + public override void Mask(in MaskContext context, MaskValueWriter output) { - if (tag.Kind == MaskKind.Full) + if (context.Tag.Kind == MaskKind.Full) { - writer.WriteStringValue("[redacted]"u8); + output.String("[redacted]"u8); return; } - Default.Mask(value, tokenType, tag, writer, options); + Default.Mask(context, output); } } ``` diff --git a/skills/json-observer-testing/SKILL.md b/skills/json-observer-testing/SKILL.md index 59815f1..31e016b 100644 --- a/skills/json-observer-testing/SKILL.md +++ b/skills/json-observer-testing/SKILL.md @@ -29,7 +29,7 @@ Packages: `DragoAnt.System.Text.Json.Observer` (and `.Http` for handler tests), 3. **Cover the shapes that defeat rules:** the field nested deeper, inside an array, as a number or boolean, as an object, with different casing, cut off mid-value. Each is a separate case. 4. **Prove the test can fail.** Remove the rule (or build the observer without it) and the test must go red; a check that passes either way protects nothing. Run this once by hand when writing the test, or keep it as a test ([examples.md#the-rule-is-load-bearing](./examples.md#the-rule-is-load-bearing)). 5. **Fuzz with an allow-list observer.** Random corruption can rename a property (`password` → `pas#word`); under `BlockList` that value is then legitimately unmasked, so a block-list fuzz fails for the wrong reason. Under an allow-list, unknown names are masked, so "no secret" must always hold. Truncation fuzz (cutting, not changing bytes) is safe with any observer. -6. **When output is non-empty it must parse.** For every status, `BytesWritten > 0` means the output is valid JSON; `NotJson` means empty output. +6. **When output is non-empty it must parse.** For every status, `BytesWritten > 0` means the output is valid JSON; `Unrecognized` means empty output. 7. **Set allocation budgets from measurements**, with warm-up calls first and a reused output buffer, on the UTF-8 API. Never assert zero. 8. **HTTP response entries are written asynchronously.** Wait on the sink with a timeout; never read it straight after `SendAsync`. diff --git a/skills/json-observer-testing/examples.md b/skills/json-observer-testing/examples.md index bee5675..29271c5 100644 --- a/skills/json-observer-testing/examples.md +++ b/skills/json-observer-testing/examples.md @@ -8,15 +8,15 @@ Exact input → output pairs. Raw string literals keep the JSON readable; one `[ ```csharp using DragoAnt.System.Text.Json.Observer; -using DragoAnt.System.Text.Json.Observer.Strategies; using Xunit; -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; public static class LogMaskers { - public static readonly JsonObserver Body = JsonObserver.Obj(Relative(rules => rules - .Match(PropMatches.OneOf("password", "pin")).MaskAny(MaskTag.Full) - .Match("card", "number").MaskAny(MaskTag.Last4), + public static readonly JsonObserver Body = JsonObserver.Obj(AnyDepth(rules => rules + .Match(Names.OneOf("password", "pin")).Mask(MaskTag.Full) + .Path("card", "number").Mask(MaskTag.Last4), BlockList)); } @@ -39,15 +39,15 @@ One secret value, many shapes that could defeat a rule. Assert absence first, th ```csharp using DragoAnt.System.Text.Json.Observer; -using DragoAnt.System.Text.Json.Observer.Strategies; using Xunit; -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; public static class LogMaskers { - public static readonly JsonObserver Body = JsonObserver.Any(_ => { }, _ => { }, Relative(rules => rules - .Match(PropMatches.Contains("password")).MaskAny("***") - .Match(PropMatches.EndsWith("token")).MaskAny("***"), + public static readonly JsonObserver Body = JsonObserver.Any(_ => { }, _ => { }, AnyDepth(rules => rules + .Match(Names.Contains("password")).Mask("***") + .Match(Names.EndsWith("token")).Mask("***"), BlockList)); } @@ -88,18 +88,19 @@ A secret-absent test is only worth something if it fails without the rule. Keep ```csharp using DragoAnt.System.Text.Json.Observer; using Xunit; -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; public static class LogMaskers { public static readonly JsonObserver Body = Build(maskCvv: true); - internal static JsonObserver Build(bool maskCvv) => JsonObserver.Obj(Relative(rules => + internal static JsonObserver Build(bool maskCvv) => JsonObserver.Obj(AnyDepth(rules => { - rules.Match("password").MaskAny("***"); + rules.Match("password").Mask("***"); if (maskCvv) { - rules.Match("cvv").MaskAny("***"); + rules.Match("cvv").Mask("***"); } }, BlockList)); } @@ -127,13 +128,14 @@ using System.Text; using System.Text.Json; using DragoAnt.System.Text.Json.Observer; using Xunit; -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; public static class LogMaskers { - public static readonly JsonObserver Body = JsonObserver.Obj(Relative(rules => rules - .Match("password").MaskAny("***") - .Match("pin").MaskAny("***"), + public static readonly JsonObserver Body = JsonObserver.Obj(AnyDepth(rules => rules + .Match("password").Mask("***") + .Match("pin").Mask("***"), BlockList)); } @@ -161,7 +163,7 @@ public sealed class LogMaskersTruncationTests using var parsed = JsonDocument.Parse(text); } - var expected = cut == payload.Length ? MaskStatus.Masked : cut == 0 ? MaskStatus.NotJson : MaskStatus.Truncated; + var expected = cut == payload.Length ? MaskStatus.Masked : cut == 0 ? MaskStatus.Unrecognized : MaskStatus.Truncated; Assert.Equal(expected, result.Status); } } @@ -228,11 +230,12 @@ using System.Buffers; using System.Text; using DragoAnt.System.Text.Json.Observer; using Xunit; -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; public static class LogMaskers { - public static readonly JsonObserver Body = JsonObserver.Obj(Relative(rules => rules.Match("password").MaskAny("***"), BlockList)); + public static readonly JsonObserver Body = JsonObserver.Obj(AnyDepth(rules => rules.Match("password").Mask("***"), BlockList)); } public sealed class LogMaskersAllocationTests @@ -282,14 +285,15 @@ An observer is meant to be shared. Run the same inputs on many threads at once ```csharp using DragoAnt.System.Text.Json.Observer; using Xunit; -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; public sealed class SharedObserverTests { [Fact] public void ConcurrentCalls_GiveTheSameOutput() { - var observer = JsonObserver.Obj(Relative(rules => rules.Match("password").MaskAny("***"), BlockList)); + var observer = JsonObserver.Obj(AnyDepth(rules => rules.Match("password").Mask("***"), BlockList)); var deep = """{"a":{"b":{"c":{"d":{"e":{"f":{"g":{"password":"p","keep":1}}}}}}}}"""; var flat = """{"password":"p","keep":1}"""; var expectedDeep = """{"a":{"b":{"c":{"d":{"e":{"f":{"g":{"password":"***","keep":1}}}}}}}}"""; @@ -321,7 +325,8 @@ using System.Text; using DragoAnt.System.Text.Json.Observer; using DragoAnt.System.Text.Json.Observer.Http; using Xunit; -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; public sealed record Login(string User, string Password); @@ -329,9 +334,9 @@ public sealed record Session(string Token, string User); public sealed class AuthMaskers : IJsonBodyMaskerProvider { - private static readonly JsonObserver Auth = JsonObserver.Obj(Relative(rules => rules - .Match("password").MaskAny("***") - .Match("token").MaskAny("***"), + private static readonly JsonObserver Auth = JsonObserver.Obj(AnyDepth(rules => rules + .Match("password").Mask("***") + .Match("token").Mask("***"), BlockList)); public JsonObserver? GetMasker(Type? modelType, string clientName) => diff --git a/skills/json-observer-testing/pitfalls.md b/skills/json-observer-testing/pitfalls.md index fb328ef..a621fbe 100644 --- a/skills/json-observer-testing/pitfalls.md +++ b/skills/json-observer-testing/pitfalls.md @@ -26,7 +26,8 @@ Random corruption can change a property name; under `BlockList` the renamed prop using System.Text; using DragoAnt.System.Text.Json.Observer; using Xunit; -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; public sealed class WhyNotBlockListCorruptionTests { @@ -35,7 +36,7 @@ public sealed class WhyNotBlockListCorruptionTests { var corrupted = """{"pas#word":"S3cr3t-7f2a"}"""; - var blockList = JsonObserver.Obj(Relative(rules => rules.Match("password").MaskAny("***"), BlockList)); + var blockList = JsonObserver.Obj(AnyDepth(rules => rules.Match("password").Mask("***"), BlockList)); var allowList = JsonObserver.Obj(root => root.Match("user").Unmasked()); Assert.Contains("S3cr3t-7f2a", blockList.Mask(corrupted)); @@ -46,12 +47,13 @@ public sealed class WhyNotBlockListCorruptionTests ## False red: expecting `Masked` for a cut or non-JSON input -A cut payload is `Truncated`; plain text is `Invalid`; a JSON scalar root (`42`, `"text"`) or empty input is `NotJson`; a root array given to `JsonObserver.Obj(...)` is `Invalid`. +A cut payload is `Truncated`; plain text is `Invalid`; a JSON scalar root (`42`, `"text"`) or empty input is `Unrecognized`; a root array given to `JsonObserver.Obj(...)` is `Invalid`. ```csharp using DragoAnt.System.Text.Json.Observer; using Xunit; -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; public sealed class StatusExpectationTests { @@ -61,8 +63,8 @@ public sealed class StatusExpectationTests [InlineData("""{"a":1}""", MaskStatus.Masked)] [InlineData("""{"a":1""", MaskStatus.Truncated)] [InlineData("hello", MaskStatus.Invalid)] - [InlineData("42", MaskStatus.NotJson)] - [InlineData("", MaskStatus.NotJson)] + [InlineData("42", MaskStatus.Unrecognized)] + [InlineData("", MaskStatus.Unrecognized)] [InlineData("[1]", MaskStatus.Invalid)] public void Status_MatchesTheInput(string input, MaskStatus expected) { @@ -74,30 +76,30 @@ public sealed class StatusExpectationTests ## False red: `MaskTag.Hash` output in a golden string -Without `JsonObserverOptions.HashKey`, the key is random per process, so the hash changes every run. Pass a fixed key in the test, or assert the shape (`hash:` + 16 hex characters) and equality between two calls. +Without `JsonObserverOptions.HashKey`, the key is random per process, so the hash changes every run. Pass a fixed key in the test, or assert the shape (24 base64 characters, `HmacRedactor`'s format) and equality between two calls. ```csharp using System.Text; using System.Text.RegularExpressions; using DragoAnt.System.Text.Json.Observer; -using DragoAnt.System.Text.Json.Observer.Strategies; using Xunit; -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; public sealed class HashTests { - private static readonly JsonObserver Observer = JsonObserver.Obj(Relative(rules => rules.Match("email").MaskAny(MaskTag.Hash), BlockList)); + private static readonly JsonObserver Observer = JsonObserver.Obj(AnyDepth(rules => rules.Match("email").Mask(MaskTag.Hash), BlockList)); [Fact] public void Hash_HasAStableShape_AndCorrelates() { - var options = new JsonObserverOptions(HashKey: Encoding.UTF8.GetBytes("test-key")); + var options = new JsonObserverOptions { HashKey = Encoding.UTF8.GetBytes("test-key") }; var first = Observer.Mask("""{"email":"a@b.c"}""", options); var second = Observer.Mask("""{"email":"a@b.c"}""", options); var other = Observer.Mask("""{"email":"z@b.c"}""", options); - Assert.Matches("""^\{"email":"hash:[0-9a-f]{16}"\}$""", first); + Assert.Matches("""^\{"email":"[A-Za-z0-9+/]{22}=="\}$""", first); Assert.Equal(first, second); Assert.NotEqual(first, other); } diff --git a/skills/json-observer-testing/recipes.md b/skills/json-observer-testing/recipes.md index 2cd4da7..ec8b6e6 100644 --- a/skills/json-observer-testing/recipes.md +++ b/skills/json-observer-testing/recipes.md @@ -12,11 +12,12 @@ using System.Text; using System.Text.Json; using DragoAnt.System.Text.Json.Observer; using Xunit; -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; public static class LogMaskers { - public static readonly JsonObserver BlockListed = JsonObserver.Obj(Relative(rules => rules.Match("password").MaskAny("***"), BlockList)); + public static readonly JsonObserver BlockListed = JsonObserver.Obj(AnyDepth(rules => rules.Match("password").Mask("***"), BlockList)); public static readonly JsonObserver AllowListed = JsonObserver.Obj(root => root.Match("user").Unmasked()); } @@ -118,21 +119,21 @@ Below, a leak through an array: `Match("credentials", "secret")` does not cross ```csharp using DragoAnt.System.Text.Json.Observer; -using DragoAnt.System.Text.Json.Observer.Strategies; using Xunit; -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; public static class LogMaskers { - private static readonly PropMatchingStrategy AnyItem = new(_ => true); + private static readonly NameMatch AnyItem = new(_ => true); - public static readonly JsonObserver BeforeFix = JsonObserver.Obj(Relative(rules => rules - .Match("credentials", "secret").MaskAny("***"), + public static readonly JsonObserver BeforeFix = JsonObserver.Obj(AnyDepth(rules => rules + .Path("credentials", "secret").Mask("***"), BlockList)); - public static readonly JsonObserver Body = JsonObserver.Obj(Relative(rules => rules - .Match("credentials", "secret").MaskAny("***") - .Match("credentials", AnyItem, "secret").MaskAny("***"), + public static readonly JsonObserver Body = JsonObserver.Obj(AnyDepth(rules => rules + .Path("credentials", "secret").Mask("***") + .Path("credentials", AnyItem, "secret").Mask("***"), BlockList)); } @@ -163,7 +164,7 @@ using System.Text.Json; using System.Text.Json.Serialization.Metadata; using DragoAnt.System.Text.Json.Observer; using DragoAnt.System.Text.Json.Observer.Http; -using DragoAnt.System.Text.Json.Observer.Strategies; +using DragoAnt.Observer; using Xunit; public sealed record SignUp(string Email, string Password, string Country); From 40435f286010cd52d22e532a945803a8ed893242 Mon Sep 17 00:00:00 2001 From: VasiliyF <5789590+vfofanov@users.noreply.github.com> Date: Mon, 5 Oct 2026 09:52:54 +0200 Subject: [PATCH 03/10] Report why a result is not clean in MaskResult.Flags - InputTruncated, OutputCapped, ValueCut, TrailingData (data after the root, which used to read as Masked), InvalidUtf8Replaced, Depth - Masked now means no flag: trailing data and replaced UTF-8 report Truncated with the whole document written --- .../MaskFlagsTests.cs | 136 ++++++++++++++++++ .../RobustnessTests.cs | 7 +- .../RunMaskFlagsTests.cs | 3 + .../BoundedJsonWriter.cs | 17 +++ .../JsonObserver.cs | 71 +++++++-- 5 files changed, 218 insertions(+), 16 deletions(-) create mode 100644 DragoAnt.System.Text.Json.Observer.Tests.Shared/MaskFlagsTests.cs create mode 100644 DragoAnt.System.Text.Json.Observer.Tests/RunMaskFlagsTests.cs diff --git a/DragoAnt.System.Text.Json.Observer.Tests.Shared/MaskFlagsTests.cs b/DragoAnt.System.Text.Json.Observer.Tests.Shared/MaskFlagsTests.cs new file mode 100644 index 0000000..6c771ad --- /dev/null +++ b/DragoAnt.System.Text.Json.Observer.Tests.Shared/MaskFlagsTests.cs @@ -0,0 +1,136 @@ +using System.Buffers; +using System.Text; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; + +namespace DragoAnt.System.Text.Json.Observer.Tests.Shared; + +public abstract class MaskFlagsTests +{ + private static readonly JsonObserver Observer = JsonObserver.Any(_ => { }, _ => { }, AnyDepth(b => b.Match("pin").Mask(MaskTag.Full), BlockList)); + + private static (string Output, MaskResult Result) Mask(byte[] utf8, JsonObserverOptions? options = null) + { + var output = new ArrayBufferWriter(); + var result = Observer.Mask(utf8, output, options); + return (Encoding.UTF8.GetString(output.WrittenSpan), result); + } + + private static (string Output, MaskResult Result) Mask(string json, JsonObserverOptions? options = null) => Mask(Encoding.UTF8.GetBytes(json), options); + + [Fact] + public void CleanPayload_HasNoFlags() + { + var (output, result) = Mask("""{"name":"Bob","pin":1} """); + + output.Should().Be("""{"name":"Bob","pin":"***"}"""); + result.Should().Be(new MaskResult { Status = MaskStatus.Masked, BytesWritten = output.Length, FailedAtByte = -1 }); + } + + [Theory] + [InlineData("""{"name":"Bob"}{"pin":1}""")] + [InlineData("""{"name":"Bob"} [""")] + [InlineData("""{"name":"Bob"} x""")] + [InlineData("""{"name":"Bob"}]""")] + public void DataAfterTheRoot_IsTrailingData(string json) + { + var (output, result) = Mask(json); + + output.Should().Be("""{"name":"Bob"}"""); + result.Status.Should().Be(MaskStatus.Truncated); + result.Flags.Should().Be(MaskFlags.TrailingData); + result.FailedAtByte.Should().Be(-1); + } + + [Fact] + public void DataAfterTheRoot_IsReportedByRead() + => JsonObserver.Obj(_ => { }).Read("""{"a":1}{"b":2}""", NoContext.Instance).Flags.Should().Be(MaskFlags.TrailingData); + + [Fact] + public void CutInput_IsInputTruncated() + { + var (output, result) = Mask("""{"name":"Bob","pin":98"""); + + output.Should().Be("""{"name":"Bob"}"""); + result.Status.Should().Be(MaskStatus.Truncated); + result.Flags.Should().Be(MaskFlags.InputTruncated); + result.FailedAtByte.Should().BeGreaterThan(0); + } + + [Fact] + public void OutputLimit_IsOutputCapped() + { + var (output, result) = Mask("""{"name":"Bob","city":"Berlin","pin":1}""", new JsonObserverOptions { MaxOutputBytes = 16 }); + + output.Should().Be("""{"name":"Bob"}"""); + result.Status.Should().Be(MaskStatus.Truncated); + result.Flags.Should().HaveFlag(MaskFlags.OutputCapped); + } + + [Fact] + public void LongValue_IsValueCut() + { + var (output, result) = Mask("""{"name":"Bartholomew"}""", new JsonObserverOptions { MaxValueBytes = 4 }); + + output.Should().Be("""{"name":"Bart…"}"""); + result.Status.Should().Be(MaskStatus.Truncated); + result.Flags.Should().Be(MaskFlags.ValueCut); + result.FailedAtByte.Should().Be(-1); + } + + public static TheoryData InvalidUtf8 => new() + { + Bytes("{\"name\":\"a", [0xC3], "b\"}"), + Bytes("{\"name\":\"a", [0xC0, 0xAF], "\"}"), + Bytes("{\"name\":\"", [0xED, 0xA0, 0x80], "\"}"), + Bytes("{\"n", [0xC3], "\":1}"), + }; + + private static byte[] Bytes(string before, byte[] invalid, string after) => + [.. Encoding.UTF8.GetBytes(before), .. invalid, .. Encoding.UTF8.GetBytes(after)]; + + [Theory] + [MemberData(nameof(InvalidUtf8))] + public void InvalidUtf8_InAValueOrName_IsReported(byte[] utf8) + { + var (output, result) = Mask(utf8); + + output.Should().Contain(@"\uFFFD"); + result.Status.Should().Be(MaskStatus.Truncated); + result.Flags.Should().Be(MaskFlags.InvalidUtf8Replaced); + } + + [Fact] + public void InvalidUtf8_InAMaskedValue_IsNotReported() + { + var (output, result) = Mask(Bytes("{\"pin\":\"a", [0xC3], "\"}")); + + output.Should().Be("""{"pin":"***"}"""); + result.Should().Be(new MaskResult { Status = MaskStatus.Masked, BytesWritten = output.Length, FailedAtByte = -1 }); + } + + [Fact] + public void TooDeep_IsDepth() + { + var json = string.Concat(Enumerable.Repeat("""{"a":""", 10)) + "1" + new string('}', 10); + + var (output, result) = Mask(json, new JsonObserverOptions { MaxDepth = 5 }); + + result.Status.Should().Be(MaskStatus.Invalid); + result.Flags.Should().Be(MaskFlags.Depth); + JsonDocument.Parse(output).RootElement.ValueKind.Should().Be(JsonValueKind.Object); + } + + [Fact] + public void InvalidJson_HasNoDepthFlag() + => Mask("""{"a":tru e}""").Result.Should().Match(r => r.Status == MaskStatus.Invalid && r.Flags == MaskFlags.None); + + [Fact] + public void SeveralCauses_Combine() + { + var (_, result) = Mask("""{"name":"Bartholomew","x":"y"}{}""", new JsonObserverOptions { MaxValueBytes = 4 }); + + result.Flags.Should().Be(MaskFlags.ValueCut | MaskFlags.TrailingData); + result.Status.Should().Be(MaskStatus.Truncated); + } +} diff --git a/DragoAnt.System.Text.Json.Observer.Tests.Shared/RobustnessTests.cs b/DragoAnt.System.Text.Json.Observer.Tests.Shared/RobustnessTests.cs index c93dad0..59a962a 100644 --- a/DragoAnt.System.Text.Json.Observer.Tests.Shared/RobustnessTests.cs +++ b/DragoAnt.System.Text.Json.Observer.Tests.Shared/RobustnessTests.cs @@ -82,7 +82,12 @@ public void InvalidUtf8_NeverThrows_OutputIsValidJson(byte[] utf8) { var (result, output) = BytesApiTests.Mask(Observer, utf8); - result.Status.Should().BeOneOf(MaskStatus.Masked, MaskStatus.Invalid); + result.Status.Should().BeOneOf(MaskStatus.Masked, MaskStatus.Truncated, MaskStatus.Invalid); + if (result.Status == MaskStatus.Truncated) + { + result.Flags.Should().Be(MaskFlags.InvalidUtf8Replaced); + } + if (output.Length > 0) { var parse = () => JsonDocument.Parse(output).Dispose(); diff --git a/DragoAnt.System.Text.Json.Observer.Tests/RunMaskFlagsTests.cs b/DragoAnt.System.Text.Json.Observer.Tests/RunMaskFlagsTests.cs new file mode 100644 index 0000000..c8a0eaf --- /dev/null +++ b/DragoAnt.System.Text.Json.Observer.Tests/RunMaskFlagsTests.cs @@ -0,0 +1,3 @@ +namespace DragoAnt.System.Text.Json.Observer.Tests; + +public sealed class RunMaskFlagsTests : Shared.MaskFlagsTests; diff --git a/DragoAnt.System.Text.Json.Observer/BoundedJsonWriter.cs b/DragoAnt.System.Text.Json.Observer/BoundedJsonWriter.cs index c7cf94c..911d08c 100644 --- a/DragoAnt.System.Text.Json.Observer/BoundedJsonWriter.cs +++ b/DragoAnt.System.Text.Json.Observer/BoundedJsonWriter.cs @@ -51,6 +51,11 @@ private BoundedJsonWriter(JsonObserverOptions options) /// public bool ValuesTruncated { get; private set; } + /// + /// At least one string or name written held invalid UTF-8, which the writer replaced with U+FFFD. + /// + public bool InvalidUtf8Replaced { get; private set; } + internal override JsonObserverOptions Options => _options; private JsonObserverOptions _options = null!; @@ -88,6 +93,7 @@ private void Start(JsonObserverOptions options) _safeDepth = 0; Exhausted = false; ValuesTruncated = false; + InvalidUtf8Replaced = false; MaskOutput = false; } @@ -183,6 +189,8 @@ public override void WriteStringValue(ReadOnlySpan utf8Value) return; } + CheckUtf8(utf8Value); + if (utf8Value.Length <= MaxValueBytes) { _writer.WriteStringValue(utf8Value); @@ -316,10 +324,19 @@ public override void WritePropertyName(ReadOnlySpan utf8PropertyName) { if (!Exhausted) { + CheckUtf8(utf8PropertyName); _writer.WritePropertyName(utf8PropertyName); } } + private void CheckUtf8(ReadOnlySpan utf8) + { + if (!InvalidUtf8Replaced && !global::System.Text.Unicode.Utf8.IsValid(utf8)) + { + InvalidUtf8Replaced = true; + } + } + public override void WriteStartObject() => Start(isArray: false); public override void WriteStartArray() => Start(isArray: true); diff --git a/DragoAnt.System.Text.Json.Observer/JsonObserver.cs b/DragoAnt.System.Text.Json.Observer/JsonObserver.cs index 76d074e..ed64c5b 100644 --- a/DragoAnt.System.Text.Json.Observer/JsonObserver.cs +++ b/DragoAnt.System.Text.Json.Observer/JsonObserver.cs @@ -330,7 +330,7 @@ private MaskResult Mask(ref Utf8JsonReader reader, ReadOnlySpan input, IBu { using var bounded = BoundedJsonWriter.Rent(options); using var ignoreNulls = options.IgnoreNulls ? IgnoreNullsJsonWriter.Rent(bounded) : null; - var (status, failedAt) = Observe(ref reader, input, (JsonWriter?)ignoreNulls ?? bounded, context, options); + var (status, failedAt, flags) = Observe(ref reader, input, (JsonWriter?)ignoreNulls ?? bounded, context, options); if (status == MaskStatus.Unrecognized) { return new MaskResult { Status = MaskStatus.Unrecognized }; @@ -338,17 +338,32 @@ private MaskResult Mask(ref Utf8JsonReader reader, ReadOnlySpan input, IBu try { - var written = bounded.CopyTo(output, status == MaskStatus.Masked); - if (status == MaskStatus.Masked && bounded.ValuesTruncated) + var written = bounded.CopyTo(output, status is MaskStatus.Masked or MaskStatus.Truncated && failedAt < 0); + if (bounded.Exhausted) + { + flags |= MaskFlags.OutputCapped; + } + + if (bounded.ValuesTruncated) + { + flags |= MaskFlags.ValueCut; + } + + if (bounded.InvalidUtf8Replaced) + { + flags |= MaskFlags.InvalidUtf8Replaced; + } + + if (status == MaskStatus.Masked && flags != MaskFlags.None) { status = MaskStatus.Truncated; } - return new MaskResult { Status = status, BytesWritten = written, FailedAtByte = failedAt }; + return new MaskResult { Status = status, BytesWritten = written, FailedAtByte = failedAt, Flags = flags }; } catch (Exception) { - return new MaskResult { Status = MaskStatus.Invalid, FailedAtByte = failedAt }; + return new MaskResult { Status = MaskStatus.Invalid, FailedAtByte = failedAt, Flags = flags }; } } @@ -399,8 +414,8 @@ public MaskResult Read(ReadOnlySpan utf8, TContext context, JsonObserverOp options ??= JsonObserverOptions.Default; utf8 = SkipBom(utf8); var reader = CreateReader(utf8, options); - var (status, failedAt) = Observe(ref reader, utf8, JsonWriter.Empty, context, options); - return new MaskResult { Status = status, FailedAtByte = failedAt }; + var (status, failedAt, flags) = Observe(ref reader, utf8, JsonWriter.Empty, context, options); + return new MaskResult { Status = status, FailedAtByte = failedAt, Flags = flags }; } /// @@ -421,8 +436,8 @@ public MaskResult Read(in ReadOnlySequence utf8, TContext context, JsonObs options ??= JsonObserverOptions.Default; var reader = CreateReader(SkipBom(utf8), options); - var (status, failedAt) = Observe(ref reader, default, JsonWriter.Empty, context, options); - return new MaskResult { Status = status, FailedAtByte = failedAt }; + var (status, failedAt, flags) = Observe(ref reader, default, JsonWriter.Empty, context, options); + return new MaskResult { Status = status, FailedAtByte = failedAt, Flags = flags }; } private static ReadOnlySpan SkipBom(ReadOnlySpan utf8) => utf8.StartsWith(Utf8Bom) ? utf8[Utf8Bom.Length..] : utf8; @@ -452,7 +467,7 @@ private static Utf8JsonReader CreateReader(ReadOnlySpan utf8, JsonObserver private static Utf8JsonReader CreateReader(in ReadOnlySequence utf8, JsonObserverOptions options) => new(utf8, isFinalBlock: false, ReaderState(options)); - private (MaskStatus Status, long FailedAt) Observe( + private (MaskStatus Status, long FailedAt, MaskFlags Flags) Observe( ref Utf8JsonReader reader, ReadOnlySpan input, JsonWriter writer, @@ -464,18 +479,29 @@ private static Utf8JsonReader CreateReader(in ReadOnlySequence utf8, JsonO { if (!reader.Read() || reader.TokenType is not (JsonTokenType.StartObject or JsonTokenType.StartArray)) { - return (MaskStatus.Unrecognized, 0); + return (MaskStatus.Unrecognized, 0, MaskFlags.None); } _maskDelegate(ref reader, writer, context, 0, ref propPath, JsonValuePolicy.Default.Rule); UpdateMaxDepth(propPath.MaxLength); - return propPath.Stopped || writer.Stopped - ? (MaskStatus.Truncated, reader.BytesConsumed) - : (MaskStatus.Masked, -1); + if (writer.Stopped) + { + return (MaskStatus.Truncated, reader.BytesConsumed, MaskFlags.OutputCapped); + } + + if (propPath.Stopped) + { + return (MaskStatus.Truncated, reader.BytesConsumed, MaskFlags.InputTruncated); + } + + return HasTrailingData(ref reader) + ? (MaskStatus.Truncated, -1, MaskFlags.TrailingData) + : (MaskStatus.Masked, -1, MaskFlags.None); } catch (Exception) { - return (MaskStatus.Invalid, reader.BytesConsumed); + var depth = reader.CurrentDepth >= Math.Max(options.MaxDepth, 1) ? MaskFlags.Depth : MaskFlags.None; + return (MaskStatus.Invalid, reader.BytesConsumed, depth); } finally { @@ -483,6 +509,21 @@ private static Utf8JsonReader CreateReader(in ReadOnlySequence utf8, JsonO } } + /// + /// Whether anything but whitespace follows the root, which the reader reports as a second token or an error. + /// + private static bool HasTrailingData(ref Utf8JsonReader reader) + { + try + { + return reader.Read(); + } + catch (JsonException) + { + return true; + } + } + /// /// Remembers the deepest path seen, so that later calls start with a large enough path buffer. /// From c7a7b2aee5f95b2a3af3f517cd6cb29edf567d21 Mon Sep 17 00:00:00 2001 From: VasiliyF <5789590+vfofanov@users.noreply.github.com> Date: Mon, 5 Oct 2026 10:08:34 +0200 Subject: [PATCH 04/10] Keep, mask or drop JSON comments by policy and rule - the reader allows comments when the policy or a rule may keep one, and skips them otherwise at no cost - Before / Inline / After ownership by line, rules via .Comment(kinds, rule) after any path rule, masked owners write kept comments masked - kept comments are written as block comments and count toward MaxOutputBytes; shape observers follow the policy - the HTTP body logger drops every comment (CommentPolicy.DropAll) --- .../JsonBodyLoggingHandler.cs | 2 +- .../CommentTests.cs | 247 ++++++++++++++++ .../RunCommentTests.cs | 3 + .../BoundedJsonWriter.cs | 37 +++ .../Builders/JsonAnyDepthBuilder.cs | 23 ++ .../Builders/JsonArrayBuilder.cs | 23 ++ .../Builders/JsonObjBuilder.cs | 23 ++ .../CommentRuleScan.cs | 27 ++ .../JsonComments.cs | 270 ++++++++++++++++++ .../JsonObserver.cs | 119 ++++++-- .../JsonObserverItem.cs | 156 +++++++++- .../JsonWalk.cs | 63 +++- .../JsonWriter.cs | 13 + .../ShapeWalker.cs | 140 ++++++++- .../TagMasking.cs | 13 + README.md | 2 +- skills/json-observer-masking/SKILL.md | 2 +- .../migrating-from-1x.md | 2 +- 18 files changed, 1125 insertions(+), 40 deletions(-) create mode 100644 DragoAnt.System.Text.Json.Observer.Tests.Shared/CommentTests.cs create mode 100644 DragoAnt.System.Text.Json.Observer.Tests/RunCommentTests.cs create mode 100644 DragoAnt.System.Text.Json.Observer/CommentRuleScan.cs create mode 100644 DragoAnt.System.Text.Json.Observer/JsonComments.cs diff --git a/DragoAnt.System.Text.Json.Observer.Http/JsonBodyLoggingHandler.cs b/DragoAnt.System.Text.Json.Observer.Http/JsonBodyLoggingHandler.cs index ee3294e..859201b 100644 --- a/DragoAnt.System.Text.Json.Observer.Http/JsonBodyLoggingHandler.cs +++ b/DragoAnt.System.Text.Json.Observer.Http/JsonBodyLoggingHandler.cs @@ -293,7 +293,7 @@ private Body Render(ReadOnlySpan utf8, bool truncated, Type? modelType, Js return new Body("[body withheld]", JsonBodyStatus.Withheld, truncated); } - result = masker.Mask(utf8, output, new JsonObserverOptions { MaxOutputBytes = Math.Max(limit, 1) }); + result = masker.Mask(utf8, output, new JsonObserverOptions { MaxOutputBytes = Math.Max(limit, 1), Comments = CommentPolicy.DropAll }); } catch (Exception ex) { diff --git a/DragoAnt.System.Text.Json.Observer.Tests.Shared/CommentTests.cs b/DragoAnt.System.Text.Json.Observer.Tests.Shared/CommentTests.cs new file mode 100644 index 0000000..84b15be --- /dev/null +++ b/DragoAnt.System.Text.Json.Observer.Tests.Shared/CommentTests.cs @@ -0,0 +1,247 @@ +using System.Buffers; +using System.Text; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; + +namespace DragoAnt.System.Text.Json.Observer.Tests.Shared; + +public abstract class CommentTests +{ + private const string Service = """ + { // service account + "user": "svc-orders", + "password": "hunter2", // rotated 2026-09 + /* timeouts in seconds */ + "timeout": 30 // TODO: lower + } + """; + + private static readonly JsonObserver Plain = JsonObserver.Obj(AnyDepth(b => b.Match("password").Mask(MaskTag.Full), BlockList)); + + private static readonly JsonObserver WithRules = JsonObserver.Obj(AnyDepth(b => b + .Match("password").Mask(MaskTag.Full).Comment(CommentKind.Any, CommentRules.Drop) + .Match("timeout").Unmasked().Comment(CommentKind.Inline, (ref CommentContext c) => + { + if (c.Text.TrimStart(" "u8).StartsWith("TODO"u8)) + { + c.Drop(); + } + else + { + c.Keep(); + } + }), + BlockList)); + + private static string Lines(params string[] lines) => string.Join('\n', lines); + + private static JsonObserverOptions Policy(CommentPolicy policy) => new() { Comments = policy }; + + private static string Parse(string output) + { + using var document = JsonDocument.Parse(output, new JsonDocumentOptions { CommentHandling = JsonCommentHandling.Skip }); + return JsonSerializer.Serialize(document.RootElement); + } + + [Fact] + public void Default_DropsEveryComment() + => Plain.Mask(Service).Should().Be("""{"user":"svc-orders","password":"***","timeout":30}"""); + + [Fact] + public void BlockList_KeepsComments_AndMasksTheMaskedOwners() + => Plain.Mask(Service, Policy(CommentPolicy.BlockList)) + .Should().Be("""{/* service account*/"user":"svc-orders","password":"***"/*****//* timeouts in seconds */,"timeout":30/* TODO: lower*/}"""); + + [Fact] + public void RulesOverrideThePolicy() + => WithRules.Mask(Service, Policy(CommentPolicy.BlockList)) + .Should().Be("""{/* service account*/"user":"svc-orders","password":"***"/* timeouts in seconds */,"timeout":30}"""); + + [Fact] + public void RulesKeepUnderTheDefaultPolicy() + { + var observer = JsonObserver.Obj(AnyDepth(b => b.Match("timeout").Unmasked().Comment(CommentKind.Any, CommentRules.Keep), BlockList)); + + observer.Mask(Service).Should().Be("""{"user":"svc-orders","password":"hunter2"/* timeouts in seconds */,"timeout":30/* TODO: lower*/}"""); + } + + [Fact] + public void MaskedOwner_KeepMasks_RawKeepsClear() + { + const string json = """{"password":"hunter2" /* was hunter1 */}"""; + var keep = JsonObserver.Obj(b => b.Match("password").Mask(MaskTag.Full).Comment(CommentKind.Inline, CommentRules.Keep), BlockList); + var raw = JsonObserver.Obj(b => b.Match("password").Mask(MaskTag.Full).Comment(CommentKind.Inline, CommentRules.Raw), BlockList); + + keep.Mask(json).Should().Be("""{"password":"***"/*****/}"""); + raw.Mask(json).Should().Be("""{"password":"***"/* was hunter1 */}"""); + } + + [Fact] + public void MaskAll_And_MaskWithATag() + { + var json = Lines("""{"a":1 // note""", "}"); + + Plain.Mask(json, Policy(CommentPolicy.MaskAll)).Should().Be("""{"a":1/*****/}"""); + Plain.Mask(json, Policy(CommentPolicy.Mask(MaskTag.Last4))).Should().Be("""{"a":1/*****/}"""); + Plain.Mask(Lines("""{"a":1 // a longer note""", "}"), Policy(CommentPolicy.Mask(MaskTag.Last4))).Should().Be("""{"a":1/****note*/}"""); + } + + [Fact] + public void DropAll_IgnoresRules() + => WithRules.Mask(Service, Policy(CommentPolicy.DropAll)).Should().Be("""{"user":"svc-orders","password":"***","timeout":30}"""); + + [Fact] + public void Replace_WritesTheNewText() + { + var observer = JsonObserver.Obj(b => b.Match("a").Unmasked().Comment(CommentKind.Any, (ref CommentContext c) => c.Replace("checked")), BlockList); + + observer.Mask("""{/* x */"a":1}""").Should().Be("""{/*checked*/"a":1}"""); + } + + [Fact] + public void Ownership_FollowsTheLines() + { + var kinds = new List(); + CommentRule record = (ref CommentContext c) => + { + kinds.Add($"{c.Kind} {c.Owner.ToString()} {Encoding.UTF8.GetString(c.Text).Trim()}"); + c.Drop(); + }; + var observer = JsonObserver.Any( + o => o.Match("a").Unmasked().Comment(CommentKind.Any, record).Match("b").Unmasked().Comment(CommentKind.Any, record), + l => l.Unmasked().Comment(CommentKind.Any, record), + BlockList); + + observer.Mask(""" + { + // before a + "a": /* between */ 1, // inline a + // before b + "b": 2 + // after the last member + } + """, Policy(CommentPolicy.BlockList)); + observer.Mask(Lines("""[1, /* inline 0 */ 2 // inline 1""", "]"), Policy(CommentPolicy.BlockList)); + + kinds.Should().Equal( + "Before a before a", + "Before a between", + "Inline a inline a", + "Before b before b", + "Inline [0] inline 0", + "Inline [1] inline 1"); + } + + [Fact] + public void ContainerAndRootComments_AreAfterAndBefore() + { + var output = Plain.Mask(""" + // head + {"a":{"b":1 + /* end of a */}} // tail + // after + + """, Policy(CommentPolicy.BlockList)); + + output.Should().Be("""/* head*/{"a":{"b":1/* end of a */}}/* tail*//* after*/"""); + Parse(output!).Should().Be("""{"a":{"b":1}}"""); + } + + [Fact] + public void CommentsInArraysAndNestedRules() + { + var observer = JsonObserver.Obj(b => b.Match("lines").Array(l => l.Obj(x => x.Match("qty").Unmasked())), BlockList); + + observer.Mask("""{"lines":[/* first */{"qty":1 /* one */},{"qty":2}]}""", Policy(CommentPolicy.BlockList)) + .Should().Be("""{"lines":[/* first */{"qty":1/* one */},{"qty":2}]}"""); + } + + [Fact] + public void ShapeObserver_FollowsThePolicy() + { + var shape = JsonShape.Object(("id", JsonShape.Scalar), ("secret", JsonShape.Masked(MaskTag.Full))); + var observer = JsonObserver.FromShape(shape); + var json = Lines("""{"id":1, /* the id */ "secret":"x" // was y""", "}"); + + observer.Mask(json).Should().Be("""{"id":1,"secret":"***"}"""); + observer.Mask(json, Policy(CommentPolicy.BlockList)).Should().Be("""{"id":1/* the id */,"secret":"***"/*****/}"""); + } + + [Fact] + public void CommentTerminatorInside_IsDefused() + { + var output = Plain.Mask(Lines("""{"a":1 // x */ y""", "}"), Policy(CommentPolicy.BlockList)); + + output.Should().Be("""{"a":1/* x * / y*/}"""); + Parse(output!).Should().Be("""{"a":1}"""); + } + + [Fact] + public void SequenceInput_SplitAnywhere_SameOutput() + { + var utf8 = Encoding.UTF8.GetBytes(Service); + var options = Policy(CommentPolicy.BlockList); + var expected = Plain.Mask(Service, options); + for (var split = 1; split < utf8.Length; split++) + { + var first = new Segment(utf8.AsMemory(0, split)); + var last = first.Append(utf8.AsMemory(split)); + var output = new ArrayBufferWriter(); + Plain.Mask(new ReadOnlySequence(first, 0, last, last.Memory.Length), output, options); + Encoding.UTF8.GetString(output.WrittenSpan).Should().Be(expected, $"split at {split}"); + } + } + + [Fact] + public void OutputCap_CountsComments_AndStaysParseable() + { + for (var cap = 0; cap < 80; cap++) + { + var output = Plain.Mask(Service, out var result, new JsonObserverOptions { Comments = CommentPolicy.BlockList, MaxOutputBytes = cap })!; + Encoding.UTF8.GetByteCount(output).Should().BeLessThanOrEqualTo(Math.Max(cap, 0) == 0 ? 0 : cap); + if (output.Length > 0) + { + Parse(output).Should().NotBeNull(); + } + + result.Status.Should().BeOneOf(MaskStatus.Masked, MaskStatus.Truncated); + } + } + + [Theory] + [InlineData("""{"password":"hunter2" /* hunter2 */}""")] + [InlineData("""{/* hunter2 */"password":"hunter2"}""")] + [InlineData("""{"password": /* hunter2 */ "hunter2"}""")] + [InlineData("""{"x":{"password":"hunter2"} // hunter2|}""")] + [InlineData("""// hunter2|{"password":"hunter2"}""")] + public void SecretInAComment_NeverLeaksUnderTheDefault(string lines) + { + var json = lines.Replace('|', '\n'); + foreach (var observer in new[] { Plain, WithRules, JsonObserver.Obj(_ => { }) }) + { + observer.Mask(json).Should().NotContain("hunter2"); + observer.Mask(json, Policy(CommentPolicy.MaskAll)).Should().NotContain("hunter2"); + } + } + + [Fact] + public void Read_IgnoresComments() + => JsonObserver.Obj(b => b.Match("a").Unmasked().Comment(CommentKind.Any, CommentRules.Keep)) + .Read("""{/*x*/"a":1}""", NoContext.Instance).Status.Should().Be(MaskStatus.Masked); + + [Fact] + public void Comment_WithoutARule_Throws() + => FluentActions.Invoking(() => JsonObserver.Obj(b => b.Comment(CommentKind.Any, CommentRules.Drop))).Should().Throw(); + + private sealed class Segment : ReadOnlySequenceSegment + { + public Segment(ReadOnlyMemory memory) => Memory = memory; + + public Segment Append(ReadOnlyMemory memory) + { + var next = new Segment(memory) { RunningIndex = RunningIndex + Memory.Length }; + Next = next; + return next; + } + } +} diff --git a/DragoAnt.System.Text.Json.Observer.Tests/RunCommentTests.cs b/DragoAnt.System.Text.Json.Observer.Tests/RunCommentTests.cs new file mode 100644 index 0000000..038dbd2 --- /dev/null +++ b/DragoAnt.System.Text.Json.Observer.Tests/RunCommentTests.cs @@ -0,0 +1,3 @@ +namespace DragoAnt.System.Text.Json.Observer.Tests; + +public sealed class RunCommentTests : Shared.CommentTests; diff --git a/DragoAnt.System.Text.Json.Observer/BoundedJsonWriter.cs b/DragoAnt.System.Text.Json.Observer/BoundedJsonWriter.cs index 911d08c..6ca9611 100644 --- a/DragoAnt.System.Text.Json.Observer/BoundedJsonWriter.cs +++ b/DragoAnt.System.Text.Json.Observer/BoundedJsonWriter.cs @@ -304,6 +304,43 @@ public override void WriteBase64StringValue(ReadOnlySpan bytes) } } + internal override void WriteComment(ReadOnlySpan utf8Text) + { + if (Exhausted) + { + return; + } + + if (utf8Text.IndexOf("*/"u8) < 0) + { + _writer.WriteCommentValue(utf8Text); + } + else + { + var safe = ArrayPool.Shared.Rent(utf8Text.Length * 2); + try + { + var length = 0; + for (var i = 0; i < utf8Text.Length; i++) + { + safe[length++] = utf8Text[i]; + if (utf8Text[i] == (byte)'*' && i + 1 < utf8Text.Length && utf8Text[i + 1] == (byte)'/') + { + safe[length++] = (byte)' '; + } + } + + _writer.WriteCommentValue(safe.AsSpan(0, length)); + } + finally + { + ArrayPool.Shared.Return(safe, clearArray: true); + } + } + + Completed(); + } + public override void WritePropertyName(string propertyName) { if (!Exhausted) diff --git a/DragoAnt.System.Text.Json.Observer/Builders/JsonAnyDepthBuilder.cs b/DragoAnt.System.Text.Json.Observer/Builders/JsonAnyDepthBuilder.cs index 6184fe8..717628e 100644 --- a/DragoAnt.System.Text.Json.Observer/Builders/JsonAnyDepthBuilder.cs +++ b/DragoAnt.System.Text.Json.Observer/Builders/JsonAnyDepthBuilder.cs @@ -30,6 +30,29 @@ public PropertyMaskingStrategyBuilder Match(NameMatch match) => public PropertyMaskingStrategyBuilder Path(params NameMatch[] path) => new(this, new NamePathMatch(path, isPath: true)); + /// + /// Decides the comments of the values the rule just added matches, for the placements in , + /// instead of the call's policy. A kept comment of a masked value is still + /// written masked unless calls . + /// + /// Placements the rule decides, for example . + /// The comment rule, for example . + /// is null. + /// No rule was added yet. + public JsonAnyDepthBuilder Comment(CommentKind kinds, CommentRule rule) + { + ArgumentNullException.ThrowIfNull(rule); + if (_policies.Count == 0) + { + throw new InvalidOperationException("Add a rule before its comment rule."); + } + + var last = _policies[^1]; + last.CommentKinds = kinds; + last.CommentRule = rule; + return this; + } + internal static ObserveRule Build(JsonAnyDepthBuilder builder) => builder.Build(); internal static JsonObserverItem[] BuildItems(JsonAnyDepthBuilder builder) => [.. builder._policies]; diff --git a/DragoAnt.System.Text.Json.Observer/Builders/JsonArrayBuilder.cs b/DragoAnt.System.Text.Json.Observer/Builders/JsonArrayBuilder.cs index e76cd2d..0b77214 100644 --- a/DragoAnt.System.Text.Json.Observer/Builders/JsonArrayBuilder.cs +++ b/DragoAnt.System.Text.Json.Observer/Builders/JsonArrayBuilder.cs @@ -127,6 +127,29 @@ public JsonArrayBuilder Unmasked() => internal JsonArrayBuilder MaskWhole(ObserveRule policy, string action, bool keepsNull = false) => Add(_ => true, policy, new RuleInfo(AnyItem, action, PathOutcome.Masked, KeepsNull: keepsNull)); + /// + /// Decides the comments of the values the rule just added matches, for the placements in , + /// instead of the call's policy. A kept comment of a masked value is still + /// written masked unless calls . + /// + /// Placements the rule decides, for example . + /// The comment rule, for example . + /// is null. + /// No rule was added yet. + public JsonArrayBuilder Comment(CommentKind kinds, CommentRule rule) + { + ArgumentNullException.ThrowIfNull(rule); + if (_policies.Count == 0) + { + throw new InvalidOperationException("Add a rule before its comment rule."); + } + + var last = _policies[^1]; + last.CommentKinds = kinds; + last.CommentRule = rule; + return this; + } + internal static (ObserveRule Delegate, RuleSet Set) Build(JsonArrayBuilder builder) => builder.Build(); private JsonArrayBuilder Read(JsonObserverItem.ReadValue read, string action) diff --git a/DragoAnt.System.Text.Json.Observer/Builders/JsonObjBuilder.cs b/DragoAnt.System.Text.Json.Observer/Builders/JsonObjBuilder.cs index f465551..23b2e78 100644 --- a/DragoAnt.System.Text.Json.Observer/Builders/JsonObjBuilder.cs +++ b/DragoAnt.System.Text.Json.Observer/Builders/JsonObjBuilder.cs @@ -31,6 +31,29 @@ public PropertyMaskingStrategyBuilder Match(NameMatch match) => public PropertyMaskingStrategyBuilder Path(params NameMatch[] path) => new(this, new NamePathMatch(path, isPath: true), _builderDefaultValuePolicy); + /// + /// Decides the comments of the values the rule just added matches, for the placements in , + /// instead of the call's policy. A kept comment of a masked value is still + /// written masked unless calls . + /// + /// Placements the rule decides, for example . + /// The comment rule, for example . + /// is null. + /// No rule was added yet. + public JsonObjBuilder Comment(CommentKind kinds, CommentRule rule) + { + ArgumentNullException.ThrowIfNull(rule); + if (_policies.Count == 0) + { + throw new InvalidOperationException("Add a rule before its comment rule."); + } + + var last = _policies[^1]; + last.CommentKinds = kinds; + last.CommentRule = rule; + return this; + } + internal static (ObserveRule Delegate, RuleSet Set) Build(JsonObjBuilder builder) => builder.Build(); private (ObserveRule, RuleSet) Build() diff --git a/DragoAnt.System.Text.Json.Observer/CommentRuleScan.cs b/DragoAnt.System.Text.Json.Observer/CommentRuleScan.cs new file mode 100644 index 0000000..b22ed10 --- /dev/null +++ b/DragoAnt.System.Text.Json.Observer/CommentRuleScan.cs @@ -0,0 +1,27 @@ +namespace DragoAnt.System.Text.Json.Observer; + +/// +/// Finds whether an observer's rules decide comments anywhere, so that a call can skip comments when none would be written. +/// +internal static class CommentRuleScan +{ + public static bool Any(RuleSet? set) => set is not null && Any(set.Items, set.ValuePolicy, 0); + + private static bool Any(JsonObserverItem[] items, JsonValuePolicy? policy, int depth) + { + if (depth > 64) + { + return false; + } + + foreach (var item in items) + { + if (item.CommentRule is not null || (item.Info.Child is { } child && Any(child.Items, child.ValuePolicy, depth + 1))) + { + return true; + } + } + + return policy?.Relative is { } relative && Any(relative.Items, relative.Fallback, depth + 1); + } +} diff --git a/DragoAnt.System.Text.Json.Observer/JsonComments.cs b/DragoAnt.System.Text.Json.Observer/JsonComments.cs new file mode 100644 index 0000000..2704e1e --- /dev/null +++ b/DragoAnt.System.Text.Json.Observer/JsonComments.cs @@ -0,0 +1,270 @@ +using System.Buffers; +using System.Text; + +namespace DragoAnt.System.Text.Json.Observer; + +/// +/// Comments read but not yet written: those before a member wait until its rule is known. +/// +internal sealed class CommentBuffer +{ + [ThreadStatic] + private static CommentBuffer? t_cached; + + private byte[] _bytes = []; + private Entry[] _entries = []; + private int _used; + + public int Count { get; private set; } + + public static CommentBuffer Rent() + { + var buffer = t_cached ?? new CommentBuffer(); + t_cached = null; + return buffer; + } + + public void Return() + { + Clear(); + if (_bytes.Length > 0) + { + ArrayPool.Shared.Return(_bytes, clearArray: true); + _bytes = []; + } + + t_cached = this; + } + + public void Add(ref Utf8JsonReader reader, CommentStyle style) + { + var length = reader.HasValueSequence ? checked((int)reader.ValueSequence.Length) : reader.ValueSpan.Length; + if (_used + length > _bytes.Length) + { + var grown = ArrayPool.Shared.Rent(Math.Max(_used + length, Math.Max(_bytes.Length * 2, 256))); + _bytes.AsSpan(0, _used).CopyTo(grown); + if (_bytes.Length > 0) + { + ArrayPool.Shared.Return(_bytes, clearArray: true); + } + + _bytes = grown; + } + + if (reader.HasValueSequence) + { + reader.ValueSequence.CopyTo(_bytes.AsSpan(_used)); + } + else + { + reader.ValueSpan.CopyTo(_bytes.AsSpan(_used)); + } + + if (Count == _entries.Length) + { + global::System.Array.Resize(ref _entries, Math.Max(4, _entries.Length * 2)); + } + + _entries[Count++] = new Entry(_used, length, style); + _used += length; + } + + public ReadOnlySpan Text(int index) => _bytes.AsSpan(_entries[index].Start, _entries[index].Length); + + public CommentStyle Style(int index) => _entries[index].Style; + + public void Clear() + { + Count = 0; + _used = 0; + } + + public void Truncate(int count) + { + _used = count == 0 ? 0 : _entries[count].Start; + Count = count; + } + + private readonly record struct Entry(int Start, int Length, CommentStyle Style); +} + +/// +/// Decides and writes comments for the JSON walkers, following the call's and the comment +/// rules of the owners' path rules. +/// +internal static class JsonComments +{ + /// + /// Handles a comment token inside a container: written at once as when it stands on + /// the line of the value just written (whose member is still on the path), otherwise kept for the next member. + /// + public static void OnComment( + ref Utf8JsonReader reader, + JsonWriter writer, + ref JsonWalk walk, + bool previousOpen, + JsonObserverItem? previousItem, + bool previousMasked) + { + var style = walk.StyleAt(reader.TokenStartIndex); + if (previousOpen && walk.IsSameLine(walk.LastValueEnd, reader.TokenStartIndex)) + { + var pending = walk.Pending!; + var mark = pending.Count; + pending.Add(ref reader, style); + Write(pending, mark, writer, ref walk, CommentKind.Inline, previousItem?.CommentRuleFor(CommentKind.Inline), previousMasked); + return; + } + + walk.Pending!.Add(ref reader, style); + } + + /// + /// Writes and clears the kept comments as of the member at the end of the path. + /// + public static void Flush(JsonWriter writer, ref JsonWalk walk, CommentKind kind, CommentRule? rule, bool ownerMasked) + { + var pending = walk.Pending; + if (pending is null || pending.Count == 0) + { + return; + } + + Write(pending, 0, writer, ref walk, kind, rule, ownerMasked); + } + + private static void Write(CommentBuffer pending, int from, JsonWriter writer, ref JsonWalk walk, CommentKind kind, CommentRule? rule, bool ownerMasked) + { + var policy = walk.Comments!; + for (var i = from; i < pending.Count; i++) + { + var text = pending.Text(i); + var context = new CommentContext(text, kind, pending.Style(i), in walk.Path, ownerMasked); + policy.Decide(ref context, rule); + switch (context.Action) + { + case CommentAction.Keep: + case CommentAction.Raw: + writer.WriteComment(text); + break; + case CommentAction.Mask: + WriteMasked(text, context.MaskTag, writer, ref walk); + break; + case CommentAction.Replace: + WriteReplacement(context.Replacement ?? string.Empty, writer); + break; + } + } + + if (from == 0) + { + pending.Clear(); + } + else + { + pending.Truncate(from); + } + } + + private static void WriteReplacement(string replacement, JsonWriter writer) + { + var bytes = ArrayPool.Shared.Rent(Encoding.UTF8.GetMaxByteCount(replacement.Length)); + try + { + writer.WriteComment(bytes.AsSpan(0, Encoding.UTF8.GetBytes(replacement, bytes))); + } + finally + { + ArrayPool.Shared.Return(bytes, clearArray: true); + } + } + + private static void WriteMasked(ReadOnlySpan text, MaskTag tag, JsonWriter writer, ref JsonWalk walk) + { + var options = walk.Options; + var strategy = options.Strategy ?? ValueMaskStrategy.Default; + var output = CommentTextWriter.Rent(text); + try + { + strategy.Mask(new MaskContext(text, ValueKind.String, tag, options, in walk.Path, walk.ValueIndex++), output); + if (output.HasText) + { + writer.WriteComment(output.Text); + } + } + finally + { + output.Return(); + } + } + + /// + /// Collects a strategy's replacement of a comment's text. + /// + private sealed class CommentTextWriter : MaskValueWriter + { + [ThreadStatic] + private static CommentTextWriter? t_cached; + + private byte[] _buffer = []; + private byte[] _original = []; + private int _length; + private int _originalLength; + + public bool HasText { get; private set; } + + public ReadOnlySpan Text => _buffer.AsSpan(0, _length); + + public static CommentTextWriter Rent(ReadOnlySpan original) + { + var writer = t_cached ?? new CommentTextWriter(); + t_cached = null; + writer.HasText = false; + writer._length = 0; + writer._originalLength = original.Length; + writer.Ensure(ref writer._original, original.Length); + original.CopyTo(writer._original); + return writer; + } + + public void Return() + { + _buffer.AsSpan(0, _length).Clear(); + _original.AsSpan(0, _originalLength).Clear(); + t_cached = this; + } + + public override void String(ReadOnlySpan utf8) => Set(utf8); + + public override void String(ReadOnlySpan chars) + { + Ensure(ref _buffer, Encoding.UTF8.GetMaxByteCount(chars.Length)); + _length = Encoding.UTF8.GetBytes(chars, _buffer); + HasText = true; + } + + public override void Boolean(bool value) => Set(value ? "true"u8 : "false"u8); + + public override void Null() => HasText = false; + + public override void Keep() => Set(_original.AsSpan(0, _originalLength)); + + protected override void WriteNumber(ReadOnlySpan utf8Literal) => Set(utf8Literal); + + private void Set(ReadOnlySpan utf8) + { + Ensure(ref _buffer, utf8.Length); + utf8.CopyTo(_buffer); + _length = utf8.Length; + HasText = true; + } + + private void Ensure(ref byte[] buffer, int length) + { + if (buffer.Length < length) + { + buffer = new byte[Math.Max(length, 64)]; + } + } + } +} diff --git a/DragoAnt.System.Text.Json.Observer/JsonObserver.cs b/DragoAnt.System.Text.Json.Observer/JsonObserver.cs index ed64c5b..c13464f 100644 --- a/DragoAnt.System.Text.Json.Observer/JsonObserver.cs +++ b/DragoAnt.System.Text.Json.Observer/JsonObserver.cs @@ -76,7 +76,7 @@ public static JsonObserver Any( JsonValuePolicy? defaultMasking = null) { var (masking, obj, array) = JsonObserverItem.Any(initObj, initArray, defaultMasking); - return new JsonObserver(masking, new RuleExplainer(obj, array)); + return new JsonObserver(masking, new RuleExplainer(obj, array), CommentRuleScan.Any(obj) || CommentRuleScan.Any(array)); } /// @@ -98,7 +98,7 @@ public static JsonObserver Obj( JsonValuePolicy? defaultMasking = null) { var (masking, set) = JsonObserverItem.Obj(init, defaultMasking); - return new JsonObserver(masking, new RuleExplainer(set, null)); + return new JsonObserver(masking, new RuleExplainer(set, null), CommentRuleScan.Any(set)); } /// @@ -120,7 +120,7 @@ public static JsonObserver Array( JsonValuePolicy? defaultMasking = null) { var (masking, set) = JsonObserverItem.Array(init, defaultMasking); - return new JsonObserver(masking, new RuleExplainer(null, set)); + return new JsonObserver(masking, new RuleExplainer(null, set), CommentRuleScan.Any(set)); } /// @@ -134,7 +134,7 @@ public static JsonObserver FromShape(JsonShape shape, JsonShapeOptions? options { ArgumentNullException.ThrowIfNull(shape); var walker = new ShapeWalker(shape, options ?? JsonShapeOptions.Default); - return new JsonObserver(new JsonObserver(walker.Invoke, walker)); + return new JsonObserver(new JsonObserver(walker.Invoke, walker, hasCommentRules: false)); } private JsonObserver(JsonObserver masking) @@ -209,8 +209,11 @@ public sealed class JsonObserver private readonly PathExplainer _explainer; private int _maxDepth = 6; - internal JsonObserver(ObserveRule maskDelegate, PathExplainer explainer) + private readonly bool _hasCommentRules; + + internal JsonObserver(ObserveRule maskDelegate, PathExplainer explainer, bool hasCommentRules) { + _hasCommentRules = hasCommentRules; _maskDelegate = maskDelegate; _explainer = explainer; } @@ -299,8 +302,9 @@ public MaskResult Mask(ReadOnlySpan utf8, IBufferWriter output, TCon { options ??= JsonObserverOptions.Default; utf8 = SkipBom(utf8); - var reader = CreateReader(utf8, options); - return Mask(ref reader, utf8, output, context, options); + var comments = CommentsOf(options); + var reader = CreateReader(utf8, options, comments is not null); + return Mask(ref reader, utf8, default, output, context, options, comments); } /// @@ -322,15 +326,24 @@ public MaskResult Mask(in ReadOnlySequence utf8, IBufferWriter outpu } options ??= JsonObserverOptions.Default; - var reader = CreateReader(SkipBom(utf8), options); - return Mask(ref reader, default, output, context, options); + var comments = CommentsOf(options); + var input = SkipBom(utf8); + var reader = CreateReader(input, options, comments is not null); + return Mask(ref reader, default, input, output, context, options, comments); } - private MaskResult Mask(ref Utf8JsonReader reader, ReadOnlySpan input, IBufferWriter output, TContext context, JsonObserverOptions options) + private MaskResult Mask( + ref Utf8JsonReader reader, + ReadOnlySpan input, + in ReadOnlySequence sequence, + IBufferWriter output, + TContext context, + JsonObserverOptions options, + CommentPolicy? comments) { using var bounded = BoundedJsonWriter.Rent(options); using var ignoreNulls = options.IgnoreNulls ? IgnoreNullsJsonWriter.Rent(bounded) : null; - var (status, failedAt, flags) = Observe(ref reader, input, (JsonWriter?)ignoreNulls ?? bounded, context, options); + var (status, failedAt, flags) = Observe(ref reader, input, sequence, (JsonWriter?)ignoreNulls ?? bounded, context, options, comments); if (status == MaskStatus.Unrecognized) { return new MaskResult { Status = MaskStatus.Unrecognized }; @@ -413,8 +426,8 @@ public MaskResult Read(ReadOnlySpan utf8, TContext context, JsonObserverOp { options ??= JsonObserverOptions.Default; utf8 = SkipBom(utf8); - var reader = CreateReader(utf8, options); - var (status, failedAt, flags) = Observe(ref reader, utf8, JsonWriter.Empty, context, options); + var reader = CreateReader(utf8, options, comments: false); + var (status, failedAt, flags) = Observe(ref reader, utf8, default, JsonWriter.Empty, context, options, null); return new MaskResult { Status = status, FailedAtByte = failedAt, Flags = flags }; } @@ -435,8 +448,8 @@ public MaskResult Read(in ReadOnlySequence utf8, TContext context, JsonObs } options ??= JsonObserverOptions.Default; - var reader = CreateReader(SkipBom(utf8), options); - var (status, failedAt, flags) = Observe(ref reader, default, JsonWriter.Empty, context, options); + var reader = CreateReader(SkipBom(utf8), options, comments: false); + var (status, failedAt, flags) = Observe(ref reader, default, default, JsonWriter.Empty, context, options, null); return new MaskResult { Status = status, FailedAtByte = failedAt, Flags = flags }; } @@ -454,34 +467,51 @@ private static ReadOnlySequence SkipBom(in ReadOnlySequence utf8) return head.SequenceEqual(Utf8Bom) ? utf8.Slice(Utf8Bom.Length) : utf8; } - private static JsonReaderState ReaderState(JsonObserverOptions options) => new(new JsonReaderOptions + /// + /// The comment policy of a writing call, or null when every comment would be dropped anyway, so that the + /// reader skips them at no cost. + /// + private CommentPolicy? CommentsOf(JsonObserverOptions options) { - CommentHandling = JsonCommentHandling.Skip, + var policy = options.Comments ?? CommentPolicy.AllowList; + return policy.Kind == CommentPolicyKind.DropAll || (policy.Kind == CommentPolicyKind.AllowList && !_hasCommentRules) ? null : policy; + } + + private static JsonReaderState ReaderState(JsonObserverOptions options, bool comments) => new(new JsonReaderOptions + { + CommentHandling = comments ? JsonCommentHandling.Allow : JsonCommentHandling.Skip, AllowTrailingCommas = true, MaxDepth = Math.Max(options.MaxDepth, 1), }); - private static Utf8JsonReader CreateReader(ReadOnlySpan utf8, JsonObserverOptions options) => - new(utf8, isFinalBlock: false, ReaderState(options)); + private static Utf8JsonReader CreateReader(ReadOnlySpan utf8, JsonObserverOptions options, bool comments) => + new(utf8, isFinalBlock: false, ReaderState(options, comments)); - private static Utf8JsonReader CreateReader(in ReadOnlySequence utf8, JsonObserverOptions options) => - new(utf8, isFinalBlock: false, ReaderState(options)); + private static Utf8JsonReader CreateReader(in ReadOnlySequence utf8, JsonObserverOptions options, bool comments) => + new(utf8, isFinalBlock: false, ReaderState(options, comments)); private (MaskStatus Status, long FailedAt, MaskFlags Flags) Observe( ref Utf8JsonReader reader, ReadOnlySpan input, + in ReadOnlySequence sequence, JsonWriter writer, TContext context, - JsonObserverOptions options) + JsonObserverOptions options, + CommentPolicy? comments) { - var propPath = new JsonWalk(_maxDepth, input, options); + var propPath = new JsonWalk(_maxDepth, input, sequence, options, comments); try { - if (!reader.Read() || reader.TokenType is not (JsonTokenType.StartObject or JsonTokenType.StartArray)) + if (!ReadRoot(ref reader, ref propPath) || reader.TokenType is not (JsonTokenType.StartObject or JsonTokenType.StartArray)) { return (MaskStatus.Unrecognized, 0, MaskFlags.None); } + if (comments is not null) + { + JsonComments.Flush(writer, ref propPath, CommentKind.Before, null, ownerMasked: false); + } + _maskDelegate(ref reader, writer, context, 0, ref propPath, JsonValuePolicy.Default.Rule); UpdateMaxDepth(propPath.MaxLength); if (writer.Stopped) @@ -494,13 +524,13 @@ private static Utf8JsonReader CreateReader(in ReadOnlySequence utf8, JsonO return (MaskStatus.Truncated, reader.BytesConsumed, MaskFlags.InputTruncated); } - return HasTrailingData(ref reader) + return HasTrailingData(ref reader, writer, ref propPath) ? (MaskStatus.Truncated, -1, MaskFlags.TrailingData) : (MaskStatus.Masked, -1, MaskFlags.None); } catch (Exception) { - var depth = reader.CurrentDepth >= Math.Max(options.MaxDepth, 1) ? MaskFlags.Depth : MaskFlags.None; + var depth = reader.CurrentDepth >= Math.Max(options.MaxDepth, 1) - 1 ? MaskFlags.Depth : MaskFlags.None; return (MaskStatus.Invalid, reader.BytesConsumed, depth); } finally @@ -512,11 +542,26 @@ private static Utf8JsonReader CreateReader(in ReadOnlySequence utf8, JsonO /// /// Whether anything but whitespace follows the root, which the reader reports as a second token or an error. /// - private static bool HasTrailingData(ref Utf8JsonReader reader) + private static bool HasTrailingData(ref Utf8JsonReader reader, JsonWriter writer, ref JsonWalk walk) { + var end = reader.BytesConsumed; + var inline = true; try { - return reader.Read(); + while (reader.Read()) + { + if (reader.TokenType != JsonTokenType.Comment) + { + return true; + } + + inline = inline && walk.IsSameLine(end, reader.TokenStartIndex); + var kind = inline ? CommentKind.Inline : CommentKind.After; + walk.Pending!.Add(ref reader, walk.StyleAt(reader.TokenStartIndex)); + JsonComments.Flush(writer, ref walk, kind, null, ownerMasked: false); + } + + return false; } catch (JsonException) { @@ -524,6 +569,24 @@ private static bool HasTrailingData(ref Utf8JsonReader reader) } } + /// + /// Reads the root token, keeping the comments before it. + /// + private static bool ReadRoot(ref Utf8JsonReader reader, ref JsonWalk walk) + { + while (reader.Read()) + { + if (reader.TokenType != JsonTokenType.Comment) + { + return true; + } + + walk.Pending!.Add(ref reader, walk.StyleAt(reader.TokenStartIndex)); + } + + return false; + } + /// /// Remembers the deepest path seen, so that later calls start with a large enough path buffer. /// diff --git a/DragoAnt.System.Text.Json.Observer/JsonObserverItem.cs b/DragoAnt.System.Text.Json.Observer/JsonObserverItem.cs index 55511a0..487a2b8 100644 --- a/DragoAnt.System.Text.Json.Observer/JsonObserverItem.cs +++ b/DragoAnt.System.Text.Json.Observer/JsonObserverItem.cs @@ -18,6 +18,18 @@ internal sealed class JsonObserverItem(JsonPropertyMatchDelegate propM /// public RuleInfo Info { get; init; } = RuleInfo.Unknown; + /// + /// Placements whose comments decides for the members this rule matches. + /// + public CommentKind CommentKinds { get; set; } + + /// + /// The rule's comment rule, set with .Comment(kinds, rule); null leaves comments to the call's policy. + /// + public CommentRule? CommentRule { get; set; } + + public CommentRule? CommentRuleFor(CommentKind kind) => (CommentKinds & kind) != 0 ? CommentRule : null; + /// /// Any payload object or array. /// @@ -410,21 +422,39 @@ private static ObserveRule ApplyObjPolicy( RuntimeHelpers.EnsureSufficientExecutionStack(); writer.WriteStartObject(); + var comments = propPath.Comments is not null; + var previousOpen = false; + JsonObserverItem? previousItem = null; + var previousMasked = false; while (true) { if (propPath.Stopped || writer.Stopped || !reader.Read()) { + if (previousOpen) + { + propPath.RemovePropertyName(); + } + propPath.Stop(); return; } switch (reader.TokenType) { + case Comment: + JsonComments.OnComment(ref reader, writer, ref propPath, previousOpen, previousItem, previousMasked); + break; case PropertyName: + if (previousOpen) + { + propPath.RemovePropertyName(); + previousOpen = false; + } + propPath.AddPropertyName(ref reader); - if (!reader.Read()) + if (!ReadMemberValue(ref reader, ref propPath)) { propPath.RemovePropertyName(); propPath.Stop(); @@ -438,6 +468,11 @@ private static ObserveRule ApplyObjPolicy( } var (matchPolicy, nextDepth) = MatchPolicy(policies, depth, ref propPath, tokenType); + if (comments) + { + (previousMasked, previousItem) = CommentOwner(matchPolicy, effective, tokenType, ref propPath); + JsonComments.Flush(writer, ref propPath, CommentKind.Before, previousItem?.CommentRuleFor(CommentKind.Before), previousMasked); + } writer.WritePropertyName(propPath.CurrentUtf8); if (matchPolicy is not null) @@ -449,10 +484,28 @@ private static ObserveRule ApplyObjPolicy( defaultPolicy(ref reader, writer, context, nextDepth, ref propPath, effective); } - propPath.RemovePropertyName(); + if (comments) + { + propPath.LastValueEnd = reader.BytesConsumed; + previousOpen = true; + } + else + { + propPath.RemovePropertyName(); + } break; case EndObject: + if (previousOpen) + { + propPath.RemovePropertyName(); + } + + if (comments) + { + JsonComments.Flush(writer, ref propPath, CommentKind.After, null, ownerMasked: false); + } + writer.WriteEndObject(); return; case None: @@ -495,18 +548,30 @@ private static ObserveRule ApplyArrayPolicy( RuntimeHelpers.EnsureSufficientExecutionStack(); writer.WriteStartArray(); + var comments = propPath.Comments is not null; + var previousOpen = false; + JsonObserverItem? previousItem = null; + var previousMasked = false; var index = 0; while (true) { if (propPath.Stopped || writer.Stopped || !reader.Read()) { + if (previousOpen) + { + propPath.RemovePropertyName(); + } + propPath.Stop(); return; } switch (reader.TokenType) { + case Comment: + JsonComments.OnComment(ref reader, writer, ref propPath, previousOpen, previousItem, previousMasked); + break; case StartObject: case StartArray: case JsonTokenType.String: @@ -514,6 +579,12 @@ private static ObserveRule ApplyArrayPolicy( case True: case False: case Null: + if (previousOpen) + { + propPath.RemovePropertyName(); + previousOpen = false; + } + var tokenType = reader.TokenType; propPath.AddArrayItem(index++); @@ -523,6 +594,11 @@ private static ObserveRule ApplyArrayPolicy( } var (matchPolicy, nextDepth) = MatchPolicy(policies, depth, ref propPath, tokenType); + if (comments) + { + (previousMasked, previousItem) = CommentOwner(matchPolicy, effective, tokenType, ref propPath); + JsonComments.Flush(writer, ref propPath, CommentKind.Before, previousItem?.CommentRuleFor(CommentKind.Before), previousMasked); + } if (matchPolicy is not null) { @@ -532,9 +608,29 @@ private static ObserveRule ApplyArrayPolicy( { defaultPolicy(ref reader, writer, context, nextDepth, ref propPath, effective); } - propPath.RemovePropertyName(); + + if (comments) + { + propPath.LastValueEnd = reader.BytesConsumed; + previousOpen = true; + } + else + { + propPath.RemovePropertyName(); + } + break; case EndArray: + if (previousOpen) + { + propPath.RemovePropertyName(); + } + + if (comments) + { + JsonComments.Flush(writer, ref propPath, CommentKind.After, null, ownerMasked: false); + } + writer.WriteEndArray(); return; case PropertyName: @@ -547,6 +643,60 @@ private static ObserveRule ApplyArrayPolicy( }; } + /// + /// Moves from a property name to its value, keeping the comments in between for the member. + /// + private static bool ReadMemberValue(ref Utf8JsonReader reader, ref JsonWalk walk) + { + while (reader.Read()) + { + if (reader.TokenType != Comment) + { + return true; + } + + walk.Pending!.Add(ref reader, walk.StyleAt(reader.TokenStartIndex)); + } + + return false; + } + + /// + /// Whether the member's value is masked, and the rule whose comment rules apply to its comments. + /// + internal static (bool Masked, JsonObserverItem? RuleItem) CommentOwner( + JsonObserverItem? item, + ValueRule effective, + JsonTokenType tokenType, + ref JsonWalk walk) + { + if (item is not null) + { + var masked = item.Info.Outcome switch + { + PathOutcome.Masked => !(tokenType is Null && item.Info.KeepsNull), + PathOutcome.Custom => true, + _ => false, + }; + return (masked, item); + } + + if (effective.Target is RelativeValuePolicy relative) + { + var (relativeItem, _) = MatchPolicy(relative.Items, 0, ref walk, tokenType); + return relativeItem is not null + ? CommentOwner(relativeItem, relative.Fallback.Rule, tokenType, ref walk) + : CommentOwner(null, relative.Fallback.Rule, tokenType, ref walk); + } + + if (tokenType is StartObject or StartArray or Null) + { + return (false, null); + } + + return (!ReferenceEquals(effective, BuiltInPolicies.BlockList), null); + } + private static ObserveRule GetApplyDefaultPolicy(ValueRule? valuePolicy, UnknownContainers unknown) { return ( diff --git a/DragoAnt.System.Text.Json.Observer/JsonWalk.cs b/DragoAnt.System.Text.Json.Observer/JsonWalk.cs index 4e836db..fdfd7f0 100644 --- a/DragoAnt.System.Text.Json.Observer/JsonWalk.cs +++ b/DragoAnt.System.Text.Json.Observer/JsonWalk.cs @@ -1,3 +1,5 @@ +using System.Buffers; + namespace DragoAnt.System.Text.Json.Observer; /// @@ -6,15 +8,70 @@ namespace DragoAnt.System.Text.Json.Observer; internal ref struct JsonWalk { public DataPath Path; + private readonly ReadOnlySpan _input; + private readonly ReadOnlySequence _sequence; private readonly bool _hasInput; public JsonWalk(int capacity, ReadOnlySpan input, JsonObserverOptions options) { Path = new DataPath(input, capacity, options.NameCaseInsensitive); Options = options; + _input = input; _hasInput = !input.IsEmpty; } + public JsonWalk(int capacity, ReadOnlySpan input, scoped in ReadOnlySequence sequence, JsonObserverOptions options, CommentPolicy? comments) + : this(capacity, input, options) + { + _sequence = sequence; + Comments = comments; + Pending = comments is null ? null : CommentBuffer.Rent(); + } + + /// + /// The comment policy of a pass that writes comments; null when the reader skips them. + /// + public CommentPolicy? Comments { get; } + + public CommentBuffer? Pending { get; } + + /// + /// Input offset just after the last value written, to tell an inline comment from one on a later line. + /// + public long LastValueEnd { get; set; } + + public readonly bool IsSameLine(long from, long to) + { + if (to <= from) + { + return true; + } + + if (!_input.IsEmpty) + { + return _input.Slice((int)from, (int)(to - from)).IndexOfAny((byte)'\n', (byte)'\r') < 0; + } + + var slice = _sequence.Slice(from, to - from); + return slice.PositionOf((byte)'\n') is null && slice.PositionOf((byte)'\r') is null; + } + + public readonly CommentStyle StyleAt(long tokenStart) + { + var at = tokenStart + 1; + byte marker; + if (!_input.IsEmpty) + { + marker = at < _input.Length ? _input[(int)at] : (byte)'*'; + } + else + { + marker = at < _sequence.Length ? _sequence.Slice(at, 1).FirstSpan[0] : (byte)'*'; + } + + return marker == (byte)'/' ? CommentStyle.Line : CommentStyle.Block; + } + public JsonObserverOptions Options { get; } /// @@ -57,5 +114,9 @@ public void AddPropertyName(ref Utf8JsonReader reader) public void RemovePropertyName() => Path.Pop(); - public void Dispose() => Path.Dispose(); + public void Dispose() + { + Path.Dispose(); + Pending?.Return(); + } } diff --git a/DragoAnt.System.Text.Json.Observer/JsonWriter.cs b/DragoAnt.System.Text.Json.Observer/JsonWriter.cs index f55e18f..29fd4b8 100644 --- a/DragoAnt.System.Text.Json.Observer/JsonWriter.cs +++ b/DragoAnt.System.Text.Json.Observer/JsonWriter.cs @@ -133,6 +133,13 @@ internal virtual bool MaskOutput set { } } + /// + /// Writes a comment with the given UTF-8 text, for the comment policy; a writer that has no comments ignores it. + /// + internal virtual void WriteComment(ReadOnlySpan utf8Text) + { + } + internal void CopyStringValue(ref Utf8JsonReader reader) { if (ReferenceEquals(this, Empty)) @@ -360,6 +367,12 @@ public override void WriteBase64StringValue(ReadOnlySpan bytes) _inner.WriteBase64StringValue(bytes); } + internal override void WriteComment(ReadOnlySpan utf8Text) + { + Flush(); + _inner.WriteComment(utf8Text); + } + public override void WriteNumberValue(double value) { Flush(); diff --git a/DragoAnt.System.Text.Json.Observer/ShapeWalker.cs b/DragoAnt.System.Text.Json.Observer/ShapeWalker.cs index d6ccbb7..5e342d4 100644 --- a/DragoAnt.System.Text.Json.Observer/ShapeWalker.cs +++ b/DragoAnt.System.Text.Json.Observer/ShapeWalker.cs @@ -90,33 +90,75 @@ private void WriteObject(ref Utf8JsonReader reader, JsonWriter writer, ref JsonW { RuntimeHelpers.EnsureSufficientExecutionStack(); writer.WriteStartObject(); + var comments = propPath.Comments is not null; + var previousOpen = false; + var previousMasked = false; while (true) { if (propPath.Stopped || writer.Stopped || !reader.Read()) { + if (previousOpen) + { + propPath.RemovePropertyName(); + } + propPath.Stop(); return; } switch (reader.TokenType) { + case Comment: + JsonComments.OnComment(ref reader, writer, ref propPath, previousOpen, null, previousMasked); + break; case EndObject: + if (previousOpen) + { + propPath.RemovePropertyName(); + } + + if (comments) + { + JsonComments.Flush(writer, ref propPath, CommentKind.After, null, ownerMasked: false); + } + writer.WriteEndObject(); return; case PropertyName: + if (previousOpen) + { + propPath.RemovePropertyName(); + previousOpen = false; + } + propPath.AddPropertyName(ref reader); var name = propPath.CurrentUtf8; var child = values ?? shape!.Find(name, _ignoreCase ?? propPath.Options.NameCaseInsensitive) ?? _unknown; - if (!reader.Read()) + if (!ReadMemberValue(ref reader, ref propPath)) { propPath.RemovePropertyName(); propPath.Stop(); return; } - writer.WritePropertyName(name); + if (comments) + { + previousMasked = IsMasked(child, reader.TokenType); + JsonComments.Flush(writer, ref propPath, CommentKind.Before, null, previousMasked); + } + + writer.WritePropertyName(propPath.CurrentUtf8); Write(ref reader, writer, ref propPath, child); - propPath.RemovePropertyName(); + if (comments) + { + propPath.LastValueEnd = reader.BytesConsumed; + previousOpen = true; + } + else + { + propPath.RemovePropertyName(); + } + break; } } @@ -126,11 +168,19 @@ private void WriteArray(ref Utf8JsonReader reader, JsonWriter writer, ref JsonWa { RuntimeHelpers.EnsureSufficientExecutionStack(); writer.WriteStartArray(); + var comments = propPath.Comments is not null; + var previousOpen = false; + var previousMasked = false; var index = 0; while (true) { if (propPath.Stopped || writer.Stopped || !reader.Read()) { + if (previousOpen) + { + propPath.RemovePropertyName(); + } + propPath.Stop(); return; } @@ -138,19 +188,101 @@ private void WriteArray(ref Utf8JsonReader reader, JsonWriter writer, ref JsonWa switch (reader.TokenType) { case EndArray: + if (previousOpen) + { + propPath.RemovePropertyName(); + } + + if (comments) + { + JsonComments.Flush(writer, ref propPath, CommentKind.After, null, ownerMasked: false); + } + writer.WriteEndArray(); return; case Comment: + JsonComments.OnComment(ref reader, writer, ref propPath, previousOpen, null, previousMasked); break; default: + if (previousOpen) + { + propPath.RemovePropertyName(); + previousOpen = false; + } + propPath.AddArrayItem(index++); + if (comments) + { + previousMasked = IsMasked(item, reader.TokenType); + JsonComments.Flush(writer, ref propPath, CommentKind.Before, null, previousMasked); + } + Write(ref reader, writer, ref propPath, item); - propPath.RemovePropertyName(); + if (comments) + { + propPath.LastValueEnd = reader.BytesConsumed; + previousOpen = true; + } + else + { + propPath.RemovePropertyName(); + } + break; } } } + private static bool ReadMemberValue(ref Utf8JsonReader reader, ref JsonWalk walk) + { + while (reader.Read()) + { + if (reader.TokenType != Comment) + { + return true; + } + + walk.Pending!.Add(ref reader, walk.StyleAt(reader.TokenStartIndex)); + } + + return false; + } + + /// + /// Whether the shape masks the value, for the comments the value owns. + /// + private bool IsMasked(JsonShape shape, JsonTokenType token) + { + if (token is Null && _keepNulls) + { + return false; + } + + if (ReferenceEquals(shape, JsonShape.UnknownPassThrough)) + { + return false; + } + + if (token is Null && shape.Kind is JsonShapeKind.Object or JsonShapeKind.Map or JsonShapeKind.Array) + { + return false; + } + + var container = token is StartObject or StartArray; + if (ReferenceEquals(shape, JsonShape.UnknownDescend)) + { + return !container; + } + + return shape.Kind switch + { + JsonShapeKind.Scalar => container, + JsonShapeKind.Object or JsonShapeKind.Map => token is not StartObject, + JsonShapeKind.Array => token is not StartArray, + _ => true, + }; + } + /// /// Writes an unknown value: containers are descended with the same treatment, scalars are masked or copied. /// diff --git a/DragoAnt.System.Text.Json.Observer/TagMasking.cs b/DragoAnt.System.Text.Json.Observer/TagMasking.cs index 44ad023..fd4f0d5 100644 --- a/DragoAnt.System.Text.Json.Observer/TagMasking.cs +++ b/DragoAnt.System.Text.Json.Observer/TagMasking.cs @@ -157,6 +157,19 @@ public void Return() public override void Keep() => KeepRequested = true; + public override void Comment(ReadOnlySpan text) + { + var bytes = ArrayPool.Shared.Rent(global::System.Text.Encoding.UTF8.GetMaxByteCount(text.Length)); + try + { + _writer.WriteComment(bytes.AsSpan(0, global::System.Text.Encoding.UTF8.GetBytes(text, bytes))); + } + finally + { + ArrayPool.Shared.Return(bytes, clearArray: true); + } + } + protected override void WriteNumber(ReadOnlySpan utf8Literal) => _writer.WriteRawValue(utf8Literal); protected override void InvalidNumber() => Failed = true; diff --git a/README.md b/README.md index a911e13..c5ec3e6 100644 --- a/README.md +++ b/README.md @@ -284,7 +284,7 @@ Console.WriteLine($"{result.Status} {Encoding.UTF8.GetString(output.WrittenSpan) | `Indented` | `false` | indented output | | `NameCaseInsensitive` | `true` | match rule names, `Names` tests (`Regex` included) and shapes ignoring case; pass the serializer's setting to match names as deserialization does | -Input may contain comments and trailing commas; a UTF-8 byte order mark is skipped. Comments are not written. +Input may contain comments and trailing commas; a UTF-8 byte order mark is skipped. Comments are dropped by default (`Comments = CommentPolicy.AllowList`): `CommentPolicy.BlockList` keeps them, `MaskAll` keeps them masked, and `.Comment(CommentKind.Inline, CommentRules.Keep)` after a rule decides the comments of the values it matches. A kept comment of a masked value is written masked. Kept comments are written as `/* … */`; see [comments](https://github.com/DragoAnt/Observer/blob/main/docs/comments.md). ## Performance diff --git a/skills/json-observer-masking/SKILL.md b/skills/json-observer-masking/SKILL.md index d4cebda..e8aeb0a 100644 --- a/skills/json-observer-masking/SKILL.md +++ b/skills/json-observer-masking/SKILL.md @@ -52,7 +52,7 @@ Console.WriteLine(masker.Mask("""{"user":"alice","password":"s3cret","card":{"nu - "The secret is still visible" → the rule is absolute but the field is nested, or the name differs (`Password` vs `passwd`); use `AnyDepth` and a `Names`. - A rule written before an `Obj(...)` rule for the same name wins and masks the whole object. - `JsonObserver.Obj(...)` on a root array returns `Invalid`; use `JsonObserver.Any(...)` when the root can be either. -- Comments in the input are accepted and never written; a UTF-8 BOM is skipped. +- Comments in the input are accepted and dropped by default; `new JsonObserverOptions { Comments = CommentPolicy.BlockList }` keeps them (masked when their value is masked), and `.Comment(kinds, rule)` after a rule overrides it. A UTF-8 BOM is skipped. ## Companions diff --git a/skills/json-observer-masking/migrating-from-1x.md b/skills/json-observer-masking/migrating-from-1x.md index 78ce39d..984b79e 100644 --- a/skills/json-observer-masking/migrating-from-1x.md +++ b/skills/json-observer-masking/migrating-from-1x.md @@ -16,7 +16,7 @@ Console.WriteLine(JsonObserver.Obj(root => root.Match("id").Unmasked()).Mask(""" ## 2. `Mask(string)` never throws, and its options moved -1.x threw on invalid JSON and took `JsonReaderOptions`, `JsonWriterOptions`, `ignoreNulls` and `ignoreComments`. 2.0 never throws, always skips comments, accepts trailing commas, writes non-ASCII unescaped, and takes one `JsonObserverOptions`. Drop the `try/catch` around `Mask`, and read `MaskResult` when you need to know what happened. +1.x threw on invalid JSON and took `JsonReaderOptions`, `JsonWriterOptions`, `ignoreNulls` and `ignoreComments`. 2.0 never throws, drops comments unless `JsonObserverOptions.Comments` or a comment rule keeps them, accepts trailing commas, writes non-ASCII unescaped, and takes one `JsonObserverOptions`. Drop the `try/catch` around `Mask`, and read `MaskResult` when you need to know what happened. ```csharp From 93804dc49a3e3c7a7d04395f7b822ffb5cb38bfd Mon Sep 17 00:00:00 2001 From: VasiliyF <5789590+vfofanov@users.noreply.github.com> Date: Mon, 5 Oct 2026 10:11:35 +0200 Subject: [PATCH 05/10] Add the char-span API, non-generic Read and FromShape - Mask(ReadOnlySpan, IBufferWriter | IBufferWriter) and Read(ReadOnlySpan) for both observer types - JsonObserver.Read(...) reports status and flags without a context - Explain rejects wildcards and tags, and declares the DragoAnt.ObserverPath string syntax - tests pin that existing Mask(null) / Mask("...") calls stay unambiguous --- .../FrozenSurfaceTests.cs | 127 ++++++++++++++ .../RunFrozenSurfaceTests.cs | 3 + .../JsonObserver.cs | 160 +++++++++++++++++- .../PathExplainer.cs | 5 + .../ShapeWalker.cs | 6 +- 5 files changed, 297 insertions(+), 4 deletions(-) create mode 100644 DragoAnt.System.Text.Json.Observer.Tests.Shared/FrozenSurfaceTests.cs create mode 100644 DragoAnt.System.Text.Json.Observer.Tests/RunFrozenSurfaceTests.cs diff --git a/DragoAnt.System.Text.Json.Observer.Tests.Shared/FrozenSurfaceTests.cs b/DragoAnt.System.Text.Json.Observer.Tests.Shared/FrozenSurfaceTests.cs new file mode 100644 index 0000000..90c7e24 --- /dev/null +++ b/DragoAnt.System.Text.Json.Observer.Tests.Shared/FrozenSurfaceTests.cs @@ -0,0 +1,127 @@ +using System.Buffers; +using System.Diagnostics.CodeAnalysis; +using System.Reflection; +using System.Text; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; + +namespace DragoAnt.System.Text.Json.Observer.Tests.Shared; + +public abstract class FrozenSurfaceTests +{ + private const string Json = """{"user":"ännä","password":"s3cret","card":{"number":"4111111111111111"}}"""; + + private static readonly JsonObserver Observer = JsonObserver.Obj(AnyDepth(b => b + .Match("password").Mask(MaskTag.Full) + .Path("card", "number").Mask(MaskTag.Last4), + BlockList)); + + [Fact] + public void CharSpan_ToBytes_EqualsTheStringApi() + { + var output = new ArrayBufferWriter(); + + var result = Observer.Mask(Json.AsSpan(), output); + + Encoding.UTF8.GetString(output.WrittenSpan).Should().Be(Observer.Mask(Json)); + result.Status.Should().Be(MaskStatus.Masked); + result.BytesWritten.Should().Be(output.WrittenCount); + } + + [Theory] + [InlineData(0)] + [InlineData(1)] + [InlineData(40)] + [InlineData(int.MaxValue)] + public void CharSpan_ToChars_EqualsTheStringApi(int maxOutputBytes) + { + var options = new JsonObserverOptions { MaxOutputBytes = maxOutputBytes }; + var output = new ArrayBufferWriter(); + + var result = Observer.Mask(Json.AsSpan(), output, options); + + new string(output.WrittenSpan).Should().Be(Observer.Mask(Json, out var expected, options)); + result.Should().Be(expected); + } + + [Fact] + public void CharSpan_LoneSurrogate_IsReplaced() + { + var output = new ArrayBufferWriter(); + + var result = JsonObserver.Obj(BlockList).Mask("{\"a\":\"x\uD800y\"}".AsSpan(), output); + + new string(output.WrittenSpan).Should().Be("{\"a\":\"x\uFFFDy\"}"); + result.Status.Should().Be(MaskStatus.Masked); + } + + [Fact] + public void CharSpan_WithAContext_ReadsAndMasks() + { + var context = new Holder(); + var observer = JsonObserver.Obj(b => b.Match("id").ReadInt((v, c) => c.Id = v).Unmasked(), ValuePolicy.AllowList); + var output = new ArrayBufferWriter(); + + observer.Mask("""{"id":7,"x":1}""".AsSpan(), output, context); + observer.Read("""{"id":8}""".AsSpan(), context).Status.Should().Be(MaskStatus.Masked); + + new string(output.WrittenSpan).Should().Be("""{"id":7,"x":"***"}"""); + context.Id.Should().Be(8); + } + + [Fact] + public void NonGenericRead_ReportsWithoutWriting() + { + Observer.Read(Json).Should().Be(new MaskResult { Status = MaskStatus.Masked, FailedAtByte = -1 }); + Observer.Read(Json.AsSpan()).Status.Should().Be(MaskStatus.Masked); + Observer.Read(Encoding.UTF8.GetBytes(Json)).Status.Should().Be(MaskStatus.Masked); + Observer.Read(new ReadOnlySequence(Encoding.UTF8.GetBytes("""{"a":1}{"b":2}"""))).Flags.Should().Be(MaskFlags.TrailingData); + Observer.Read("""{"a":""").Flags.Should().Be(MaskFlags.InputTruncated); + Observer.Read((string?)null).Status.Should().Be(MaskStatus.Unrecognized); + } + + [Fact] + public void FromShape_WithAContext() + { + var shape = JsonShape.Object(("id", JsonShape.Scalar), ("pin", JsonShape.Masked(MaskTag.Full))); + var observer = JsonObserver.FromShape(shape); + + observer.Mask("""{"id":1,"pin":"1234","x":2}""", new Holder()).Should().Be("""{"id":1,"pin":"***","x":"***"}"""); + observer.Explain("pin").Outcome.Should().Be(PathOutcome.Masked); + } + + [Fact] + public void ExistingCalls_StayUnambiguous() + { + Observer.Mask(null).Should().BeNull(); + Observer.Mask("""{"password":"x"}""").Should().Be("""{"password":"***"}"""); + Observer.Mask("""{"password":"x"}""", (JsonObserverOptions?)null).Should().Be("""{"password":"***"}"""); + Observer.Mask("""{"password":"x"}""", out var result).Should().Be("""{"password":"***"}"""); + result.Status.Should().Be(MaskStatus.Masked); + Observer.Mask("""{"password":"x"}""".AsSpan(), new ArrayBufferWriter()).Status.Should().Be(MaskStatus.Masked); + Observer.Mask("""{"password":"x"}"""u8, new ArrayBufferWriter()).Status.Should().Be(MaskStatus.Masked); + } + + [Fact] + public void Explain_DeclaresItsPathSyntax() + { + var parameter = typeof(JsonObserver).GetMethod(nameof(JsonObserver.Explain))!.GetParameters()[0]; + + parameter.GetCustomAttribute()!.Syntax.Should().Be(JsonObserver.PathSyntax).And.Be("DragoAnt.ObserverPath"); + typeof(JsonObserver).GetMethod(nameof(JsonObserver.Explain))!.GetParameters()[0] + .GetCustomAttribute()!.Syntax.Should().Be("DragoAnt.ObserverPath"); + } + + [Theory] + [InlineData("..password")] + [InlineData("a.*")] + [InlineData("a[*]")] + [InlineData("a:Last4")] + public void Explain_AcceptsOnlyConcretePaths(string path) + => FluentActions.Invoking(() => Observer.Explain(path)).Should().Throw(); + + public sealed class Holder + { + public int? Id { get; set; } + } +} diff --git a/DragoAnt.System.Text.Json.Observer.Tests/RunFrozenSurfaceTests.cs b/DragoAnt.System.Text.Json.Observer.Tests/RunFrozenSurfaceTests.cs new file mode 100644 index 0000000..75bccd7 --- /dev/null +++ b/DragoAnt.System.Text.Json.Observer.Tests/RunFrozenSurfaceTests.cs @@ -0,0 +1,3 @@ +namespace DragoAnt.System.Text.Json.Observer.Tests; + +public sealed class RunFrozenSurfaceTests : Shared.FrozenSurfaceTests; diff --git a/DragoAnt.System.Text.Json.Observer/JsonObserver.cs b/DragoAnt.System.Text.Json.Observer/JsonObserver.cs index c13464f..a5456ee 100644 --- a/DragoAnt.System.Text.Json.Observer/JsonObserver.cs +++ b/DragoAnt.System.Text.Json.Observer/JsonObserver.cs @@ -134,7 +134,21 @@ public static JsonObserver FromShape(JsonShape shape, JsonShapeOptions? options { ArgumentNullException.ThrowIfNull(shape); var walker = new ShapeWalker(shape, options ?? JsonShapeOptions.Default); - return new JsonObserver(new JsonObserver(walker.Invoke, walker, hasCommentRules: false)); + return new JsonObserver(new JsonObserver(walker.Invoke, walker, hasCommentRules: false)); + } + + /// + /// Creates an observer with a context that masks against an expected structure, like + /// ; the context is handed through for symmetry with the rule-based observers. + /// + /// Expected structure, for example from . It cannot change afterwards. + /// Treatment of unknown properties and of null. + /// Type of the context of the calls. + public static JsonObserver FromShape(JsonShape shape, JsonShapeOptions? options = null) + { + ArgumentNullException.ThrowIfNull(shape); + var walker = new ShapeWalker(shape, options ?? JsonShapeOptions.Default); + return new JsonObserver(walker.Invoke, walker, hasCommentRules: false); } private JsonObserver(JsonObserver masking) @@ -188,6 +202,57 @@ public MaskResult Mask(ReadOnlySpan utf8, IBufferWriter output, Json public MaskResult Mask(in ReadOnlySequence utf8, IBufferWriter output, JsonObserverOptions? options = null) => _masking.Mask(utf8, output, NoContext.Instance, options); + /// + /// Masks a JSON text given as characters into UTF-8 , without a . + /// Never throws; writes exactly what returns, as UTF-8. + /// + /// JSON text; it may be cut short. Lone surrogates are read as U+FFFD. + /// Receives the masked JSON as UTF-8. + /// Limits and output settings; when omitted. + /// Status, UTF-8 bytes written and the UTF-8 offset where reading stopped. + public MaskResult Mask(ReadOnlySpan json, IBufferWriter utf8Output, JsonObserverOptions? options = null) + => _masking.Mask(json, utf8Output, NoContext.Instance, options); + + /// + /// Masks a JSON text given as characters into as characters, without a . + /// Never throws; writes exactly what returns. + /// + /// JSON text; it may be cut short. Lone surrogates are read as U+FFFD. + /// Receives the masked JSON. + /// Limits and output settings; when omitted. + /// Status, UTF-8 length of the output and UTF-8 offset where reading stopped. + public MaskResult Mask(ReadOnlySpan json, IBufferWriter output, JsonObserverOptions? options = null) + => _masking.Mask(json, output, NoContext.Instance, options); + + /// + /// Reads a JSON text without writing anything, to learn whether it is complete and valid: the result has the same + /// status, offset and flags a masking call would report, except the output limits. Never throws. + /// + /// JSON text; it may be cut short. + /// Limits; when omitted. + /// Status, the UTF-8 offset where reading stopped and the flags; for null. + public MaskResult Read(string? json, JsonObserverOptions? options = null) => _masking.Read(json, NoContext.Instance, options); + + /// + public MaskResult Read(ReadOnlySpan json, JsonObserverOptions? options = null) => _masking.Read(json, NoContext.Instance, options); + + /// + /// Reads a UTF-8 JSON payload without writing anything; see . + /// + /// UTF-8 JSON payload; it may be cut short. A leading byte order mark is skipped. + /// Limits; when omitted. + /// Status, the offset where reading stopped and the flags. + public MaskResult Read(ReadOnlySpan utf8, JsonObserverOptions? options = null) => _masking.Read(utf8, NoContext.Instance, options); + + /// + /// Reads a UTF-8 JSON payload held in several buffers without writing anything or copying it into one buffer; see + /// . + /// + /// UTF-8 JSON payload; it may be cut short. A leading byte order mark is skipped. + /// Limits; when omitted. + /// Status, the offset where reading stopped and the flags. + public MaskResult Read(in ReadOnlySequence utf8, JsonObserverOptions? options = null) => _masking.Read(utf8, NoContext.Instance, options); + /// public PathExplanation Explain([StringSyntax(PathSyntax)] string path, ValueKind valueKind = ValueKind.String, JsonObserverOptions? options = null) => _masking.Explain(path, valueKind, options); @@ -332,6 +397,99 @@ public MaskResult Mask(in ReadOnlySequence utf8, IBufferWriter outpu return Mask(ref reader, default, input, output, context, options, comments); } + /// + /// Masks a JSON text given as characters into UTF-8 and hands values to + /// , without a . Never throws. + /// + /// JSON text; it may be cut short. Lone surrogates are read as U+FFFD. + /// Receives the masked JSON as UTF-8. + /// Receives the values read rules extract. + /// Limits and output settings; when omitted. + /// Status, UTF-8 bytes written and the UTF-8 offset where reading stopped. + public MaskResult Mask(ReadOnlySpan json, IBufferWriter utf8Output, TContext context, JsonObserverOptions? options = null) + { + byte[]? input = null; + try + { + input = ArrayPool.Shared.Rent(Encoding.UTF8.GetMaxByteCount(json.Length)); + global::System.Text.Unicode.Utf8.FromUtf16(json, input, out _, out var length, replaceInvalidSequences: true); + return Mask(input.AsSpan(0, length), utf8Output, context, options); + } + catch (Exception) + { + return new MaskResult { Status = MaskStatus.Invalid }; + } + finally + { + if (input is not null) + { + ArrayPool.Shared.Return(input, clearArray: true); + } + } + } + + /// + /// Masks a JSON text given as characters into as characters and hands values to + /// , without a . Never throws. + /// + /// JSON text; it may be cut short. Lone surrogates are read as U+FFFD. + /// Receives the masked JSON. + /// Receives the values read rules extract. + /// Limits and output settings; when omitted. + /// Status, UTF-8 length of the output and UTF-8 offset where reading stopped. + public MaskResult Mask(ReadOnlySpan json, IBufferWriter output, TContext context, JsonObserverOptions? options = null) + { + PooledBufferWriter? utf8 = null; + try + { + utf8 = PooledBufferWriter.Rent(Math.Clamp(json.Length, 256, 64 * 1024)); + var result = Mask(json, utf8, context, options); + var written = utf8.WrittenSpan; + var chars = output.GetSpan(Encoding.UTF8.GetMaxCharCount(written.Length)); + global::System.Text.Unicode.Utf8.ToUtf16(written, chars, out _, out var length); + output.Advance(length); + return result; + } + catch (Exception) + { + return new MaskResult { Status = MaskStatus.Invalid }; + } + finally + { + utf8?.Return(); + } + } + + /// + /// Hands values of a JSON text given as characters to without writing anything or + /// creating a . Never throws. + /// + /// JSON text; it may be cut short. + /// Receives the values read rules extract. + /// Limits; when omitted. + /// Status, the UTF-8 offset where reading stopped and the flags. + public MaskResult Read(ReadOnlySpan json, TContext context, JsonObserverOptions? options = null) + { + byte[]? input = null; + try + { + input = ArrayPool.Shared.Rent(Encoding.UTF8.GetMaxByteCount(json.Length)); + global::System.Text.Unicode.Utf8.FromUtf16(json, input, out _, out var length, replaceInvalidSequences: true); + return Read(input.AsSpan(0, length), context, options); + } + catch (Exception) + { + return new MaskResult { Status = MaskStatus.Invalid }; + } + finally + { + if (input is not null) + { + ArrayPool.Shared.Return(input, clearArray: true); + } + } + } + private MaskResult Mask( ref Utf8JsonReader reader, ReadOnlySpan input, diff --git a/DragoAnt.System.Text.Json.Observer/PathExplainer.cs b/DragoAnt.System.Text.Json.Observer/PathExplainer.cs index c412717..7216bd3 100644 --- a/DragoAnt.System.Text.Json.Observer/PathExplainer.cs +++ b/DragoAnt.System.Text.Json.Observer/PathExplainer.cs @@ -147,6 +147,11 @@ internal static List Parse(string path) var start = i; while (i < path.Length && path[i] is not ('.' or '[')) { + if (path[i] is '*' or ':') + { + throw Invalid(path, i); + } + i++; } diff --git a/DragoAnt.System.Text.Json.Observer/ShapeWalker.cs b/DragoAnt.System.Text.Json.Observer/ShapeWalker.cs index 5e342d4..9b4fc0e 100644 --- a/DragoAnt.System.Text.Json.Observer/ShapeWalker.cs +++ b/DragoAnt.System.Text.Json.Observer/ShapeWalker.cs @@ -30,13 +30,13 @@ public ShapeWalker(JsonShape root, JsonShapeOptions options) }; } - public void Invoke( + public void Invoke( ref Utf8JsonReader reader, JsonWriter writer, - NoContext context, + TContext context, int depth, ref JsonWalk propPath, - ValueRule defaultValue) + ValueRule defaultValue) => Write(ref reader, writer, ref propPath, _root); private void Write(ref Utf8JsonReader reader, JsonWriter writer, ref JsonWalk propPath, JsonShape shape) From c1bf6daee710ff696ce9e2d61e9aeb3ed6377941 Mon Sep 17 00:00:00 2001 From: VasiliyF <5789590+vfofanov@users.noreply.github.com> Date: Mon, 5 Oct 2026 10:15:38 +0200 Subject: [PATCH 06/10] Validate the package against 1.0.2 and list the 2.0 breaks - package validation baseline 1.0.2 (the unlisted 9.0.0 / 9999.0.0 are stray uploads, not baselines) - CompatibilitySuppressions.xml lists every intended 1.0.2 -> 2.0 break, so any other break fails the pack - the buildTransitive props are packed by MSBuildKit --- .../CompatibilitySuppressions.xml | 960 ++++++++++++++++++ .../DragoAnt.System.Text.Json.Observer.csproj | 4 +- 2 files changed, 961 insertions(+), 3 deletions(-) create mode 100644 DragoAnt.System.Text.Json.Observer/CompatibilitySuppressions.xml diff --git a/DragoAnt.System.Text.Json.Observer/CompatibilitySuppressions.xml b/DragoAnt.System.Text.Json.Observer/CompatibilitySuppressions.xml new file mode 100644 index 0000000..3e093f1 --- /dev/null +++ b/DragoAnt.System.Text.Json.Observer/CompatibilitySuppressions.xml @@ -0,0 +1,960 @@ + + + + + CP0001 + T:DragoAnt.System.Text.Json.Observer.Builders.JsonValuePolicyBuilder`1 + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0001 + T:DragoAnt.System.Text.Json.Observer.JsonObserverDelegate`1 + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0001 + T:DragoAnt.System.Text.Json.Observer.JsonObserverException + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0001 + T:DragoAnt.System.Text.Json.Observer.JsonObserveringEmptyContext + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0001 + T:DragoAnt.System.Text.Json.Observer.JsonObserverValueDelegate`1 + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0001 + T:DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0001 + T:DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies`1 + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0001 + T:DragoAnt.System.Text.Json.Observer.JsonPropertyMatchDelegate + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0001 + T:DragoAnt.System.Text.Json.Observer.JsonPropertyPathMatchDelegate + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0001 + T:DragoAnt.System.Text.Json.Observer.JsonWriter.IgnoreNullsJsonTokenType + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0001 + T:DragoAnt.System.Text.Json.Observer.PropertyPath + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0001 + T:DragoAnt.System.Text.Json.Observer.PropertyPathMatch + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0001 + T:DragoAnt.System.Text.Json.Observer.Strategies.PropMatches + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0001 + T:DragoAnt.System.Text.Json.Observer.Strategies.PropMatchingStrategy + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0001 + T:DragoAnt.System.Text.Json.Observer.Strategies.StringMaskingStrategy`1 + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0001 + T:DragoAnt.System.Text.Json.Observer.Builders.JsonValuePolicyBuilder`1 + lib/net9.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0001 + T:DragoAnt.System.Text.Json.Observer.JsonObserverDelegate`1 + lib/net9.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0001 + T:DragoAnt.System.Text.Json.Observer.JsonObserverException + lib/net9.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0001 + T:DragoAnt.System.Text.Json.Observer.JsonObserveringEmptyContext + lib/net9.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0001 + T:DragoAnt.System.Text.Json.Observer.JsonObserverValueDelegate`1 + lib/net9.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0001 + T:DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies + lib/net9.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0001 + T:DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies`1 + lib/net9.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0001 + T:DragoAnt.System.Text.Json.Observer.JsonPropertyMatchDelegate + lib/net9.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0001 + T:DragoAnt.System.Text.Json.Observer.JsonPropertyPathMatchDelegate + lib/net9.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0001 + T:DragoAnt.System.Text.Json.Observer.JsonWriter.IgnoreNullsJsonTokenType + lib/net9.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0001 + T:DragoAnt.System.Text.Json.Observer.PropertyPath + lib/net9.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0001 + T:DragoAnt.System.Text.Json.Observer.PropertyPathMatch + lib/net9.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0001 + T:DragoAnt.System.Text.Json.Observer.Strategies.PropMatches + lib/net9.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0001 + T:DragoAnt.System.Text.Json.Observer.Strategies.PropMatchingStrategy + lib/net9.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0001 + T:DragoAnt.System.Text.Json.Observer.Strategies.StringMaskingStrategy`1 + lib/net9.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + F:DragoAnt.System.Text.Json.Observer.JsonWriter.Empty + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.Builders.JsonArrayBuilder`1.#ctor(DragoAnt.System.Text.Json.Observer.JsonObserverValueDelegate{`0}) + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.Builders.JsonArrayBuilder`1.Array(System.Action{DragoAnt.System.Text.Json.Observer.Builders.JsonArrayBuilder{`0}},DragoAnt.System.Text.Json.Observer.JsonObserverValueDelegate{`0}) + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.Builders.JsonArrayBuilder`1.MaskRawValue(System.Func{System.String,`0,System.String}) + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.Builders.JsonArrayBuilder`1.MaskStr(DragoAnt.System.Text.Json.Observer.Strategies.StringMaskingStrategy{`0}) + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.Builders.JsonArrayBuilder`1.MaskStr(System.Func{System.String,`0,System.String}) + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.Builders.JsonArrayBuilder`1.MaskValue(DragoAnt.System.Text.Json.Observer.JsonObserverDelegate{`0}) + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.Builders.JsonArrayBuilder`1.MaskValue(DragoAnt.System.Text.Json.Observer.JsonObserverValueDelegate{`0}) + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.Builders.JsonArrayBuilder`1.Obj(System.Action{DragoAnt.System.Text.Json.Observer.Builders.JsonObjBuilder{`0}},DragoAnt.System.Text.Json.Observer.JsonObserverValueDelegate{`0}) + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.Builders.JsonObjBuilder`1.#ctor(DragoAnt.System.Text.Json.Observer.JsonObserverValueDelegate{`0}) + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.Builders.JsonObjBuilder`1.Match(DragoAnt.System.Text.Json.Observer.Strategies.PropMatchingStrategy) + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.Builders.JsonObjBuilder`1.Match(DragoAnt.System.Text.Json.Observer.Strategies.PropMatchingStrategy[]) + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.Builders.JsonObjBuilder`1.PropertyMaskingStrategyBuilder.#ctor(DragoAnt.System.Text.Json.Observer.Builders.JsonObjBuilder{`0},DragoAnt.System.Text.Json.Observer.PropertyPathMatch,DragoAnt.System.Text.Json.Observer.JsonObserverValueDelegate{`0}) + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.Builders.JsonObjBuilder`1.PropertyMaskingStrategyBuilder.Array(DragoAnt.System.Text.Json.Observer.JsonObserverDelegate{`0}) + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.Builders.JsonObjBuilder`1.PropertyMaskingStrategyBuilder.Array(System.Action{DragoAnt.System.Text.Json.Observer.Builders.JsonArrayBuilder{`0}},DragoAnt.System.Text.Json.Observer.JsonObserverValueDelegate{`0}) + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.Builders.JsonObjBuilder`1.PropertyMaskingStrategyBuilder.MaskRawValue(System.Func{System.String,`0,System.String}) + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.Builders.JsonObjBuilder`1.PropertyMaskingStrategyBuilder.MaskStr(DragoAnt.System.Text.Json.Observer.Strategies.StringMaskingStrategy{`0}) + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.Builders.JsonObjBuilder`1.PropertyMaskingStrategyBuilder.MaskStr(System.Func{System.String,`0,System.String}) + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.Builders.JsonObjBuilder`1.PropertyMaskingStrategyBuilder.MaskValue(DragoAnt.System.Text.Json.Observer.JsonObserverDelegate{`0}) + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.Builders.JsonObjBuilder`1.PropertyMaskingStrategyBuilder.MaskValue(DragoAnt.System.Text.Json.Observer.JsonObserverValueDelegate{`0}) + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.Builders.JsonObjBuilder`1.PropertyMaskingStrategyBuilder.Obj(DragoAnt.System.Text.Json.Observer.JsonObserverDelegate{`0}) + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.Builders.JsonObjBuilder`1.PropertyMaskingStrategyBuilder.Obj(System.Action{DragoAnt.System.Text.Json.Observer.Builders.JsonObjBuilder{`0}},DragoAnt.System.Text.Json.Observer.JsonObserverValueDelegate{`0}) + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.Builders.JsonObjBuilder`1.PropertyMaskingStrategyBuilder.ReadBool(System.Action{System.Nullable{System.Boolean},`0}) + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.Builders.JsonObjBuilder`1.PropertyMaskingStrategyBuilder.ReadDecimal(System.Action{System.Nullable{System.Decimal},`0}) + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.Builders.JsonObjBuilder`1.PropertyMaskingStrategyBuilder.ReadInt(System.Action{System.Nullable{System.Int32},`0}) + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.Builders.JsonObjBuilder`1.PropertyMaskingStrategyBuilder.ReadLong(System.Action{System.Nullable{System.Int64},`0}) + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.Builders.JsonObjBuilder`1.PropertyMaskingStrategyBuilder.ReadRaw(System.Action{System.String,`0}) + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.Builders.JsonObjBuilder`1.PropertyMaskingStrategyBuilder.ReadStr(System.Action{System.String,`0}) + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.JsonObserver.Any(System.Action{DragoAnt.System.Text.Json.Observer.Builders.JsonObjBuilder{DragoAnt.System.Text.Json.Observer.JsonObserveringEmptyContext}},System.Action{DragoAnt.System.Text.Json.Observer.Builders.JsonArrayBuilder{DragoAnt.System.Text.Json.Observer.JsonObserveringEmptyContext}},DragoAnt.System.Text.Json.Observer.JsonObserverValueDelegate{DragoAnt.System.Text.Json.Observer.JsonObserveringEmptyContext}) + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.JsonObserver.Any``1(System.Action{DragoAnt.System.Text.Json.Observer.Builders.JsonObjBuilder{``0}},System.Action{DragoAnt.System.Text.Json.Observer.Builders.JsonArrayBuilder{``0}},DragoAnt.System.Text.Json.Observer.JsonObserverValueDelegate{``0}) + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.JsonObserver.Array(DragoAnt.System.Text.Json.Observer.JsonObserverValueDelegate{DragoAnt.System.Text.Json.Observer.JsonObserveringEmptyContext}) + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.JsonObserver.Array(System.Action{DragoAnt.System.Text.Json.Observer.Builders.JsonArrayBuilder{DragoAnt.System.Text.Json.Observer.JsonObserveringEmptyContext}},DragoAnt.System.Text.Json.Observer.JsonObserverValueDelegate{DragoAnt.System.Text.Json.Observer.JsonObserveringEmptyContext}) + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.JsonObserver.Array``1(DragoAnt.System.Text.Json.Observer.JsonObserverValueDelegate{``0}) + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.JsonObserver.Array``1(System.Action{DragoAnt.System.Text.Json.Observer.Builders.JsonArrayBuilder{``0}},DragoAnt.System.Text.Json.Observer.JsonObserverValueDelegate{``0}) + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.JsonObserver.Mask(System.String,System.Text.Json.JsonReaderOptions,System.Text.Json.JsonWriterOptions,System.Boolean,System.Boolean) + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.JsonObserver.Obj(DragoAnt.System.Text.Json.Observer.JsonObserverValueDelegate{DragoAnt.System.Text.Json.Observer.JsonObserveringEmptyContext}) + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.JsonObserver.Obj(System.Action{DragoAnt.System.Text.Json.Observer.Builders.JsonObjBuilder{DragoAnt.System.Text.Json.Observer.JsonObserveringEmptyContext}},DragoAnt.System.Text.Json.Observer.JsonObserverValueDelegate{DragoAnt.System.Text.Json.Observer.JsonObserveringEmptyContext}) + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.JsonObserver.Obj``1(DragoAnt.System.Text.Json.Observer.JsonObserverValueDelegate{``0}) + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.JsonObserver.Obj``1(System.Action{DragoAnt.System.Text.Json.Observer.Builders.JsonObjBuilder{``0}},DragoAnt.System.Text.Json.Observer.JsonObserverValueDelegate{``0}) + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.JsonObserver`1.Mask(System.String,`0,System.Text.Json.JsonReaderOptions,System.Text.Json.JsonWriterOptions,System.Boolean,System.Boolean) + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.JsonObserver`1.Read(System.Byte[],`0,System.Text.Json.JsonReaderOptions) + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.JsonObserver`1.Read(System.String,`0,System.Text.Json.JsonReaderOptions) + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.JsonWriter.#ctor + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.JsonWriter.FromUtf8JsonWriter(System.Text.Json.Utf8JsonWriter,System.Boolean,System.Boolean) + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.JsonWriter.WriteCommentValue(System.String) + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + F:DragoAnt.System.Text.Json.Observer.JsonWriter.Empty + lib/net9.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.Builders.JsonArrayBuilder`1.#ctor(DragoAnt.System.Text.Json.Observer.JsonObserverValueDelegate{`0}) + lib/net9.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.Builders.JsonArrayBuilder`1.Array(System.Action{DragoAnt.System.Text.Json.Observer.Builders.JsonArrayBuilder{`0}},DragoAnt.System.Text.Json.Observer.JsonObserverValueDelegate{`0}) + lib/net9.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.Builders.JsonArrayBuilder`1.MaskRawValue(System.Func{System.String,`0,System.String}) + lib/net9.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.Builders.JsonArrayBuilder`1.MaskStr(DragoAnt.System.Text.Json.Observer.Strategies.StringMaskingStrategy{`0}) + lib/net9.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.Builders.JsonArrayBuilder`1.MaskStr(System.Func{System.String,`0,System.String}) + lib/net9.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.Builders.JsonArrayBuilder`1.MaskValue(DragoAnt.System.Text.Json.Observer.JsonObserverDelegate{`0}) + lib/net9.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.Builders.JsonArrayBuilder`1.MaskValue(DragoAnt.System.Text.Json.Observer.JsonObserverValueDelegate{`0}) + lib/net9.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.Builders.JsonArrayBuilder`1.Obj(System.Action{DragoAnt.System.Text.Json.Observer.Builders.JsonObjBuilder{`0}},DragoAnt.System.Text.Json.Observer.JsonObserverValueDelegate{`0}) + lib/net9.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.Builders.JsonObjBuilder`1.#ctor(DragoAnt.System.Text.Json.Observer.JsonObserverValueDelegate{`0}) + lib/net9.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.Builders.JsonObjBuilder`1.Match(DragoAnt.System.Text.Json.Observer.Strategies.PropMatchingStrategy) + lib/net9.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.Builders.JsonObjBuilder`1.Match(DragoAnt.System.Text.Json.Observer.Strategies.PropMatchingStrategy[]) + lib/net9.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.Builders.JsonObjBuilder`1.PropertyMaskingStrategyBuilder.#ctor(DragoAnt.System.Text.Json.Observer.Builders.JsonObjBuilder{`0},DragoAnt.System.Text.Json.Observer.PropertyPathMatch,DragoAnt.System.Text.Json.Observer.JsonObserverValueDelegate{`0}) + lib/net9.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.Builders.JsonObjBuilder`1.PropertyMaskingStrategyBuilder.Array(DragoAnt.System.Text.Json.Observer.JsonObserverDelegate{`0}) + lib/net9.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.Builders.JsonObjBuilder`1.PropertyMaskingStrategyBuilder.Array(System.Action{DragoAnt.System.Text.Json.Observer.Builders.JsonArrayBuilder{`0}},DragoAnt.System.Text.Json.Observer.JsonObserverValueDelegate{`0}) + lib/net9.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.Builders.JsonObjBuilder`1.PropertyMaskingStrategyBuilder.MaskRawValue(System.Func{System.String,`0,System.String}) + lib/net9.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.Builders.JsonObjBuilder`1.PropertyMaskingStrategyBuilder.MaskStr(DragoAnt.System.Text.Json.Observer.Strategies.StringMaskingStrategy{`0}) + lib/net9.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.Builders.JsonObjBuilder`1.PropertyMaskingStrategyBuilder.MaskStr(System.Func{System.String,`0,System.String}) + lib/net9.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.Builders.JsonObjBuilder`1.PropertyMaskingStrategyBuilder.MaskValue(DragoAnt.System.Text.Json.Observer.JsonObserverDelegate{`0}) + lib/net9.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.Builders.JsonObjBuilder`1.PropertyMaskingStrategyBuilder.MaskValue(DragoAnt.System.Text.Json.Observer.JsonObserverValueDelegate{`0}) + lib/net9.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.Builders.JsonObjBuilder`1.PropertyMaskingStrategyBuilder.Obj(DragoAnt.System.Text.Json.Observer.JsonObserverDelegate{`0}) + lib/net9.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.Builders.JsonObjBuilder`1.PropertyMaskingStrategyBuilder.Obj(System.Action{DragoAnt.System.Text.Json.Observer.Builders.JsonObjBuilder{`0}},DragoAnt.System.Text.Json.Observer.JsonObserverValueDelegate{`0}) + lib/net9.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.Builders.JsonObjBuilder`1.PropertyMaskingStrategyBuilder.ReadBool(System.Action{System.Nullable{System.Boolean},`0}) + lib/net9.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.Builders.JsonObjBuilder`1.PropertyMaskingStrategyBuilder.ReadDecimal(System.Action{System.Nullable{System.Decimal},`0}) + lib/net9.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.Builders.JsonObjBuilder`1.PropertyMaskingStrategyBuilder.ReadInt(System.Action{System.Nullable{System.Int32},`0}) + lib/net9.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.Builders.JsonObjBuilder`1.PropertyMaskingStrategyBuilder.ReadLong(System.Action{System.Nullable{System.Int64},`0}) + lib/net9.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.Builders.JsonObjBuilder`1.PropertyMaskingStrategyBuilder.ReadRaw(System.Action{System.String,`0}) + lib/net9.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.Builders.JsonObjBuilder`1.PropertyMaskingStrategyBuilder.ReadStr(System.Action{System.String,`0}) + lib/net9.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.JsonObserver.Any(System.Action{DragoAnt.System.Text.Json.Observer.Builders.JsonObjBuilder{DragoAnt.System.Text.Json.Observer.JsonObserveringEmptyContext}},System.Action{DragoAnt.System.Text.Json.Observer.Builders.JsonArrayBuilder{DragoAnt.System.Text.Json.Observer.JsonObserveringEmptyContext}},DragoAnt.System.Text.Json.Observer.JsonObserverValueDelegate{DragoAnt.System.Text.Json.Observer.JsonObserveringEmptyContext}) + lib/net9.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.JsonObserver.Any``1(System.Action{DragoAnt.System.Text.Json.Observer.Builders.JsonObjBuilder{``0}},System.Action{DragoAnt.System.Text.Json.Observer.Builders.JsonArrayBuilder{``0}},DragoAnt.System.Text.Json.Observer.JsonObserverValueDelegate{``0}) + lib/net9.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.JsonObserver.Array(DragoAnt.System.Text.Json.Observer.JsonObserverValueDelegate{DragoAnt.System.Text.Json.Observer.JsonObserveringEmptyContext}) + lib/net9.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.JsonObserver.Array(System.Action{DragoAnt.System.Text.Json.Observer.Builders.JsonArrayBuilder{DragoAnt.System.Text.Json.Observer.JsonObserveringEmptyContext}},DragoAnt.System.Text.Json.Observer.JsonObserverValueDelegate{DragoAnt.System.Text.Json.Observer.JsonObserveringEmptyContext}) + lib/net9.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.JsonObserver.Array``1(DragoAnt.System.Text.Json.Observer.JsonObserverValueDelegate{``0}) + lib/net9.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.JsonObserver.Array``1(System.Action{DragoAnt.System.Text.Json.Observer.Builders.JsonArrayBuilder{``0}},DragoAnt.System.Text.Json.Observer.JsonObserverValueDelegate{``0}) + lib/net9.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.JsonObserver.Mask(System.String,System.Text.Json.JsonReaderOptions,System.Text.Json.JsonWriterOptions,System.Boolean,System.Boolean) + lib/net9.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.JsonObserver.Obj(DragoAnt.System.Text.Json.Observer.JsonObserverValueDelegate{DragoAnt.System.Text.Json.Observer.JsonObserveringEmptyContext}) + lib/net9.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.JsonObserver.Obj(System.Action{DragoAnt.System.Text.Json.Observer.Builders.JsonObjBuilder{DragoAnt.System.Text.Json.Observer.JsonObserveringEmptyContext}},DragoAnt.System.Text.Json.Observer.JsonObserverValueDelegate{DragoAnt.System.Text.Json.Observer.JsonObserveringEmptyContext}) + lib/net9.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.JsonObserver.Obj``1(DragoAnt.System.Text.Json.Observer.JsonObserverValueDelegate{``0}) + lib/net9.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.JsonObserver.Obj``1(System.Action{DragoAnt.System.Text.Json.Observer.Builders.JsonObjBuilder{``0}},DragoAnt.System.Text.Json.Observer.JsonObserverValueDelegate{``0}) + lib/net9.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.JsonObserver`1.Mask(System.String,`0,System.Text.Json.JsonReaderOptions,System.Text.Json.JsonWriterOptions,System.Boolean,System.Boolean) + lib/net9.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.JsonObserver`1.Read(System.Byte[],`0,System.Text.Json.JsonReaderOptions) + lib/net9.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.JsonObserver`1.Read(System.String,`0,System.Text.Json.JsonReaderOptions) + lib/net9.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.JsonWriter.#ctor + lib/net9.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.JsonWriter.FromUtf8JsonWriter(System.Text.Json.Utf8JsonWriter,System.Boolean,System.Boolean) + lib/net9.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0002 + M:DragoAnt.System.Text.Json.Observer.JsonWriter.WriteCommentValue(System.String) + lib/net9.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0005 + M:DragoAnt.System.Text.Json.Observer.JsonWriter.WriteBase64StringValue(System.ReadOnlySpan{System.Byte}) + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0005 + M:DragoAnt.System.Text.Json.Observer.JsonWriter.WriteNumberValue(System.Double) + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0005 + M:DragoAnt.System.Text.Json.Observer.JsonWriter.WritePropertyName(System.ReadOnlySpan{System.Byte}) + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0005 + M:DragoAnt.System.Text.Json.Observer.JsonWriter.WritePropertyName(System.ReadOnlySpan{System.Char}) + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0005 + M:DragoAnt.System.Text.Json.Observer.JsonWriter.WriteRawValue(System.ReadOnlySpan{System.Byte}) + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0005 + M:DragoAnt.System.Text.Json.Observer.JsonWriter.WriteStringValue(System.ReadOnlySpan{System.Byte}) + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0005 + M:DragoAnt.System.Text.Json.Observer.JsonWriter.WriteStringValue(System.ReadOnlySpan{System.Char}) + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0005 + M:DragoAnt.System.Text.Json.Observer.JsonWriter.WriteBase64StringValue(System.ReadOnlySpan{System.Byte}) + lib/net9.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0005 + M:DragoAnt.System.Text.Json.Observer.JsonWriter.WriteNumberValue(System.Double) + lib/net9.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0005 + M:DragoAnt.System.Text.Json.Observer.JsonWriter.WritePropertyName(System.ReadOnlySpan{System.Byte}) + lib/net9.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0005 + M:DragoAnt.System.Text.Json.Observer.JsonWriter.WritePropertyName(System.ReadOnlySpan{System.Char}) + lib/net9.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0005 + M:DragoAnt.System.Text.Json.Observer.JsonWriter.WriteRawValue(System.ReadOnlySpan{System.Byte}) + lib/net9.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0005 + M:DragoAnt.System.Text.Json.Observer.JsonWriter.WriteStringValue(System.ReadOnlySpan{System.Byte}) + lib/net9.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0005 + M:DragoAnt.System.Text.Json.Observer.JsonWriter.WriteStringValue(System.ReadOnlySpan{System.Char}) + lib/net9.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0009 + T:DragoAnt.System.Text.Json.Observer.JsonWriter + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + CP0009 + T:DragoAnt.System.Text.Json.Observer.JsonWriter + lib/net9.0/DragoAnt.System.Text.Json.Observer.dll + lib/net8.0/DragoAnt.System.Text.Json.Observer.dll + true + + + PKV006 + net7.0 + + \ No newline at end of file diff --git a/DragoAnt.System.Text.Json.Observer/DragoAnt.System.Text.Json.Observer.csproj b/DragoAnt.System.Text.Json.Observer/DragoAnt.System.Text.Json.Observer.csproj index cd56f87..c864699 100644 --- a/DragoAnt.System.Text.Json.Observer/DragoAnt.System.Text.Json.Observer.csproj +++ b/DragoAnt.System.Text.Json.Observer/DragoAnt.System.Text.Json.Observer.csproj @@ -2,14 +2,12 @@ Mask or extract JSON values by property-path rules in a single streaming pass from Utf8JsonReader to Utf8JsonWriter, without deserialization or a DOM. true + 1.0.2 json;system.text.json;utf8jsonreader;masking;redaction;pii;extraction;streaming;logging - - - From a689aada167236b58fa193e13c35460f9050f9e1 Mon Sep 17 00:00:00 2001 From: VasiliyF <5789590+vfofanov@users.noreply.github.com> Date: Mon, 5 Oct 2026 10:20:28 +0200 Subject: [PATCH 07/10] Write the 2.0 breaking changes and the migration table - CHANGELOG: changed-breaking items 13-27 for the Core move, the hash, flags, renames and TFMs, plus the missing IgnoreNullsJsonTokenType line - docs/migrating-to-2.0.md: rename table, hash and comment changes, the .NET 10 SDK requirement coming with Observer 1.1 / 1.2; doc-tested --- CHANGELOG.md | 50 +++++++--- .../DocSnippetTests.cs | 2 +- ...ystem.Text.Json.Observer.Docs.Tests.csproj | 1 + docs/migrating-to-2.0.md | 91 +++++++++++++++++++ 4 files changed, 132 insertions(+), 12 deletions(-) create mode 100644 docs/migrating-to-2.0.md diff --git a/CHANGELOG.md b/CHANGELOG.md index a720b4c..dce716e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,31 +4,41 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [2.0.0] - 2026-10-03 +## [2.0.0] - Unreleased + +**Build requirement (coming in DragoAnt.Observer 1.1 as a warning, 1.2 as an error):** the .NET 10 SDK. 2.0 builds with any .NET 8+ SDK; apps may keep targeting `net8.0`. See [Migrating to 2.0](./docs/migrating-to-2.0.md). ### Added +- **[DragoAnt.Observer.Core](https://www.nuget.org/packages/DragoAnt.Observer.Core) and [DragoAnt.Observer.Abstractions](https://www.nuget.org/packages/DragoAnt.Observer.Abstractions)** carry the format-neutral types (see *Changed — breaking*). The package imports the `DragoAnt.Observer` namespace into every C# file; set `false` to opt out. +- **`MaskResult.Flags`** (`MaskFlags`): `InputTruncated`, `OutputCapped`, `ValueCut`, `TrailingData`, `InvalidUtf8Replaced`, `Depth` say why a result is not a clean `Masked`. +- **Comment policy:** `JsonObserverOptions.Comments` (`CommentPolicy.AllowList` by default, `BlockList`, `MaskAll`, `Mask(tag)`, `DropAll`) and `.Comment(CommentKind, CommentRule)` after any rule decide what happens to comments in the input; a kept comment of a masked value is written masked unless `CommentRules.Raw`. Comments are still dropped by default. +- **`Mask(ReadOnlySpan, IBufferWriter)`, `Mask(ReadOnlySpan, IBufferWriter)`, `Read(ReadOnlySpan, …)`**, a non-generic `JsonObserver.Read(...)` that reports status and flags, and `JsonObserver.FromShape(...)`. +- **`ValuePolicy.Tagged(tag)`**: masks every unmatched value with the call's strategy, for example hashed. +- **`HashKeyId`** and `WithBase64HashKey(key)` (takes a Microsoft `HmacRedactorOptions.Key`); `Mask(strategy, tag)` gives one rule its own strategy; `JsonShapeOptions.UnknownTag`; `MaskTag.Create` and `WithKey`. - **UTF-8 API:** `JsonObserver.Mask(ReadOnlySpan, IBufferWriter, JsonObserverOptions?)` masks bytes into a reusable writer. With constant or tag rules it allocates a small constant amount per call, whatever the body size. -- **Never-throw contract with `MaskResult`:** every `Mask` and `Read` overload reports `MaskStatus` (`Masked`, `Truncated`, `Invalid`, `NotJson`), the bytes written and the input offset where reading stopped. On cut-off or invalid input the output is the masked part read so far, with every open object and array closed, so it is always valid JSON and never holds a masked value in clear. -- **`JsonObserverOptions`** for both APIs: `MaxOutputBytes`, `MaxValueBytes`, `MaxDepth`, `RelaxedEscaping`, `HashKey`, `MaskStrategy`, `IgnoreNulls`, `Indented`. -- **Tag rules:** `MaskAny(MaskTag)` with `Full`, `Last4`, `Hash` (keyed HMAC-SHA256) and `Omit`, served by a replaceable `Utf8MaskStrategy`. +- **Never-throw contract with `MaskResult`:** every `Mask` and `Read` overload reports `MaskStatus` (`Masked`, `Truncated`, `Invalid`, `Unrecognized`), the bytes written and the input offset where reading stopped. On cut-off or invalid input the output is the masked part read so far, with every open object and array closed, so it is always valid JSON and never holds a masked value in clear. +- **`JsonObserverOptions`** for both APIs: `MaxOutputBytes`, `MaxValueBytes`, `MaxDepth`, `RelaxedEscaping`, `HashKey`, `HashKeyId`, `Strategy`, `IgnoreNulls`, `Comments`, `Indented`. +- **Tag rules:** `Mask(MaskTag)` with `Full`, `Last4`, `Hash` (keyed HMAC-SHA256) and `Null`, served by a replaceable `ValueMaskStrategy`. - **Allow-lists from your types:** `JsonShape.FromTypeInfo(...)` and `JsonObserver.FromShape(...)`; anything the shape does not describe is masked (`UnknownMemberPolicy`). - **UTF-8 name matching:** property names are compared on their UTF-8 bytes, without creating strings. - **Verbatim pass-through:** unmasked numbers and strings are copied as written (`1.50`, `1e400`, `-0`, 20-digit integers). -- `PropertyPath.Length`. +- `DataPath.Length`. - **`ReadOnlySequence` input:** `Mask(in ReadOnlySequence, …)` and `Read(in ReadOnlySequence, …)` mask a payload held in several buffers, for example from a `PipeReader`, without copying it into one; the output is byte for byte what the span overload writes, however the bytes are split. - **No per-call allocation on the bytes API:** the writers are reused per thread, so a warm `Mask`/`Read` of bytes with constant or tag rules allocates nothing (pinned by a test for the span, sequence, ignore-nulls and read paths). -- **`Explain(path)`:** `JsonObserver.Explain("lines[0].qty", JsonTokenType.Number)` returns a `JsonPathExplanation` naming the rule or policy that handles the value, its action, the outcome (`Unchanged`, `Masked`, `Read`, `Custom`, `Invalid`) and one step per level, for rule-based and shape observers. +- **`Explain(path)`:** `JsonObserver.Explain("lines[0].qty", ValueKind.Number)` returns a `PathExplanation` naming the rule or policy that handles the value, its action, the outcome (`Unchanged`, `Masked`, `Read`, `Custom`, `Invalid`) and one step per level, for rule-based and shape observers. - **Classified tags:** `MaskTag` carries an optional `Key` (a data classification, a redactor name) and `MaskKind.Custom`, so a strategy maps its own taxonomy without casting enum values; `MaskTag.Custom(key)`, `TryGetKey`. -- **Strategies see where a value is:** `Utf8MaskStrategy.Mask(in Utf8MaskContext, JsonWriter)` receives the value, its JSON type, the tag, the options, the property name and the whole path without allocating. Both `Mask` overloads are virtual; a strategy overrides the one it needs. +- **Strategies see where a value is:** `ValueMaskStrategy.Mask(in MaskContext, MaskValueWriter)` receives the value, its kind, the tag, the options, the property name and the whole path without allocating. Both `Mask` overloads are virtual; a strategy overrides the one it needs. - **`JsonWriter` span overloads:** `WriteStringValue(ReadOnlySpan)`, `WritePropertyName(ReadOnlySpan)`, `WriteBase64StringValue(ReadOnlySpan)` and `WriteNumberValue(double)`. -- **Array indices in paths:** `PropertyPath.ToString()` renders `items[2].sku` (names that need it as `['a.b']`); `TryGetArrayIndex`, `IsArrayItem` and `TryGetPropertyNameUtf8` give zero-allocation access. -- **Case sensitivity:** `JsonObserverOptions.PropertyNameCaseInsensitive` (default `true`) makes rules, `PropMatches` tests and shapes match names exactly when set to `false`, the way the serializer does; `JsonShapeOptions.PropertyNameCaseInsensitive` and `JsonShapeOptions.FromSerializerOptions(...)` set it for one shape observer; `PropertyPath.PropertyNameCaseInsensitive` tells a custom rule. -- **Metadata on shapes:** `JsonShape.Members` lists `JsonShapeProperty` items with the `JsonPropertyInfo`, CLR member, property and declaring type, `IsRequired`, `IsNullable` and custom attributes; nodes carry their `JsonTypeInfo`/`ClrType`; nodes and properties have `Annotations` for integrations; `FromTypeInfo` takes an `annotate` callback, and `FindMember` looks a property up by its UTF-8 name. On .NET 8, source-generated metadata has no attributes or reference-type nullability. +- **Array indices in paths:** `DataPath.ToString()` renders `items[2].sku` (names that need it as `['a.b']`); `TryGetItemIndex`, `IsItem` and `TryGetName` give zero-allocation access. +- **Case sensitivity:** `JsonObserverOptions.NameCaseInsensitive` (default `true`) makes rules, `Names` tests and shapes match names exactly when set to `false`, the way the serializer does; `JsonShapeOptions.NameCaseInsensitive` and `JsonShapeOptions.FromSerializerOptions(...)` set it for one shape observer; `JsonValueContext.Options` tells a custom rule. +- **Metadata on shapes:** `JsonShape.Members` lists `JsonShapeProperty` items with the `JsonPropertyInfo`, CLR member, property and declaring type, `IsRequired`, `IsNullable` and custom attributes; nodes carry their `JsonTypeInfo`/`ClrType`; nodes and properties have `Annotations` for integrations; `FromTypeInfo` takes an `annotate` callback. On .NET 8, source-generated metadata has no attributes or reference-type nullability. - **New package `DragoAnt.System.Text.Json.Observer.Http`:** `JsonBodyLoggingHandler` logs masked `HttpClient` request and response bodies; register it with `AddJsonBodyLogging`, pick maskers per body model type with `IJsonBodyMaskerProvider`, and attach model types per request with `WithBodyLogging()`. ### Changed — breaking +Items 13 and later are new since the 2.0.0 previews; [Migrating to 2.0](./docs/migrating-to-2.0.md) has the rename table. + 1. **The default policy masks everything a rule does not name.** `AllowList` (still the default) writes every string, number **and boolean** as `"***"`; in 1.x it wrote `"#str#*****"` / `"#number#*****"` and kept booleans. The 1.x output is available as the obsolete `LegacyAllowList`. 2. **`Mask(string)` never throws.** It runs the same UTF-8 path as the bytes API and produces the same output for the same text: comments are skipped (never written), trailing commas are accepted, non-ASCII and HTML characters are written unescaped (`RelaxedEscaping`), invalid or cut-off text yields its masked prefix. `Mask(string, out MaskResult, options)` reports the status. The `JsonReaderOptions`, `JsonWriterOptions`, `ignoreNulls` and `ignoreComments` parameters are gone: use `JsonObserverOptions` (`IgnoreNulls`, `Indented`, `MaxDepth`). 3. **`Read(...)` never throws and returns a `MaskResult`** instead of `void`; `Read(byte[])` became `Read(ReadOnlySpan)`. @@ -39,9 +49,25 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 8. **`JsonWriter` can no longer be derived from outside the library**, `JsonWriter.FromUtf8JsonWriter` and `JsonWriter.Empty` are removed, and `WriteCommentValue` is gone (comments are never written). 9. **Internal now:** `JsonObserverException`, `PropertyPathMatch`, `JsonPropertyMatchDelegate`, `JsonPropertyPathMatchDelegate`, and the constructors of `JsonObjBuilder`, `JsonArrayBuilder`, `JsonValuePolicyBuilder` and their rule builders (start rules with `Match`). 10. **A UTF-8 byte order mark at the start of the input is skipped.** -11. **Read rules no longer decide what is written.** `ReadStr`, `ReadInt`, `ReadLong`, `ReadDecimal`, `ReadBool` and `ReadRaw` hand the value to the context; the next rule on the same match or the default policy writes it, so under `AllowList` a read value is `"***"`, not clear text. Chain `.Unmasked()` to keep it clear, or a mask method to read and mask one match: `Match("ssn").ReadStr(f).MaskAny(MaskTag.Last4)`. A read rule runs wherever it stands among the rules, so `Match("ssn").MaskAny(…).Match("ssn").ReadStr(f)` reads too. `Explain` reports `Read` only for a value read and written unchanged. +11. **Read rules no longer decide what is written.** `ReadStr`, `ReadInt`, `ReadLong`, `ReadDecimal`, `ReadBool` and `ReadRaw` hand the value to the context; the next rule on the same match or the default policy writes it, so under `AllowList` a read value is `"***"`, not clear text. Chain `.Unmasked()` to keep it clear, or a mask method to read and mask one match: `Match("ssn").ReadStr(f).Mask(MaskTag.Last4)`. A read rule runs wherever it stands among the rules, so `Match("ssn").Mask(…).Match("ssn").ReadStr(f)` reads too. `Explain` reports `Read` only for a value read and written unchanged. 12. **A masking function receives the whole value, and mask output is never cut by `MaxValueBytes`.** A `Last4`-style function sees the real last characters (`***4444`, not the ones at the cap), and `MaskTag.Hash`, a function's result and the `AllowList` stars are written whole, with status `Masked`. A function therefore decodes a long value in full. +13. **Format-neutral types live in `DragoAnt.Observer` (DragoAnt.Observer.Core and .Abstractions)** under their final names: `MaskKind`, `MaskTag` (Abstractions); `MaskResult`, `MaskStatus`, `PathExplanation` (was `JsonPathExplanation`), `PathOutcome` (was `JsonPathOutcome`), `UnknownMemberPolicy`, `NoContext` (was `JsonObserveringEmptyContext`), `DataPath` (was `PropertyPath`), `NameMatch` and `Names` (were `PropMatchingStrategy` and `PropMatches`), `ValueMaskStrategy` (was `Utf8MaskStrategy`) and `MaskContext` (was `Utf8MaskContext`). The `.Strategies` namespace is gone; `StringMaskingStrategy` moved to the root namespace. +14. **`MaskStatus.NotJson` is `Unrecognized`**, and `Masked` now means no `Flags`: data after the root (`{"a":1}{"b":2}`, formerly `Masked`) and invalid UTF-8 replaced by U+FFFD report `Truncated` with the whole document written. +15. **The hash is Microsoft's `HmacRedactor` format:** HMAC-SHA256 over the value's UTF-16 text, the first 16 bytes in base64 (24 characters), after `":"` when a key id is set; `""` hashes to `""`; numbers and booleans hash their literal, so `1` and `"1"` hash alike. The 1.x/preview `hash:` + 16 hex characters over UTF-8 is gone, so stored hashes change. +16. **`MaskKind.Omit` is `MaskKind.Null`** (it writes `null` and keeps the property). +17. **Options and results have init-only properties, no positional constructors:** `new JsonObserverOptions { MaxValueBytes = 8 }`. `JsonObserverOptions` derives from `ObserverOptions`; `MaskStrategy` is `Strategy`, `PropertyNameCaseInsensitive` is `NameCaseInsensitive`. Likewise `JsonShapeOptions` (`NameCaseInsensitive`; `FromSerializerOptions(options)` takes the serializer options only), `MaskResult`, `PathExplanation`, `MaskTag` (factories: `Full`, `Last4`, `Hash`, `Null`, `Custom(key)`, `Create(kind, key)`) and Http's `JsonBodyLoggingContext`. +18. **Strategies are format-neutral:** override `ValueMaskStrategy.Mask(in MaskContext, MaskValueWriter)`; the five-argument overload is gone. The context has `Kind` (`ValueKind`, was `TokenType`), `Name` (was `PropertyName`) and `ValueIndex`; `MaskValueWriter` validates numbers (an invalid literal writes `"***"` and makes the call `Invalid`). +19. **One custom-rule delegate:** `JsonValueRule(ref JsonValueContext c)` replaces `JsonObserverDelegate` and `JsonObserverValueDelegate` in `MaskValue`, `Obj(rule)` and `Array(rule)`; `c.WriteDefault()` applies the enclosing default policy. A custom default policy is `JsonValuePolicy.Custom(rule)`. +20. **Default policies are `ValuePolicy` values that work for any context type:** `JsonObserverValuePolicies` and `JsonObserverValuePolicies` are gone — use `ValuePolicy.BlockList` / `AllowList` / `NullList` (`using static DragoAnt.Observer.ValuePolicy;`). `Relative(...)` is `JsonValuePolicy.AnyDepth(...)`, its builder `JsonAnyDepthBuilder`. +21. **One masking verb:** `MaskAny`, `MaskStr` and `MaskRawValue` are `Mask(...)` — `Mask(MaskTag)`, `Mask(strategy, tag)`, `Mask("***")`, `Mask(regex)`, `Mask(value => …)`, `Mask((value, context) => …)`. `null` stays `null` by default; `Mask(…, MaskNulls.Mask)` passes it to the function (the former `MaskStr`). A function receives a string decoded (the raw-text variant is gone). +22. **`Match(a, b)` with several names is `Path(a, b)`**; `Match(name)` tests one level. +23. **`LegacyAllowList` is removed.** +24. **`Explain(path, ValueKind)`** takes a `ValueKind` (was a `JsonTokenType`), returns `PathExplanation`, and rejects wildcards and tags (`*`, `:`); explanations name rules `Mask(...)`, `Path(...)` and `AnyDepth`. +25. **Engine details are internal:** `JsonShape.FindMember` and the case flag of the path (read `JsonValueContext.Options.NameCaseInsensitive`). +26. **`net9.0` is dropped;** the package targets `net8.0` and `net10.0` (a `net9.0` app uses the `net8.0` assets). +27. **`JsonWriter.IgnoreNullsJsonTokenType`**, a public nested enum in 1.0.2, is removed. + ### Fixed - **`PropMatches.Regex` follows `PropertyNameCaseInsensitive`:** under the default case-insensitive option it now also matches names that differ in case only (`DRiverLicensE` for `^driverLicense$`), as every other matcher does; a `Regex` built with `RegexOptions.IgnoreCase` ignores case under either option. A custom name test can follow the option through the new `PropMatchingStrategy(Func)` constructor. @@ -52,6 +78,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Observer.Http 2.0.0 — breaking (since the preview builds) - `JsonBodyOutcome.Timeout` is replaced by `Canceled`: every cancellation, an `HttpClient.Timeout` included, is logged as `Canceled`. +- `JsonBodyLoggingContext` uses `init` properties (`RequestType`, `ResponseType`, `Operation`). +- The HTTP handler never writes a comment of a body (`CommentPolicy.DropAll`). - `JsonBodyLogEntry` uses `init` properties and adds `Operation`, `RequestBodyStatus` and `ResponseBodyStatus`. - New `JsonBodyStatus` type; a body that could not be logged as JSON is a marker such as `[body not JSON]` or `[invalid JSON]`, and a truncated body is flagged. - `JsonBodyLoggingHandler` is sealed. diff --git a/DragoAnt.System.Text.Json.Observer.Docs.Tests/DocSnippetTests.cs b/DragoAnt.System.Text.Json.Observer.Docs.Tests/DocSnippetTests.cs index 2b1f0d1..d5dc128 100644 --- a/DragoAnt.System.Text.Json.Observer.Docs.Tests/DocSnippetTests.cs +++ b/DragoAnt.System.Text.Json.Observer.Docs.Tests/DocSnippetTests.cs @@ -54,7 +54,7 @@ public void Snippet_CompilesRunsAndPrintsItsOutput(string document, int index) } } - private static readonly string[] Documents = ["README.md", "package.readme.md"]; + private static readonly string[] Documents = ["README.md", "package.readme.md", "migrating-to-2.0.md"]; private sealed record Snippet(int Line, string Code, string? ExpectedOutput); diff --git a/DragoAnt.System.Text.Json.Observer.Docs.Tests/DragoAnt.System.Text.Json.Observer.Docs.Tests.csproj b/DragoAnt.System.Text.Json.Observer.Docs.Tests/DragoAnt.System.Text.Json.Observer.Docs.Tests.csproj index 0af5ecc..13d3005 100644 --- a/DragoAnt.System.Text.Json.Observer.Docs.Tests/DragoAnt.System.Text.Json.Observer.Docs.Tests.csproj +++ b/DragoAnt.System.Text.Json.Observer.Docs.Tests/DragoAnt.System.Text.Json.Observer.Docs.Tests.csproj @@ -9,5 +9,6 @@ + diff --git a/docs/migrating-to-2.0.md b/docs/migrating-to-2.0.md new file mode 100644 index 0000000..fbfadf1 --- /dev/null +++ b/docs/migrating-to-2.0.md @@ -0,0 +1,91 @@ +# Migrating to 2.0 + +2.0 moves the format-neutral types into [DragoAnt.Observer.Core](https://www.nuget.org/packages/DragoAnt.Observer.Core) (namespace `DragoAnt.Observer`, imported into every C# file by the package), renames the builder verbs once, and changes a few defaults. Every break is listed in the [changelog](../CHANGELOG.md); this page is the lookup table. + +**Build requirement (coming in DragoAnt.Observer 1.1 as a warning, 1.2 as an error):** the .NET 10 SDK. 2.0 itself builds with any .NET 8+ SDK, and apps may keep targeting `net8.0`; `true` will keep the check a warning. + +## Renames + +| 1.0.2 | 2.0 | +| --- | --- | +| `using DragoAnt.System.Text.Json.Observer.Strategies;` | nothing — `DragoAnt.Observer` is imported by the package | +| `using static …JsonObserverValuePolicies;` | `using static DragoAnt.Observer.ValuePolicy;` and `using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy;` | +| `JsonObserverValuePolicies.BlockList` / `AllowList` / `NullList` | `ValuePolicy.BlockList` / `AllowList` / `NullList` (any context type) | +| `Relative(rules => …, policy)` | `AnyDepth(rules => …, policy)` | +| `Match("card", "number")` (several names) | `Path("card", "number")`; `Match(name)` stays | +| `MaskAny(x)` | `Mask(x)` | +| `MaskStr(x)` | `Mask(x, MaskNulls.Mask)` | +| `MaskRawValue(f)` | `Mask(f, MaskNulls.Mask)` — the function gets the decoded string | +| `MaskTag.Omit` / `MaskKind.Omit` | `MaskTag.Null` / `MaskKind.Null` | +| `new MaskTag(kind, key)` | `MaskTag.Create(kind, key)` | +| `PropMatches.X(…)` / `PropMatchingStrategy` | `Names.X(…)` / `NameMatch` | +| `PropertyPath` (`GetPropertyName`, `GetPropertyNameReverse`, `TryGetPropertyNameUtf8`, `TryGetArrayIndex`, `IsArrayItem`) | `DataPath` (`GetName`, `GetNameFromEnd`, `TryGetName`, `TryGetItemIndex`, `IsItem`) | +| `JsonObserveringEmptyContext` | `NoContext` | +| `JsonPathExplanation` / `JsonPathOutcome` | `PathExplanation` / `PathOutcome` | +| `MaskStatus.NotJson` | `MaskStatus.Unrecognized` | +| `Utf8MaskStrategy`, `Utf8MaskContext`, `Mask(in Utf8MaskContext, JsonWriter)` | `ValueMaskStrategy`, `MaskContext`, `Mask(in MaskContext, MaskValueWriter)` | +| `context.TokenType` / `context.PropertyName` | `context.Kind` (`ValueKind`) / `context.Name` | +| `JsonObserverOptions(MaxValueBytes: 8)` | `new JsonObserverOptions { MaxValueBytes = 8 }` | +| `MaskStrategy` / `PropertyNameCaseInsensitive` | `Strategy` / `NameCaseInsensitive` | +| `new JsonShapeOptions(UnknownMemberPolicy.Descend)` | `new JsonShapeOptions { Unknown = UnknownMemberPolicy.Descend }` | +| `Explain(path, JsonTokenType.Number)` | `Explain(path, ValueKind.Number)` | +| custom rule `(ref Utf8JsonReader r, JsonWriter w, T c, ref PropertyPath p) => …` | `(ref JsonValueContext c) => …` with `c.Reader`, `c.Writer`, `c.Context`, `c.Path`, `c.WriteDefault()` | +| a delegate as the default policy | `JsonValuePolicy.Custom(rule)` | +| `LegacyAllowList` | removed: `AllowList`, or `ValuePolicy.Tagged(tag)` | + +## Before and after + + +```csharp +// 1.0.2 +var masker = JsonObserver.Obj(Relative(rules => rules + .Match("password").MaskAny("***") + .Match("card", "number").MaskAny(MaskTag.Last4), + BlockList)); +``` + +```csharp +using DragoAnt.System.Text.Json.Observer; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; + +var masker = JsonObserver.Obj(AnyDepth(rules => rules + .Match("password").Mask("***") + .Path("card", "number").Mask(MaskTag.Last4), + BlockList)); + +Console.WriteLine(masker.Mask("""{"password":"s3cret","card":{"number":"4111111111111111"}}""")); +// Output: +// {"password":"***","card":{"number":"***1111"}} +``` + +## The hash changes + +`MaskTag.Hash` now writes exactly what Microsoft's `HmacRedactor` (Microsoft.Extensions.Compliance.Redaction) writes for the same key: HMAC-SHA256 over the value's UTF-16 text, the first 16 bytes in base64 (24 characters), after `":"` when `HashKeyId` is set; `""` hashes to `""`. Hashes stored from the previews (`hash:` and 16 hex characters) do not match any more. Pass the `HmacRedactorOptions.Key` with `WithBase64HashKey(key)` to correlate with logs Microsoft's redactor wrote: + +```csharp +using DragoAnt.System.Text.Json.Observer; +using static DragoAnt.Observer.ValuePolicy; +using static DragoAnt.System.Text.Json.Observer.JsonValuePolicy; + +var options = new JsonObserverOptions { HashKeyId = 7 } + .WithBase64HashKey("AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIjJCUmJygpKissLS4vMDEyMzQ1Njc4OTo7PD0+Pw=="); +var masker = JsonObserver.Obj(AnyDepth(rules => rules.Match("email").Mask(MaskTag.Hash), BlockList)); + +var hash = masker.Mask("""{"email":"a@b.c"}""", options)!; +Console.WriteLine(hash.StartsWith("""{"email":"7:""") && hash.Length == """{"email":"7:"}""".Length + 24); +// Output: +// True +``` + +## Results report why + +`MaskResult.Flags` tells why a result is `Truncated` or `Invalid`. Data after the root (`{"a":1}{"b":2}`) and invalid UTF-8 replaced by U+FFFD used to report `Masked`; they report `Truncated` now, with the whole document written. + +## Comments + +Comments in the input are still dropped by default. `new JsonObserverOptions { Comments = CommentPolicy.BlockList }` keeps them (a comment of a masked value is written masked), `.Comment(CommentKind.Inline, CommentRules.Keep)` after a rule keeps those of the values it matches, and `CommentPolicy.DropAll` drops them whatever a rule says. Details: [comments](https://github.com/DragoAnt/Observer/blob/main/docs/comments.md). + +## Target frameworks + +`net9.0` is dropped: the package targets `net8.0` and `net10.0`, and a `net9.0` app uses the `net8.0` assets. From 16c5eaf3051e744d5ab8b44f4ccfba843bc8869a Mon Sep 17 00:00:00 2001 From: VasiliyF <5789590+vfofanov@users.noreply.github.com> Date: Mon, 5 Oct 2026 11:16:23 +0200 Subject: [PATCH 08/10] Gate tag and hash rules at 0 B and pack on CI - AllocationTests add a Last4 + keyed Hash observer to the 0 B budgets - CI packs the solution, which runs package validation against 1.0.2 --- .github/workflows/ci.yml | 5 +++++ .../AllocationTests.cs | 11 ++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 30751ca..333dd9b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -96,6 +96,11 @@ jobs: dotnet test --solution "$SOLUTION" -c "$CONFIGURATION" --no-build --no-progress \ --results-directory "$RESULTS_DIR" --report-xunit-trx --report-xunit-junit --coverage --coverage-output-format cobertura + - name: Pack (package validation against 1.0.2) + run: | + # shellcheck disable=SC2086 + dotnet pack "$SOLUTION" -c "$CONFIGURATION" --no-build -o "$GITHUB_WORKSPACE/packages" $BUILD_PROPS + - name: Coverage if: ${{ !cancelled() && hashFiles('TestResults/**/*.cobertura.xml') != '' }} run: | diff --git a/DragoAnt.System.Text.Json.Observer.Tests.Shared/AllocationTests.cs b/DragoAnt.System.Text.Json.Observer.Tests.Shared/AllocationTests.cs index 3c40640..4d327c7 100644 --- a/DragoAnt.System.Text.Json.Observer.Tests.Shared/AllocationTests.cs +++ b/DragoAnt.System.Text.Json.Observer.Tests.Shared/AllocationTests.cs @@ -19,12 +19,20 @@ public abstract class AllocationTests b => b.Match("password").Mask("***"), ValuePolicy.BlockList)); + private static readonly JsonObserver Tags = JsonObserver.Any( + _ => { }, + _ => { }, + AnyDepth(b => b.Match("password").Mask(MaskTag.Hash).Path("card", "number").Mask(MaskTag.Last4), BlockList)); + private static readonly JsonObserverOptions IgnoreNulls = new() { IgnoreNulls = true }; + private static readonly JsonObserverOptions Keyed = new JsonObserverOptions { HashKeyId = 3 } + .WithBase64HashKey("AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIjJCUmJygpKissLS4vMDEyMzQ1Njc4OTo7PD0+Pw=="); + public static TheoryData Budgets() { var data = new TheoryData(); - foreach (var api in new[] { "span", "sequence", "ignore-nulls", "read" }) + foreach (var api in new[] { "span", "sequence", "ignore-nulls", "read", "tag-hash" }) { data.Add("flat", 1024, api); data.Add("flat", 64 * 1024, api); @@ -54,6 +62,7 @@ void Call() "span" => Observer.Mask(utf8, output), "sequence" => Observer.Mask(sequence, output), "ignore-nulls" => Observer.Mask(utf8, output, IgnoreNulls), + "tag-hash" => Tags.Mask(utf8, output, Keyed), _ => Reader.Read(utf8, NoContext.Instance), }; } From cabcbb0223d702060a5f5244a90f373faf3b4c20 Mon Sep 17 00:00:00 2001 From: VasiliyF <5789590+vfofanov@users.noreply.github.com> Date: Mon, 5 Oct 2026 11:19:53 +0200 Subject: [PATCH 09/10] Let a PR pin the Observer ref CI builds from The picker looked only for an Observer branch named like this one, so a PR whose Observer branch has a different name built against main. An 'Observer-Ref:' line in the PR body now wins, then the same branch name, then main. --- .github/workflows/ci.yml | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 333dd9b..982c4a6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,8 @@ concurrency: cancel-in-progress: ${{ github.event_name == 'pull_request' }} # Until DragoAnt.Observer.Core 1.0.0 is on nuget.org this job builds it from source: it checks out -# DragoAnt/Observer next to this repository (the branch of the same name when it exists, else main) +# DragoAnt/Observer next to this repository (a PR body line `Observer-Ref: `, else the branch of +# the same name, else main) # and builds with DragoAntObserverLocal=true. Switch back to the shared dotnet-build.yml caller once # the package is published. jobs: @@ -49,12 +50,16 @@ jobs: working-directory: ${{ github.workspace }} env: BRANCH: ${{ github.head_ref || github.ref_name }} + PR_BODY: ${{ github.event.pull_request.body }} run: | - if git ls-remote --exit-code --heads https://github.com/DragoAnt/Observer.git "$BRANCH" > /dev/null; then - ref="$BRANCH" - else - ref=main - fi + pinned=$(printf '%s\n' "$PR_BODY" | tr -d '\r' | sed -n 's/^Observer-Ref: *\([A-Za-z0-9._/-]*\) *$/\1/p' | head -n 1) + ref=main + for candidate in "$pinned" "$BRANCH"; do + if [ -n "$candidate" ] && git ls-remote --exit-code --heads https://github.com/DragoAnt/Observer.git "$candidate" > /dev/null; then + ref="$candidate" + break + fi + done echo "ref=$ref" >> "$GITHUB_OUTPUT" echo "Building DragoAnt.Observer.Core from DragoAnt/Observer@$ref" >> "$GITHUB_STEP_SUMMARY" From 701bd0101ae981517421885eede368d12d5f5f08 Mon Sep 17 00:00:00 2001 From: VasiliyF <5789590+vfofanov@users.noreply.github.com> Date: Mon, 5 Oct 2026 11:23:09 +0200 Subject: [PATCH 10/10] Let MSBuildKit version the CI build A forced 0.0.0-ci version made the assembly 0.0.0.0, so package validation rejected it against the 1.0.2 baseline. The kit derives the CI version from VersionPrefix, as the shared workflow does. --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 982c4a6..0440850 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,7 +34,7 @@ jobs: RESULTS_DIR: ${{ github.workspace }}/TestResults COVERAGE_DIR: ${{ github.workspace }}/coverage COVERAGE_THRESHOLD: 70 - BUILD_PROPS: -p:ContinuousIntegrationBuild=true -p:Version=0.0.0-ci.${{ github.run_number }} -p:DragoAntObserverLocal=true + BUILD_PROPS: -p:ContinuousIntegrationBuild=true -p:DragoAntObserverLocal=true defaults: run: working-directory: Extensions.System.Text.Json