From 4718904709e67d5ba592cf9d8e63910ff9338421 Mon Sep 17 00:00:00 2001 From: Alan Deutscher Date: Wed, 2 Sep 2026 23:04:31 -0700 Subject: [PATCH 01/18] Initial prompt output --- README.md | 4 +- RedShirt.Example.JobWorker.slnx | 9 + docs/bar-connector.md | 59 +++++ .../Exceptions/BarException.cs | 34 +++ .../Exceptions/BarRecordNotFoundException.cs | 9 + .../Models/CreateBarConnectorRequest.cs | 6 + .../Models/CreateBarConnectorResponse.cs | 8 + .../Models/GetBarConnectorResponse.cs | 8 + ...ample.JobWorker.Connectors.Bar.Core.csproj | 9 + .../Services/IBarConnector.cs | 15 ++ .../Clients/BarApiClient.cs | 129 ++++++++++ .../Clients/BarApiClientHandler.cs | 36 +++ .../Exceptions/BarRateLimitedException.cs | 16 ++ .../Exceptions/BarReasonToWaitException.cs | 20 ++ .../BarTemporarilyUnavailableException.cs | 15 ++ .../Exceptions/BarUnauthorizedException.cs | 17 ++ .../Extensions/ServiceCollectionExtensions.cs | 49 ++++ .../Factories/BarApiClientFactory.cs | 25 ++ .../Globals.cs | 3 + .../Models/BarExceptionArbiterReport.cs | 12 + .../Requests/InternalBarCreateRequest.cs | 6 + .../Responses/InternalBarCreateResponse.cs | 8 + .../Responses/InternalBarGetResponse.cs | 8 + ...orker.Connectors.Bar.Implementation.csproj | 23 ++ .../Services/BarConnector.cs | 122 +++++++++ ...BarApiRequestHandlerRetryWrapperService.cs | 237 ++++++++++++++++++ .../Resilience/BarExceptionArbiterService.cs | 124 +++++++++ .../Resilience/BarRetryWrapperService.cs | 171 +++++++++++++ .../Enums/TokenCacheState.cs | 22 ++ .../Exceptions/OAuthRequestException.cs | 24 ++ .../Exceptions/OAuthRequestJsonException.cs | 16 ++ .../Models/OAuthClientCredentialsRequest.cs | 36 +++ .../Models/OAuthTokenCacheResponse.cs | 21 ++ .../Models/OAuthTokenResponse.cs | 14 ++ ...le.JobWorker.Connectors.Common.Http.csproj | 19 ++ .../Services/OAuthTokenCache.cs | 96 +++++++ .../Services/OAuthTokenSource.cs | 162 ++++++++++++ .../JobLogicRunner.cs | 11 +- ...dShirt.Example.JobWorker.Core.Logic.csproj | 1 + .../Extensions/ServiceCollectionExtensions.cs | 5 +- .../RedShirt.Example.JobWorker.csproj | 1 + .../Tests/JobLogicRunnerTests.cs | 13 +- test/local/docker-compose.yaml | 26 +- test/local/make-local-aws-resources.sh | 10 + test/local/readme.md | 129 ++++++++-- .../bar-rotate-oauth-credentials.sh | 104 ++++++++ .../wiremock-bar/bar-set-ssm-oauth-secret.sh | 21 ++ test/local/set-azure-key-vault-secrets.py | 2 + .../wiremock/mappings/get-api-bar-404.json | 22 ++ .../wiremock/mappings/get-api-bar-by-id.json | 21 ++ .../local/wiremock/mappings/post-api-bar.json | 26 ++ .../post-oauth-token-unauthorized.json | 18 ++ .../wiremock/mappings/post-oauth-token.json | 38 +++ .../mappings/unauthorized-api-bar.json | 17 ++ 54 files changed, 2037 insertions(+), 20 deletions(-) create mode 100644 docs/bar-connector.md create mode 100644 src/RedShirt.Example.JobWorker.Connectors.Bar.Core/Exceptions/BarException.cs create mode 100644 src/RedShirt.Example.JobWorker.Connectors.Bar.Core/Exceptions/BarRecordNotFoundException.cs create mode 100644 src/RedShirt.Example.JobWorker.Connectors.Bar.Core/Models/CreateBarConnectorRequest.cs create mode 100644 src/RedShirt.Example.JobWorker.Connectors.Bar.Core/Models/CreateBarConnectorResponse.cs create mode 100644 src/RedShirt.Example.JobWorker.Connectors.Bar.Core/Models/GetBarConnectorResponse.cs create mode 100644 src/RedShirt.Example.JobWorker.Connectors.Bar.Core/RedShirt.Example.JobWorker.Connectors.Bar.Core.csproj create mode 100644 src/RedShirt.Example.JobWorker.Connectors.Bar.Core/Services/IBarConnector.cs create mode 100644 src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Clients/BarApiClient.cs create mode 100644 src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Clients/BarApiClientHandler.cs create mode 100644 src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Exceptions/BarRateLimitedException.cs create mode 100644 src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Exceptions/BarReasonToWaitException.cs create mode 100644 src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Exceptions/BarTemporarilyUnavailableException.cs create mode 100644 src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Exceptions/BarUnauthorizedException.cs create mode 100644 src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Extensions/ServiceCollectionExtensions.cs create mode 100644 src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Factories/BarApiClientFactory.cs create mode 100644 src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Globals.cs create mode 100644 src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Models/BarExceptionArbiterReport.cs create mode 100644 src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Models/Requests/InternalBarCreateRequest.cs create mode 100644 src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Models/Responses/InternalBarCreateResponse.cs create mode 100644 src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Models/Responses/InternalBarGetResponse.cs create mode 100644 src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/RedShirt.Example.JobWorker.Connectors.Bar.Implementation.csproj create mode 100644 src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Services/BarConnector.cs create mode 100644 src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Services/Resilience/BarApiRequestHandlerRetryWrapperService.cs create mode 100644 src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Services/Resilience/BarExceptionArbiterService.cs create mode 100644 src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Services/Resilience/BarRetryWrapperService.cs create mode 100644 src/RedShirt.Example.JobWorker.Connectors.Common.Http/Enums/TokenCacheState.cs create mode 100644 src/RedShirt.Example.JobWorker.Connectors.Common.Http/Exceptions/OAuthRequestException.cs create mode 100644 src/RedShirt.Example.JobWorker.Connectors.Common.Http/Exceptions/OAuthRequestJsonException.cs create mode 100644 src/RedShirt.Example.JobWorker.Connectors.Common.Http/Models/OAuthClientCredentialsRequest.cs create mode 100644 src/RedShirt.Example.JobWorker.Connectors.Common.Http/Models/OAuthTokenCacheResponse.cs create mode 100644 src/RedShirt.Example.JobWorker.Connectors.Common.Http/Models/OAuthTokenResponse.cs create mode 100644 src/RedShirt.Example.JobWorker.Connectors.Common.Http/RedShirt.Example.JobWorker.Connectors.Common.Http.csproj create mode 100644 src/RedShirt.Example.JobWorker.Connectors.Common.Http/Services/OAuthTokenCache.cs create mode 100644 src/RedShirt.Example.JobWorker.Connectors.Common.Http/Services/OAuthTokenSource.cs create mode 100755 test/local/scripts/wiremock-bar/bar-rotate-oauth-credentials.sh create mode 100755 test/local/scripts/wiremock-bar/bar-set-ssm-oauth-secret.sh create mode 100644 test/local/wiremock/mappings/get-api-bar-404.json create mode 100644 test/local/wiremock/mappings/get-api-bar-by-id.json create mode 100644 test/local/wiremock/mappings/post-api-bar.json create mode 100644 test/local/wiremock/mappings/post-oauth-token-unauthorized.json create mode 100644 test/local/wiremock/mappings/post-oauth-token.json create mode 100644 test/local/wiremock/mappings/unauthorized-api-bar.json diff --git a/README.md b/README.md index 8ce9beec..7b814b75 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,9 @@ Repo features: the message source. * Prevents simultaneous execution of the same message in the event of a dropped message * Caches results to prevent re-running of a job if received non-concurrently -* Container health probes +* Container health probes. +* Sample API connector. + * For more information, see TOMATO. * Documentation for local testing (see `test/local/`) # Core Architecture diff --git a/RedShirt.Example.JobWorker.slnx b/RedShirt.Example.JobWorker.slnx index dff91ccf..68c6cbe8 100644 --- a/RedShirt.Example.JobWorker.slnx +++ b/RedShirt.Example.JobWorker.slnx @@ -8,6 +8,9 @@ + + + @@ -25,6 +28,12 @@ + + + +/// Classified failure from a Bar connector operation. Thrown by the connector implementation after +/// retry/arbitration so callers can react to a stable, already-handled outcome. +/// +public class BarException : Exception +{ + /// + /// When true, a retry wrapper inside the connector layer has already exhausted retries for the + /// underlying cause; outer retry layers should not retry again. + /// + public bool IsHandled { get; init; } + + /// + /// When true, suggests a possible transient or environmental cause could be resolved outside the application + /// process (with an infrastructure change, for example) without restarting the application. + /// + public bool CouldBeTransient { get; init; } + + /// + /// When true, suggests a possible environmental cause that could be resolved outside the application + /// process (for example an infrastructure change) without restarting the application. + /// + public bool CouldBeExternallySolvable { get; init; } + + public BarException(Exception innerException) : base(innerException.Message, innerException) + { + } + + public BarException(string message) : base(message) + { + } +} \ No newline at end of file diff --git a/src/RedShirt.Example.JobWorker.Connectors.Bar.Core/Exceptions/BarRecordNotFoundException.cs b/src/RedShirt.Example.JobWorker.Connectors.Bar.Core/Exceptions/BarRecordNotFoundException.cs new file mode 100644 index 00000000..eb6cb425 --- /dev/null +++ b/src/RedShirt.Example.JobWorker.Connectors.Bar.Core/Exceptions/BarRecordNotFoundException.cs @@ -0,0 +1,9 @@ +namespace RedShirt.Example.JobWorker.Connectors.Bar.Core.Exceptions; + +/// +/// The Bar dependency reported that no record exists for the requested id (HTTP 404). +/// +public sealed class BarRecordNotFoundException(int id) : Exception($"Bar record {id} was not found.") +{ + public int Id => id; +} \ No newline at end of file diff --git a/src/RedShirt.Example.JobWorker.Connectors.Bar.Core/Models/CreateBarConnectorRequest.cs b/src/RedShirt.Example.JobWorker.Connectors.Bar.Core/Models/CreateBarConnectorRequest.cs new file mode 100644 index 00000000..609d4a27 --- /dev/null +++ b/src/RedShirt.Example.JobWorker.Connectors.Bar.Core/Models/CreateBarConnectorRequest.cs @@ -0,0 +1,6 @@ +namespace RedShirt.Example.JobWorker.Connectors.Bar.Core.Models; + +public sealed class CreateBarConnectorRequest +{ + public required string Name { get; init; } +} \ No newline at end of file diff --git a/src/RedShirt.Example.JobWorker.Connectors.Bar.Core/Models/CreateBarConnectorResponse.cs b/src/RedShirt.Example.JobWorker.Connectors.Bar.Core/Models/CreateBarConnectorResponse.cs new file mode 100644 index 00000000..4bc20072 --- /dev/null +++ b/src/RedShirt.Example.JobWorker.Connectors.Bar.Core/Models/CreateBarConnectorResponse.cs @@ -0,0 +1,8 @@ +namespace RedShirt.Example.JobWorker.Connectors.Bar.Core.Models; + +public sealed class CreateBarConnectorResponse +{ + public required int Id { get; init; } + + public required string Name { get; init; } +} \ No newline at end of file diff --git a/src/RedShirt.Example.JobWorker.Connectors.Bar.Core/Models/GetBarConnectorResponse.cs b/src/RedShirt.Example.JobWorker.Connectors.Bar.Core/Models/GetBarConnectorResponse.cs new file mode 100644 index 00000000..48370135 --- /dev/null +++ b/src/RedShirt.Example.JobWorker.Connectors.Bar.Core/Models/GetBarConnectorResponse.cs @@ -0,0 +1,8 @@ +namespace RedShirt.Example.JobWorker.Connectors.Bar.Core.Models; + +public sealed class GetBarConnectorResponse +{ + public required int Id { get; init; } + + public required string Name { get; init; } +} \ No newline at end of file diff --git a/src/RedShirt.Example.JobWorker.Connectors.Bar.Core/RedShirt.Example.JobWorker.Connectors.Bar.Core.csproj b/src/RedShirt.Example.JobWorker.Connectors.Bar.Core/RedShirt.Example.JobWorker.Connectors.Bar.Core.csproj new file mode 100644 index 00000000..e0ad4b55 --- /dev/null +++ b/src/RedShirt.Example.JobWorker.Connectors.Bar.Core/RedShirt.Example.JobWorker.Connectors.Bar.Core.csproj @@ -0,0 +1,9 @@ + + + + net10.0 + enable + enable + + + diff --git a/src/RedShirt.Example.JobWorker.Connectors.Bar.Core/Services/IBarConnector.cs b/src/RedShirt.Example.JobWorker.Connectors.Bar.Core/Services/IBarConnector.cs new file mode 100644 index 00000000..5e2866b8 --- /dev/null +++ b/src/RedShirt.Example.JobWorker.Connectors.Bar.Core/Services/IBarConnector.cs @@ -0,0 +1,15 @@ +using RedShirt.Example.JobWorker.Connectors.Bar.Core.Models; + +namespace RedShirt.Example.JobWorker.Connectors.Bar.Core.Services; + +/// +/// Opaque connector for the Bar dependency. +/// Bar is a stand-in for an OAuth-backed API client; see docs/bar-connector.md for last-mile instructions. +/// +public interface IBarConnector +{ + Task CreateAsync(CreateBarConnectorRequest request, + CancellationToken cancellationToken = default); + + Task GetByIdAsync(int id, CancellationToken cancellationToken = default); +} \ No newline at end of file diff --git a/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Clients/BarApiClient.cs b/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Clients/BarApiClient.cs new file mode 100644 index 00000000..97d72c9f --- /dev/null +++ b/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Clients/BarApiClient.cs @@ -0,0 +1,129 @@ +using RedShirt.Example.JobWorker.Connectors.Bar.Core.Exceptions; +using RedShirt.Example.JobWorker.Connectors.Bar.Core.Models; +using RedShirt.Example.JobWorker.Connectors.Bar.Implementation.Exceptions; +using RedShirt.Example.JobWorker.Connectors.Bar.Implementation.Models.Requests; +using RedShirt.Example.JobWorker.Connectors.Bar.Implementation.Models.Responses; +using System.Globalization; +using System.Net; +using System.Text; +using System.Text.Json; + +namespace RedShirt.Example.JobWorker.Connectors.Bar.Implementation.Clients; + +internal interface IBarApiClient +{ + Task CreateBarAsync(CreateBarConnectorRequest request, + CancellationToken cancellationToken = default); + + Task GetBarByIdAsync(int id, CancellationToken cancellationToken = default); +} + +/// +/// HTTP transport for the Bar dependency. Failures surface as raw framework exceptions +/// (, , timeouts, etc.), +/// except get-by-id HTTP 404 which surfaces as +/// and HTTP 429 which surfaces as . +/// +internal sealed class BarApiClient(HttpClient httpClient, string baseUrl) : IBarApiClient +{ + private static void EnsureSuccessOrThrow(HttpResponseMessage response) + { + if (response.IsSuccessStatusCode) + { + return; + } + + if (response.StatusCode == HttpStatusCode.TooManyRequests) + { + throw new BarRateLimitedException(ParseRetryAfter(response)); + } + + throw new HttpRequestException( + $"Response status code does not indicate success: {(int) response.StatusCode} ({response.StatusCode}).", + null, + response.StatusCode); + } + + private static TimeSpan? ParseRetryAfter(HttpResponseMessage response) + { + if (!response.Headers.TryGetValues("Retry-After", out var values)) + { + return null; + } + + var headerValue = values.FirstOrDefault(); + if (string.IsNullOrWhiteSpace(headerValue)) + { + return null; + } + + if (int.TryParse(headerValue, NumberStyles.Integer, CultureInfo.InvariantCulture, out var seconds)) + { + return TimeSpan.FromSeconds(seconds); + } + + if (DateTimeOffset.TryParse(headerValue, CultureInfo.InvariantCulture, + DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var retryAt)) + { + var delay = retryAt - DateTimeOffset.UtcNow; + return delay > TimeSpan.Zero ? delay : TimeSpan.Zero; + } + + return null; + } + + public async Task CreateBarAsync(CreateBarConnectorRequest request, + CancellationToken cancellationToken = default) + { + using var message = new HttpRequestMessage(HttpMethod.Post, new Uri($"{baseUrl.TrimEnd('/')}/api/bar")); + message.Content = new StringContent(JsonSerializer.Serialize(new InternalBarCreateRequest + { + Name = request.Name + }), Encoding.UTF8, "application/json"); + + using var response = await httpClient.SendAsync(message, cancellationToken); + EnsureSuccessOrThrow(response); + + var stringResponse = await response.Content.ReadAsStringAsync(cancellationToken); + var responseObject = JsonSerializer.Deserialize(stringResponse); + if (responseObject is null) + { + throw new JsonException("Bar API create response body deserialized to null."); + } + + return new CreateBarConnectorResponse + { + Id = responseObject.Id, + Name = responseObject.Name + }; + } + + public async Task GetBarByIdAsync(int id, + CancellationToken cancellationToken = default) + { + using var message = new HttpRequestMessage(HttpMethod.Get, + new Uri($"{baseUrl.TrimEnd('/')}/api/bar/{id}")); + + using var response = await httpClient.SendAsync(message, cancellationToken); + + if (response.StatusCode == HttpStatusCode.NotFound) + { + throw new BarRecordNotFoundException(id); + } + + EnsureSuccessOrThrow(response); + + var stringResponse = await response.Content.ReadAsStringAsync(cancellationToken); + var responseObject = JsonSerializer.Deserialize(stringResponse); + if (responseObject is null) + { + throw new JsonException("Bar API get response body deserialized to null."); + } + + return new GetBarConnectorResponse + { + Id = responseObject.Id, + Name = responseObject.Name + }; + } +} \ No newline at end of file diff --git a/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Clients/BarApiClientHandler.cs b/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Clients/BarApiClientHandler.cs new file mode 100644 index 00000000..e224a1eb --- /dev/null +++ b/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Clients/BarApiClientHandler.cs @@ -0,0 +1,36 @@ +using RedShirt.Example.JobWorker.Connectors.Bar.Implementation.Exceptions; +using RedShirt.Example.JobWorker.Connectors.Bar.Implementation.Services.Resilience; +using System.Net; +using System.Net.Http.Headers; + +namespace RedShirt.Example.JobWorker.Connectors.Bar.Implementation.Clients; + +/// +/// Attaches a Bar OAuth bearer token to outbound requests. +/// On , signals +/// so the request handler retry wrapper can refresh the token (then credentials) and retry. +/// +internal sealed class BarApiClientHandler( + IBarApiRequestHandlerRetryWrapperService apiRequestRetryWrapperService) : DelegatingHandler +{ + protected override async Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + return await apiRequestRetryWrapperService.ExecuteAsync(async ct => + { + var accessToken = await apiRequestRetryWrapperService.GetAccessTokenAsync(ct); + request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", accessToken); + + var response = await base.SendAsync(request, ct); + + // ReSharper disable once InvertIf + if (response.StatusCode is HttpStatusCode.Unauthorized) + { + response.Dispose(); + throw new BarUnauthorizedException(); + } + + return response; + }, cancellationToken); + } +} \ No newline at end of file diff --git a/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Exceptions/BarRateLimitedException.cs b/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Exceptions/BarRateLimitedException.cs new file mode 100644 index 00000000..7b7c16ab --- /dev/null +++ b/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Exceptions/BarRateLimitedException.cs @@ -0,0 +1,16 @@ +namespace RedShirt.Example.JobWorker.Connectors.Bar.Implementation.Exceptions; + +/// +/// Suggests a rate limit response from the underlying Bar service (typically HTTP 429). +/// +internal sealed class BarRateLimitedException : BarReasonToWaitException +{ + public BarRateLimitedException(TimeSpan? retryAfter) + : base("Bar API rate limit exceeded.") + { + RetryAfter = retryAfter; + IsHandled = false; + CouldBeTransient = true; + CouldBeExternallySolvable = true; + } +} diff --git a/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Exceptions/BarReasonToWaitException.cs b/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Exceptions/BarReasonToWaitException.cs new file mode 100644 index 00000000..4f3c7269 --- /dev/null +++ b/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Exceptions/BarReasonToWaitException.cs @@ -0,0 +1,20 @@ +using RedShirt.Example.JobWorker.Connectors.Bar.Core.Exceptions; + +namespace RedShirt.Example.JobWorker.Connectors.Bar.Implementation.Exceptions; + +/// +/// The Bar dependency indicated that the caller should wait before retrying. +/// The JobWorker Bar connector respects these exceptions indefinitely; see docs/bar-connector.md. +/// +internal abstract class BarReasonToWaitException : BarException +{ + /// + /// Optional delay suggested by the dependency (for example from an HTTP Retry-After header). + /// When null, the connector uses its configured fallback wait duration. + /// + public TimeSpan? RetryAfter { get; init; } + + protected BarReasonToWaitException(string message) : base(message) + { + } +} diff --git a/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Exceptions/BarTemporarilyUnavailableException.cs b/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Exceptions/BarTemporarilyUnavailableException.cs new file mode 100644 index 00000000..5d6bee57 --- /dev/null +++ b/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Exceptions/BarTemporarilyUnavailableException.cs @@ -0,0 +1,15 @@ +namespace RedShirt.Example.JobWorker.Connectors.Bar.Implementation.Exceptions; + +/// +/// Bar is assumed to be unavailable for the time being (for example after auth or token +/// recovery failed within the refresh cooldown window). +/// +internal sealed class BarTemporarilyUnavailableException : BarReasonToWaitException +{ + public BarTemporarilyUnavailableException() : base("Bar is assumed to be unavailable for the time being.") + { + IsHandled = false; + CouldBeTransient = false; + CouldBeExternallySolvable = true; + } +} \ No newline at end of file diff --git a/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Exceptions/BarUnauthorizedException.cs b/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Exceptions/BarUnauthorizedException.cs new file mode 100644 index 00000000..0b1f8f19 --- /dev/null +++ b/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Exceptions/BarUnauthorizedException.cs @@ -0,0 +1,17 @@ +using RedShirt.Example.JobWorker.Connectors.Bar.Core.Exceptions; + +namespace RedShirt.Example.JobWorker.Connectors.Bar.Implementation.Exceptions; + +/// +/// The Bar dependency rejected the bearer token (HTTP 401), including after a force-refresh attempt. +/// Surfaced to callers as by the connector retry wrapper. +/// +internal sealed class BarUnauthorizedException : BarException +{ + public BarUnauthorizedException() : base("Bar API rejected the bearer token.") + { + IsHandled = false; + CouldBeTransient = false; + CouldBeExternallySolvable = true; + } +} \ No newline at end of file diff --git a/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Extensions/ServiceCollectionExtensions.cs b/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Extensions/ServiceCollectionExtensions.cs new file mode 100644 index 00000000..fba2dcff --- /dev/null +++ b/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Extensions/ServiceCollectionExtensions.cs @@ -0,0 +1,49 @@ +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using RedShirt.Example.JobWorker.Common.Extensions; +using RedShirt.Example.JobWorker.Connectors.Bar.Core.Services; +using RedShirt.Example.JobWorker.Connectors.Bar.Implementation.Clients; +using RedShirt.Example.JobWorker.Connectors.Bar.Implementation.Factories; +using RedShirt.Example.JobWorker.Connectors.Bar.Implementation.Services; +using RedShirt.Example.JobWorker.Connectors.Bar.Implementation.Services.Resilience; +using RedShirt.Example.JobWorker.Connectors.Common.Http.Services; + +namespace RedShirt.Example.JobWorker.Connectors.Bar.Implementation.Extensions; + +public static class ServiceCollectionExtensions +{ + public const string ConfigurationSectionName = "Connectors:Bar"; + + public static IServiceCollection AddBarConnector(this IServiceCollection services, IConfiguration configuration) + { + services + .AddCommon() + .Configure( + configuration.GetSection(ConfigurationSectionName)) + .Configure( + configuration.GetSection(ConfigurationSectionName)) + .Configure( + configuration.GetSection(ConfigurationSectionName)) + .Configure( + configuration.GetSection(ConfigurationSectionName)) + .Configure( + configuration.GetSection(ConfigurationSectionName)) + .AddSingleton() + .AddSingleton() + .AddTransient() + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton(); + + services + .AddHttpClient(nameof(OAuthTokenSource)); + + services + .AddHttpClient(nameof(BarApiClient)) + .AddHttpMessageHandler(); + + return services; + } +} \ No newline at end of file diff --git a/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Factories/BarApiClientFactory.cs b/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Factories/BarApiClientFactory.cs new file mode 100644 index 00000000..d75b0055 --- /dev/null +++ b/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Factories/BarApiClientFactory.cs @@ -0,0 +1,25 @@ +using Microsoft.Extensions.Options; +using RedShirt.Example.JobWorker.Connectors.Bar.Implementation.Clients; + +namespace RedShirt.Example.JobWorker.Connectors.Bar.Implementation.Factories; + +internal interface IBarApiClientFactory +{ + IBarApiClient CreateBarApiClient(); +} + +internal sealed class BarApiClientFactory( + IHttpClientFactory httpClientFactory, + IOptions configuration) : IBarApiClientFactory +{ + public IBarApiClient CreateBarApiClient() + { + var httpClient = httpClientFactory.CreateClient(nameof(BarApiClient)); + return new BarApiClient(httpClient, configuration.Value.BaseUrl); + } + + internal sealed class ConfigurationModel + { + public required string BaseUrl { get; init; } + } +} \ No newline at end of file diff --git a/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Globals.cs b/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Globals.cs new file mode 100644 index 00000000..62ff2128 --- /dev/null +++ b/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Globals.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("DynamicProxyGenAssembly2")] \ No newline at end of file diff --git a/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Models/BarExceptionArbiterReport.cs b/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Models/BarExceptionArbiterReport.cs new file mode 100644 index 00000000..a8d87293 --- /dev/null +++ b/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Models/BarExceptionArbiterReport.cs @@ -0,0 +1,12 @@ +namespace RedShirt.Example.JobWorker.Connectors.Bar.Implementation.Models; + +internal sealed class BarExceptionArbiterReport +{ + public required bool AlreadyHandled { get; init; } + + public required bool IsExpected { get; init; } + + public required bool CouldBeTransient { get; init; } + + public required bool CouldBeExternallySolvable { get; init; } +} \ No newline at end of file diff --git a/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Models/Requests/InternalBarCreateRequest.cs b/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Models/Requests/InternalBarCreateRequest.cs new file mode 100644 index 00000000..2c12cd61 --- /dev/null +++ b/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Models/Requests/InternalBarCreateRequest.cs @@ -0,0 +1,6 @@ +namespace RedShirt.Example.JobWorker.Connectors.Bar.Implementation.Models.Requests; + +internal sealed class InternalBarCreateRequest +{ + public required string Name { get; init; } +} \ No newline at end of file diff --git a/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Models/Responses/InternalBarCreateResponse.cs b/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Models/Responses/InternalBarCreateResponse.cs new file mode 100644 index 00000000..d297388f --- /dev/null +++ b/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Models/Responses/InternalBarCreateResponse.cs @@ -0,0 +1,8 @@ +namespace RedShirt.Example.JobWorker.Connectors.Bar.Implementation.Models.Responses; + +internal sealed class InternalBarCreateResponse +{ + public required int Id { get; init; } + + public required string Name { get; init; } +} \ No newline at end of file diff --git a/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Models/Responses/InternalBarGetResponse.cs b/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Models/Responses/InternalBarGetResponse.cs new file mode 100644 index 00000000..63e60c79 --- /dev/null +++ b/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Models/Responses/InternalBarGetResponse.cs @@ -0,0 +1,8 @@ +namespace RedShirt.Example.JobWorker.Connectors.Bar.Implementation.Models.Responses; + +internal sealed class InternalBarGetResponse +{ + public required int Id { get; init; } + + public required string Name { get; init; } +} \ No newline at end of file diff --git a/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/RedShirt.Example.JobWorker.Connectors.Bar.Implementation.csproj b/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/RedShirt.Example.JobWorker.Connectors.Bar.Implementation.csproj new file mode 100644 index 00000000..4c264bc5 --- /dev/null +++ b/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/RedShirt.Example.JobWorker.Connectors.Bar.Implementation.csproj @@ -0,0 +1,23 @@ + + + + net10.0 + enable + enable + + + + + + + + + + + + + + + + + diff --git a/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Services/BarConnector.cs b/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Services/BarConnector.cs new file mode 100644 index 00000000..056c2fb7 --- /dev/null +++ b/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Services/BarConnector.cs @@ -0,0 +1,122 @@ +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Polly; +using Polly.Retry; +using RedShirt.Example.JobWorker.Common.Services.Utility; +using RedShirt.Example.JobWorker.Connectors.Bar.Core.Models; +using RedShirt.Example.JobWorker.Connectors.Bar.Core.Services; +using RedShirt.Example.JobWorker.Connectors.Bar.Implementation.Exceptions; +using RedShirt.Example.JobWorker.Connectors.Bar.Implementation.Factories; +using RedShirt.Example.JobWorker.Connectors.Bar.Implementation.Services.Resilience; + +namespace RedShirt.Example.JobWorker.Connectors.Bar.Implementation.Services; + +/// +/// Bar connector implementation: maps Core requests onto the Bar HTTP client under the retry wrapper. +/// This connector is a stand-in for an API template client; refer to docs/bar-connector.md +/// for last-mile instructions when adapting to your target API. +/// +internal sealed class BarConnector( + IBarApiClientFactory barApiClientFactory, + IBarRetryWrapperService retryWrapperService, + ISleepService sleepService, + ILogger logger, + IOptions options) : IBarConnector +{ + private const int DefaultReasonToWaitFallbackSeconds = 15; + + private ResiliencePipeline? _reasonToWaitPipeline; + + /// + /// Recycled Polly pipeline that respects indefinitely. + /// Cancellation is the Core job worker configuration's problem; this connector keeps trying respectfully + /// when the dependency signals rate limiting or another reason to wait. + /// + private ResiliencePipeline GetReasonToWaitPipeline() + { + return _reasonToWaitPipeline ??= new ResiliencePipelineBuilder() + .AddRetry(new RetryStrategyOptions + { + // Functionally infinite retries + MaxRetryAttempts = int.MaxValue, + ShouldHandle = args => + { + if (args.Context.CancellationToken.IsCancellationRequested) + { + return PredicateResult.False(); + } + + return args.Outcome.Exception is BarReasonToWaitException + ? PredicateResult.True() + : PredicateResult.False(); + }, + // Do not delay via polly, use ISleepService. + DelayGenerator = static _ => new ValueTask(TimeSpan.Zero), + OnRetry = async args => + { + if (args.Outcome.Exception is not BarReasonToWaitException reasonToWait) + { + return; + } + + var baseDelay = reasonToWait.RetryAfter ?? options.Value.EffectiveReasonToWaitFallback; + // Polly v8 AttemptNumber is zero-based (0 on the first retry). Add linear slack on top of + // RetryAfter/fallback because the API may need a little extra time to recognize that the + // rate-limiting window has passed (0s, then 1s, then 2s, etc). + var attemptBuffer = TimeSpan.FromSeconds(args.AttemptNumber); + var delay = baseDelay + attemptBuffer; + logger.LogWarning(reasonToWait, + "Bar indicated a reason to wait with a {Type}; delaying {Delay} (base {BaseDelay}, attempt buffer {AttemptBuffer}) before retry (attempt {AttemptNumber})", + reasonToWait.GetType().Name, delay, baseDelay, attemptBuffer, args.AttemptNumber + 1); + await sleepService.DelayAsync(delay, args.Context.CancellationToken); + } + }) + .Build(); + } + + private Task ExecuteRespectingReasonToWaitAsync( + Func> func, + CancellationToken cancellationToken) + { + return GetReasonToWaitPipeline().ExecuteAsync( + async token => await func(token), + cancellationToken).AsTask(); + } + + public Task CreateAsync(CreateBarConnectorRequest request, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(request); + + return ExecuteRespectingReasonToWaitAsync(token => + retryWrapperService.RunAsync(async innerToken => + { + var client = barApiClientFactory.CreateBarApiClient(); + return await client.CreateBarAsync(request, innerToken); + }, token), cancellationToken); + } + + public Task GetByIdAsync(int id, CancellationToken cancellationToken = default) + { + return ExecuteRespectingReasonToWaitAsync(token => + retryWrapperService.RunAsync(innerToken => + { + var client = barApiClientFactory.CreateBarApiClient(); + return client.GetBarByIdAsync(id, innerToken); + }, token), cancellationToken); + } + + internal sealed class ConfigurationModel + { + /// + /// Fallback wait duration when a does not specify + /// . + /// When null, is used. + /// + public required int? ReasonToWaitFallbackSeconds { get; init; } + + public TimeSpan EffectiveReasonToWaitFallback => + TimeSpan.FromSeconds(Math.Max(1, + ReasonToWaitFallbackSeconds ?? DefaultReasonToWaitFallbackSeconds)); + } +} \ No newline at end of file diff --git a/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Services/Resilience/BarApiRequestHandlerRetryWrapperService.cs b/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Services/Resilience/BarApiRequestHandlerRetryWrapperService.cs new file mode 100644 index 00000000..2e59800b --- /dev/null +++ b/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Services/Resilience/BarApiRequestHandlerRetryWrapperService.cs @@ -0,0 +1,237 @@ +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Polly; +using Polly.Retry; +using RedShirt.Example.JobWorker.Connectors.Bar.Implementation.Exceptions; +using RedShirt.Example.JobWorker.Connectors.Common.Http.Enums; +using RedShirt.Example.JobWorker.Connectors.Common.Http.Exceptions; +using RedShirt.Example.JobWorker.Connectors.Common.Http.Models; +using RedShirt.Example.JobWorker.Connectors.Common.Http.Services; +using System.Net; + +namespace RedShirt.Example.JobWorker.Connectors.Bar.Implementation.Services.Resilience; + +internal interface IBarApiRequestHandlerRetryWrapperService +{ + Task ExecuteAsync(Func> func, CancellationToken cancellationToken = default); + + Task GetAccessTokenAsync(CancellationToken cancellationToken = default); +} + +/// +/// Obtains Bar bearer tokens via and retries on unauthorized: +/// attempt 1 forces a fresh token (escalating to fresh credentials when inside the token refresh +/// cooldown); attempt 2 forces fresh credentials and a fresh token. +/// +internal sealed class BarApiRequestHandlerRetryWrapperService( + IOAuthTokenCache oauthTokenCache, + ILogger logger, + IOptions options) + : IBarApiRequestHandlerRetryWrapperService +{ + private const int DefaultTokenRefreshCooldownSeconds = 60; + + private const string PreviousAttemptInvolvedEscalation = "e"; + + private readonly SemaphoreSlim _tokenGate = new(1, 1); + + private HttpStatusCode? _previousAttemptStatusCode; + + private ResiliencePipeline? _retryPipeline; + private DateTimeOffset? _tokenAttemptedAtUtc; + private DateTimeOffset? _tokenFetchedAtUtc; + + private OAuthTokenCacheResponse? _tokenResult; + + private OAuthClientCredentialsRequest CreateOAuthRequest() + { + var configuration = options.Value; + return new OAuthClientCredentialsRequest + { + TokenUrl = configuration.TokenUrl, + ClientIdPath = configuration.ClientIdPath, + ClientSecretPath = configuration.ClientSecretPath, + ScopeLabel = configuration.ScopeLabel, + ScopeValue = configuration.ScopeValue + }; + } + + private bool IsWithinTokenRefreshCooldown() + { + if (_tokenFetchedAtUtc is not { } fetchedAtUtc) + { + return false; + } + + return DateTimeOffset.UtcNow < fetchedAtUtc + options.Value.EffectiveTokenRefreshCooldown; + } + + private bool IsAttemptWithinTokenRefreshCooldown() + { + if (_tokenAttemptedAtUtc is not { } attemptedAtUtc) + { + return false; + } + + return DateTimeOffset.UtcNow < attemptedAtUtc + options.Value.EffectiveTokenRefreshCooldown; + } + + private async Task RefreshAndGetAccessTokenAsync(bool forceFreshToken, + bool forceFreshCredentials, + CancellationToken cancellationToken) + { + if (!forceFreshCredentials + && _previousAttemptStatusCode != HttpStatusCode.OK + && IsAttemptWithinTokenRefreshCooldown()) + { + throw new BarTemporarilyUnavailableException(); + } + + _tokenAttemptedAtUtc = DateTimeOffset.UtcNow; + OAuthTokenCacheResponse result; + try + { + result = await oauthTokenCache.GetAsync(CreateOAuthRequest(), forceFreshToken, forceFreshCredentials, + cancellationToken); + _previousAttemptStatusCode = HttpStatusCode.OK; + } + catch (OAuthRequestException e) + { + _previousAttemptStatusCode = e.StatusCode; + throw; + } + + _tokenResult = result; + _tokenFetchedAtUtc = DateTimeOffset.UtcNow; + return result; + } + + private ResiliencePipeline GetRetryPipeline() + { + return _retryPipeline ??= new ResiliencePipelineBuilder() + .AddRetry(new RetryStrategyOptions + { + MaxRetryAttempts = 2, + ShouldHandle = args => + { + // ReSharper disable once DuplicatedSequentialIfBodies + if (args is + { + AttemptNumber: 0, Outcome.Exception: OAuthRequestException + { + StatusCode: HttpStatusCode.Unauthorized, + CredentialStorageProblem: false, + FreshCredentialCacheResult: false + } + }) + { + return PredicateResult.True(); + } + + if (args.Outcome.Exception is BarUnauthorizedException + && !IsWithinTokenRefreshCooldown() + && !( + args.Context.Properties.TryGetValue( + new ResiliencePropertyKey(PreviousAttemptInvolvedEscalation), + out var previousAttemptInvolvedEscalation) + && previousAttemptInvolvedEscalation + )) + { + return PredicateResult.True(); + } + + return PredicateResult.False(); + }, + DelayGenerator = static _ => new ValueTask(TimeSpan.Zero), + OnRetry = async args => + { + await _tokenGate.WaitAsync(args.Context.CancellationToken); + try + { + var forceFreshCredentials = args.AttemptNumber >= 1 + || args.Outcome.Exception is OAuthRequestException; + + var previousAccessToken = _tokenResult?.AccessToken; + logger.LogDebug( + "Refreshing Bar bearer token from {TokenUrl} (forceFreshCredentials: {ForceFreshCredentials})", + options.Value.TokenUrl, forceFreshCredentials); + + OAuthTokenCacheResponse result; + try + { + result = await RefreshAndGetAccessTokenAsync(true, forceFreshCredentials, + args.Context.CancellationToken); + } + catch (OAuthRequestException) when (!forceFreshCredentials) + { + args.Context.Properties.Set( + new ResiliencePropertyKey(PreviousAttemptInvolvedEscalation), true); + result = await RefreshAndGetAccessTokenAsync(true, true, args.Context.CancellationToken); + } + + if (forceFreshCredentials + && (string.Equals(previousAccessToken, result.AccessToken, StringComparison.Ordinal) + || result.TokenCacheState != TokenCacheState.ForcedCredentialRetrieval)) + { + throw new BarUnauthorizedException(); + } + } + finally + { + _tokenGate.Release(); + } + } + }) + .Build(); + } + + public async Task GetAccessTokenAsync(CancellationToken cancellationToken = default) + { + if (_tokenResult is not null) + { + return _tokenResult.AccessToken; + } + + await _tokenGate.WaitAsync(cancellationToken); + try + { + if (_tokenResult is not null) + { + return _tokenResult.AccessToken; + } + + await RefreshAndGetAccessTokenAsync(false, false, cancellationToken); + return _tokenResult!.AccessToken; + } + finally + { + _tokenGate.Release(); + } + } + + public Task ExecuteAsync(Func> func, + CancellationToken cancellationToken = default) + { + return GetRetryPipeline().ExecuteAsync( + async token => await func(token), + cancellationToken).AsTask(); + } + + internal sealed class ConfigurationModel + { + public required string TokenUrl { get; init; } + + public required string ClientIdPath { get; init; } + + public required string ClientSecretPath { get; init; } + + public required string? ScopeLabel { get; init; } + + public required string? ScopeValue { get; init; } + + public required int? TokenRefreshCooldownSeconds { get; init; } + + public TimeSpan EffectiveTokenRefreshCooldown => + TimeSpan.FromSeconds(Math.Max(1, TokenRefreshCooldownSeconds ?? DefaultTokenRefreshCooldownSeconds)); + } +} \ No newline at end of file diff --git a/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Services/Resilience/BarExceptionArbiterService.cs b/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Services/Resilience/BarExceptionArbiterService.cs new file mode 100644 index 00000000..3f9851a9 --- /dev/null +++ b/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Services/Resilience/BarExceptionArbiterService.cs @@ -0,0 +1,124 @@ +using RedShirt.Example.JobWorker.Common.SecretManagers.Core.Exceptions; +using RedShirt.Example.JobWorker.Connectors.Bar.Core.Exceptions; +using RedShirt.Example.JobWorker.Connectors.Bar.Implementation.Exceptions; +using RedShirt.Example.JobWorker.Connectors.Bar.Implementation.Models; +using RedShirt.Example.JobWorker.Connectors.Common.Http.Exceptions; +using System.Net; +using System.Net.Sockets; +using System.Text.Json; + +namespace RedShirt.Example.JobWorker.Connectors.Bar.Implementation.Services.Resilience; + +/// +/// Classifies Bar connector exceptions for retry decisions. +/// +internal interface IBarExceptionArbiterService +{ + BarExceptionArbiterReport GetReport(Exception exception); +} + +/// +/// Bar-oriented exception arbiter modelled after the MySQL / Azure exception arbiters: +/// known infrastructure and retryable HTTP failures may be transient; caller cancel and bad arguments are not. +/// +internal sealed class BarExceptionArbiterService : IBarExceptionArbiterService +{ + private static readonly HashSet TransientHttpStatuses = + [ + 408, + 429, + 500, + 502, + 503, + 504 + ]; + + private static BarExceptionArbiterReport Fresh(bool isExpected, bool couldBeTransient, + bool couldBeExternallySolvable) + { + return new BarExceptionArbiterReport + { + AlreadyHandled = false, + IsExpected = isExpected, + CouldBeTransient = couldBeTransient, + CouldBeExternallySolvable = couldBeExternallySolvable + }; + } + + private static BarExceptionArbiterReport Handled( + bool isExpected, + bool couldBeTransient, + bool couldBeExternallySolvable) + { + return new BarExceptionArbiterReport + { + AlreadyHandled = true, + IsExpected = isExpected, + CouldBeTransient = couldBeTransient, + CouldBeExternallySolvable = couldBeExternallySolvable + }; + } + + private static BarExceptionArbiterReport ClassifyHttpRequestException(HttpRequestException exception) + { + if (exception.StatusCode is null) + { + return Fresh(true, true, true); + } + + var status = (int) exception.StatusCode.Value; + if (TransientHttpStatuses.Contains(status)) + { + return Fresh(true, true, true); + } + + // ReSharper disable once ConvertIfStatementToReturnStatement + if (exception.StatusCode is HttpStatusCode.Unauthorized + or HttpStatusCode.Forbidden + or HttpStatusCode.NotFound) + { + return Fresh(true, false, true); + } + + return Fresh(true, false, false); + } + + public BarExceptionArbiterReport GetReport(Exception exception) + { + ArgumentNullException.ThrowIfNull(exception); + + while (exception is AggregateException {InnerExceptions.Count: 1, InnerException: not null} aggregate) + { + exception = aggregate.InnerException!; + } + + return exception switch + { + /* + * BarReasonToWaitException is a special case. Functionally, it's absolutely transient. + * In fact, it's literally implied in the name: "you have a reason to wait, and then things could be better". + * + * However, the reason that it's a special case is that the exception is expected to be caught and respected + * for as long as necessary for the API request to go through. + */ + BarReasonToWaitException => Fresh(true, false, true), + OAuthRequestException {StatusCode: HttpStatusCode.Unauthorized} => Fresh(true, false, true), + OAuthRequestException => Fresh(true, true, true), + OAuthRequestJsonException => Fresh(true, false, false), + BarRecordNotFoundException => Fresh(true, false, false), + BarUnauthorizedException => Fresh(true, false, true), + BarException w => + Handled(true, w is {IsHandled: false, CouldBeTransient: true}, w.CouldBeExternallySolvable), + WorkerSecretManagerException w => + Handled(true, w is {IsHandled: false, CouldBeTransient: true}, w.CouldBeExternallySolvable), + HttpRequestException http => ClassifyHttpRequestException(http), + SocketException + or TimeoutException => Fresh(true, true, true), + JsonException => Fresh(true, false, false), + TaskCanceledException => Fresh(true, true, true), + OperationCanceledException => Fresh(true, false, false), + ArgumentException => Fresh(true, false, false), + _ => Fresh(false, false, false) + }; + } +} \ No newline at end of file diff --git a/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Services/Resilience/BarRetryWrapperService.cs b/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Services/Resilience/BarRetryWrapperService.cs new file mode 100644 index 00000000..c295afad --- /dev/null +++ b/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Services/Resilience/BarRetryWrapperService.cs @@ -0,0 +1,171 @@ +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Polly; +using Polly.Retry; +using RedShirt.Example.JobWorker.Common.Services.Utility; +using RedShirt.Example.JobWorker.Connectors.Bar.Core.Exceptions; +using RedShirt.Example.JobWorker.Connectors.Bar.Implementation.Exceptions; + +namespace RedShirt.Example.JobWorker.Connectors.Bar.Implementation.Services.Resilience; + +/// +/// Retries Bar connector operations that fail with expected transient exceptions, +/// then surfaces remaining failures as . +/// +internal interface IBarRetryWrapperService +{ + Task RunAsync(Func> func, CancellationToken cancellationToken = default); + + Task RunAsync(Func func, CancellationToken cancellationToken = default); +} + +/// +/// Polly v8-based retry wrapper for Bar connector calls. +/// Retries when reports an expected transient failure, +/// using exponential backoff via . +/// instances are not retried here; they propagate to +/// for indefinite respectful waiting. +/// +internal sealed class BarRetryWrapperService( + IBarExceptionArbiterService exceptionArbiterService, + ILogger logger, + ISleepService sleepService, + IOptions options) + : IBarRetryWrapperService +{ + private const int DefaultRetryCount = 3; + + private ResiliencePipeline? _retryPipeline; + + private ResiliencePipeline GetRetryPipeline() + { + return _retryPipeline ??= new ResiliencePipelineBuilder() + .AddRetry(new RetryStrategyOptions + { + MaxRetryAttempts = options.Value.EffectiveRetryCount, + ShouldHandle = args => + { + if (args.Outcome.Exception is not { } exception) + { + return PredicateResult.False(); + } + + if (args.Context.CancellationToken.IsCancellationRequested) + { + return PredicateResult.False(); + } + + if (exception is BarReasonToWaitException) + { + return PredicateResult.False(); + } + + var report = exceptionArbiterService.GetReport(exception); + return report is {IsExpected: true, CouldBeTransient: true} + ? PredicateResult.True() + : PredicateResult.False(); + }, + DelayGenerator = static _ => new ValueTask(TimeSpan.Zero), + OnRetry = async args => + { + logger.LogWarning(args.Outcome.Exception, + "Retrying Bar connector operation after attempt {AttemptNumber}", + args.AttemptNumber); + await sleepService.DelayAsync(TimeSpan.FromSeconds(Math.Pow(2, args.AttemptNumber)), + args.Context.CancellationToken); + } + }) + .Build(); + } + + private bool TryGetWrappedException(Exception exception, out Exception? wrappedException) + { + wrappedException = null; + + // ReSharper disable once ConvertIfStatementToSwitchStatement + if (exception is BarRecordNotFoundException) + { + return false; + } + + if (exception is BarReasonToWaitException) + { + // Special case, to not wrap. Will be handled infinitely in BarConnector. + return false; + } + + var report = exceptionArbiterService.GetReport(exception); + + if (report.AlreadyHandled && exception is BarException) + { + return false; + } + + if (!report.IsExpected) + { + return false; + } + + wrappedException = new BarException(exception) + { + CouldBeTransient = report.CouldBeTransient, + IsHandled = true, + CouldBeExternallySolvable = report.CouldBeExternallySolvable + }; + return true; + } + + public async Task RunAsync(Func> func, + CancellationToken cancellationToken = default) + { + try + { + return await GetRetryPipeline().ExecuteAsync( + async token => await func(token), + cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception exception) + { + if (TryGetWrappedException(exception, out var wrappedException) && wrappedException is not null) + { + throw wrappedException; + } + + throw; + } + } + + public async Task RunAsync(Func func, CancellationToken cancellationToken = default) + { + try + { + await GetRetryPipeline().ExecuteAsync( + async token => await func(token), + cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception exception) + { + if (TryGetWrappedException(exception, out var wrappedException) && wrappedException is not null) + { + throw wrappedException; + } + + throw; + } + } + + internal sealed class ConfigurationModel + { + public required int? RetryCount { get; init; } + + public int EffectiveRetryCount => Math.Max(0, RetryCount ?? DefaultRetryCount); + } +} \ No newline at end of file diff --git a/src/RedShirt.Example.JobWorker.Connectors.Common.Http/Enums/TokenCacheState.cs b/src/RedShirt.Example.JobWorker.Connectors.Common.Http/Enums/TokenCacheState.cs new file mode 100644 index 00000000..10d82958 --- /dev/null +++ b/src/RedShirt.Example.JobWorker.Connectors.Common.Http/Enums/TokenCacheState.cs @@ -0,0 +1,22 @@ +namespace RedShirt.Example.JobWorker.Connectors.Common.Http.Enums; + +/// +/// Describes how an OAuth access token was obtained for a cache lookup. +/// +public enum TokenCacheState +{ + /// + /// Client credentials were force-refreshed from the secret manager and a new token was requested. + /// + ForcedCredentialRetrieval, + + /// + /// A new token was requested (cache miss or forced token refresh). + /// + FreshToken, + + /// + /// A still-valid token was returned from the in-memory cache. + /// + CachedToken +} \ No newline at end of file diff --git a/src/RedShirt.Example.JobWorker.Connectors.Common.Http/Exceptions/OAuthRequestException.cs b/src/RedShirt.Example.JobWorker.Connectors.Common.Http/Exceptions/OAuthRequestException.cs new file mode 100644 index 00000000..f9e8845c --- /dev/null +++ b/src/RedShirt.Example.JobWorker.Connectors.Common.Http/Exceptions/OAuthRequestException.cs @@ -0,0 +1,24 @@ +using System.Net; + +namespace RedShirt.Example.JobWorker.Connectors.Common.Http.Exceptions; + +/// +/// The OAuth token endpoint returned a non-success HTTP status. +/// +public sealed class OAuthRequestException : Exception +{ + public required HttpStatusCode? StatusCode { get; init; } + public required bool CredentialStorageProblem { get; init; } + public required bool FreshCredentialCacheResult { get; init; } + + public OAuthRequestException(string message) + : base(message) + { + } + + public OAuthRequestException(string message, Exception innerException, HttpStatusCode? statusCode = null) + : base(message, innerException) + { + StatusCode = statusCode; + } +} \ No newline at end of file diff --git a/src/RedShirt.Example.JobWorker.Connectors.Common.Http/Exceptions/OAuthRequestJsonException.cs b/src/RedShirt.Example.JobWorker.Connectors.Common.Http/Exceptions/OAuthRequestJsonException.cs new file mode 100644 index 00000000..11b31009 --- /dev/null +++ b/src/RedShirt.Example.JobWorker.Connectors.Common.Http/Exceptions/OAuthRequestJsonException.cs @@ -0,0 +1,16 @@ +namespace RedShirt.Example.JobWorker.Connectors.Common.Http.Exceptions; + +/// +/// The OAuth token endpoint response could not be parsed or lacked a usable access token. +/// +public sealed class OAuthRequestJsonException : Exception +{ + public OAuthRequestJsonException(string message) : base(message) + { + } + + public OAuthRequestJsonException(string message, Exception innerException) + : base(message, innerException) + { + } +} \ No newline at end of file diff --git a/src/RedShirt.Example.JobWorker.Connectors.Common.Http/Models/OAuthClientCredentialsRequest.cs b/src/RedShirt.Example.JobWorker.Connectors.Common.Http/Models/OAuthClientCredentialsRequest.cs new file mode 100644 index 00000000..2a45b3b5 --- /dev/null +++ b/src/RedShirt.Example.JobWorker.Connectors.Common.Http/Models/OAuthClientCredentialsRequest.cs @@ -0,0 +1,36 @@ +namespace RedShirt.Example.JobWorker.Connectors.Common.Http.Models; + +/// +/// Inputs for an OAuth 2.0 client-credentials token request against a token endpoint. +/// Client id/secret are resolved from the secret manager via the configured paths. +/// +public class OAuthClientCredentialsRequest +{ + /// + /// Absolute URL of the OAuth token endpoint. + /// + public required string TokenUrl { get; init; } + + /// + /// Secret-manager path for the OAuth client id. + /// + public required string ClientIdPath { get; init; } + + /// + /// Secret-manager path for the OAuth client secret. + /// + public required string ClientSecretPath { get; init; } + + /// + /// Optional form-field name used for the scope/audience-style parameter + /// (for example scope or audience). + /// When null, no scope-style field is sent. + /// + public required string? ScopeLabel { get; init; } + + /// + /// Optional value for . + /// When null (or when is null), no scope-style field is sent. + /// + public required string? ScopeValue { get; init; } +} \ No newline at end of file diff --git a/src/RedShirt.Example.JobWorker.Connectors.Common.Http/Models/OAuthTokenCacheResponse.cs b/src/RedShirt.Example.JobWorker.Connectors.Common.Http/Models/OAuthTokenCacheResponse.cs new file mode 100644 index 00000000..1a41ab7a --- /dev/null +++ b/src/RedShirt.Example.JobWorker.Connectors.Common.Http/Models/OAuthTokenCacheResponse.cs @@ -0,0 +1,21 @@ +using RedShirt.Example.JobWorker.Connectors.Common.Http.Enums; + +namespace RedShirt.Example.JobWorker.Connectors.Common.Http.Models; + +/// +/// Result of an OAuth token cache lookup, including how the token was obtained. +/// +public class OAuthTokenCacheResponse +{ + public required string AccessToken { get; init; } + + /// + /// UTC instant when the access token should be treated as expired. + /// + public required DateTimeOffset ExpiresAtUtc { get; init; } + + /// + /// Whether the token came from cache or was freshly retrieved (and whether credentials were refreshed). + /// + public required TokenCacheState TokenCacheState { get; init; } +} \ No newline at end of file diff --git a/src/RedShirt.Example.JobWorker.Connectors.Common.Http/Models/OAuthTokenResponse.cs b/src/RedShirt.Example.JobWorker.Connectors.Common.Http/Models/OAuthTokenResponse.cs new file mode 100644 index 00000000..9561365c --- /dev/null +++ b/src/RedShirt.Example.JobWorker.Connectors.Common.Http/Models/OAuthTokenResponse.cs @@ -0,0 +1,14 @@ +namespace RedShirt.Example.JobWorker.Connectors.Common.Http.Models; + +/// +/// Result of a successful OAuth client-credentials token request. +/// +public class OAuthTokenResponse +{ + public required string AccessToken { get; init; } + + /// + /// UTC instant when the access token should be treated as expired. + /// + public required DateTimeOffset ExpiresAtUtc { get; init; } +} \ No newline at end of file diff --git a/src/RedShirt.Example.JobWorker.Connectors.Common.Http/RedShirt.Example.JobWorker.Connectors.Common.Http.csproj b/src/RedShirt.Example.JobWorker.Connectors.Common.Http/RedShirt.Example.JobWorker.Connectors.Common.Http.csproj new file mode 100644 index 00000000..2cacdc36 --- /dev/null +++ b/src/RedShirt.Example.JobWorker.Connectors.Common.Http/RedShirt.Example.JobWorker.Connectors.Common.Http.csproj @@ -0,0 +1,19 @@ + + + + net10.0 + enable + enable + + + + + + + + + + + + + diff --git a/src/RedShirt.Example.JobWorker.Connectors.Common.Http/Services/OAuthTokenCache.cs b/src/RedShirt.Example.JobWorker.Connectors.Common.Http/Services/OAuthTokenCache.cs new file mode 100644 index 00000000..52861a4a --- /dev/null +++ b/src/RedShirt.Example.JobWorker.Connectors.Common.Http/Services/OAuthTokenCache.cs @@ -0,0 +1,96 @@ +using RedShirt.Example.JobWorker.Connectors.Common.Http.Enums; +using RedShirt.Example.JobWorker.Connectors.Common.Http.Models; +using System.Collections.Concurrent; +using System.Security.Cryptography; +using System.Text; + +namespace RedShirt.Example.JobWorker.Connectors.Common.Http.Services; + +/// +/// Get a token from an OAuth provider. Results are cached. +/// +public interface IOAuthTokenCache +{ + /// + /// Get a token from an OAuth provider. + /// Tokens are cached based on a checksum derived from request properties. + /// + /// + /// Token endpoint and secret-manager paths for client credentials. + /// + /// + /// When , bypasses a still-valid cached token and requests a new one. + /// + /// + /// When , force-refreshes client id/secret via the token source + /// (and therefore also requests a new token). + /// + /// + /// Token used to cancel the operation. + /// + Task GetAsync(OAuthClientCredentialsRequest request, bool forceFreshToken, + bool forceFreshCredentials, CancellationToken cancellationToken = default); +} + +/// +/// In-memory OAuth access-token cache keyed by a checksum of +/// identity fields. +/// +public sealed class OAuthTokenCache(IOAuthTokenSource tokenSource) : IOAuthTokenCache +{ + private readonly ConcurrentDictionary _cache = new(); + + /// + /// Build a cache key derived from request parameters. + /// + /// + /// + private static string BuildCacheKeyChecksum(OAuthClientCredentialsRequest request) + { + var payload = string.Join('\n', + request.TokenUrl, + request.ClientIdPath, + request.ClientSecretPath, + request.ScopeLabel ?? string.Empty, + request.ScopeValue ?? string.Empty); + + var hash = SHA256.HashData(Encoding.UTF8.GetBytes(payload)); + return Convert.ToHexString(hash); + } + + /// + public async Task GetAsync(OAuthClientCredentialsRequest request, + bool forceFreshToken, bool forceFreshCredentials, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(request); + + var cacheKeyChecksum = BuildCacheKeyChecksum(request); + + // A new token shall be provided if forceFreshToken is true, credentials must be refreshed, + // or if the stored token is missing/expired. + if (!forceFreshToken && !forceFreshCredentials + && _cache.TryGetValue(cacheKeyChecksum, out var cached) + && cached.ExpiresAtUtc > DateTimeOffset.UtcNow) + { + return new OAuthTokenCacheResponse + { + AccessToken = cached.AccessToken, + ExpiresAtUtc = cached.ExpiresAtUtc, + TokenCacheState = TokenCacheState.CachedToken + }; + } + + var token = await tokenSource.GetTokenAsync(request, forceFreshCredentials, cancellationToken); + _cache[cacheKeyChecksum] = token; + + return new OAuthTokenCacheResponse + { + AccessToken = token.AccessToken, + ExpiresAtUtc = token.ExpiresAtUtc, + TokenCacheState = forceFreshCredentials + ? TokenCacheState.ForcedCredentialRetrieval + : TokenCacheState.FreshToken + }; + } +} \ No newline at end of file diff --git a/src/RedShirt.Example.JobWorker.Connectors.Common.Http/Services/OAuthTokenSource.cs b/src/RedShirt.Example.JobWorker.Connectors.Common.Http/Services/OAuthTokenSource.cs new file mode 100644 index 00000000..ba24d7ed --- /dev/null +++ b/src/RedShirt.Example.JobWorker.Connectors.Common.Http/Services/OAuthTokenSource.cs @@ -0,0 +1,162 @@ +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using RedShirt.Example.JobWorker.Common.SecretManagers.Core.Exceptions; +using RedShirt.Example.JobWorker.Common.SecretManagers.Core.Models; +using RedShirt.Example.JobWorker.Common.SecretManagers.Core.Services; +using RedShirt.Example.JobWorker.Connectors.Common.Http.Exceptions; +using RedShirt.Example.JobWorker.Connectors.Common.Http.Models; +using System.Net; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace RedShirt.Example.JobWorker.Connectors.Common.Http.Services; + +/// +/// Obtains OAuth 2.0 access tokens via the client-credentials grant. +/// +public interface IOAuthTokenSource +{ + /// + /// Requests an access token using client credentials resolved from the secret manager. + /// + /// + /// Token endpoint and secret-manager paths for client credentials. + /// + /// + /// When , force-refreshes client id/secret from the secret manager + /// (subject to the secret cache force-cooldown). + /// + /// + /// Token used to cancel the operation. + /// + /// + /// The access token and computed expiry. + /// + Task GetTokenAsync(OAuthClientCredentialsRequest request, bool force = false, + CancellationToken cancellationToken = default); +} + +/// +/// OAuth 2.0 client-credentials token source. Resolves client id/secret via +/// and posts a form-urlencoded token request. +/// +public sealed class OAuthTokenSource( + IHttpClientFactory httpClientFactory, + ISecretManagerCacheService secretManager, + ILogger logger, + IOptions options) : IOAuthTokenSource +{ + private const int DefaultFallbackJwtExpiryTimeMinutes = 30; + + public async Task GetTokenAsync(OAuthClientCredentialsRequest request, bool force = false, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(request); + + logger.LogTrace("Requesting OAuth client-credentials token from {TokenUrl} (force: {Force})", + request.TokenUrl, force); + + SecretManagerCacheSecretsResponse secretsResponse; + try + { + secretsResponse = await secretManager.GetSecretsAsync( + [request.ClientIdPath, request.ClientSecretPath], + force: force, + cancellationToken: cancellationToken); + } + catch (WorkerSecretManagerException e) + { + throw new OAuthRequestException(e.Message, e) + { + StatusCode = null, + CredentialStorageProblem = true, + // Assume that cache layer is working correctly and that the underlying secret manager behind the cache is misbehaving + FreshCredentialCacheResult = true + }; + } + + var parameters = new Dictionary + { + ["grant_type"] = "client_credentials", + ["client_id"] = secretsResponse.Values[request.ClientIdPath], + ["client_secret"] = secretsResponse.Values[request.ClientSecretPath] + }; + + if (!string.IsNullOrWhiteSpace(request.ScopeLabel) && !string.IsNullOrWhiteSpace(request.ScopeValue)) + { + parameters[request.ScopeLabel] = request.ScopeValue; + } + + using var httpRequest = new HttpRequestMessage(HttpMethod.Post, request.TokenUrl); + httpRequest.Content = new FormUrlEncodedContent(parameters); + + using var client = httpClientFactory.CreateClient(nameof(OAuthTokenSource)); + using var response = await client.SendAsync(httpRequest, cancellationToken); + + if (response.StatusCode != HttpStatusCode.OK) + { + logger.LogError( + "Failed to obtain OAuth token from {TokenUrl}: {StatusCodeValue} ({StatusCode})", + request.TokenUrl, (int) response.StatusCode, response.StatusCode); + throw new OAuthRequestException( + $"Response status code does not indicate success: {(int) response.StatusCode} ({response.StatusCode}).") + { + StatusCode = response.StatusCode, + CredentialStorageProblem = false, + FreshCredentialCacheResult = secretsResponse.QueriedSecretManager + }; + } + + var content = await response.Content.ReadAsStringAsync(cancellationToken); + + TokenEndpointResponse? responseObject; + try + { + responseObject = JsonSerializer.Deserialize(content); + } + catch (JsonException exception) + { + throw new OAuthRequestJsonException("OAuth token response could not be deserialized.", exception); + } + + if (string.IsNullOrWhiteSpace(responseObject?.AccessToken)) + { + throw new OAuthRequestJsonException("OAuth token response did not contain an access_token."); + } + + var issuedAtUtc = DateTimeOffset.UtcNow; + var expiresAtUtc = responseObject.ExpiresIn is > 0 + ? issuedAtUtc.AddSeconds(responseObject.ExpiresIn.Value) + : issuedAtUtc.Add(options.Value.EffectiveFallbackJwtExpiryTimeMinutes); + + return new OAuthTokenResponse + { + AccessToken = responseObject.AccessToken, + ExpiresAtUtc = expiresAtUtc + }; + } + + public sealed class ConfigurationModel + { + /// + /// Minutes to treat an access token as valid when the token response omits or has an invalid + /// expires_in. When null, is used. + /// + public required int? FallbackJwtExpiryTimeMinutes { get; init; } + + /// + /// Effective fallback lifetime for tokens without a usable expires_in. + /// + public TimeSpan EffectiveFallbackJwtExpiryTimeMinutes => + TimeSpan.FromMinutes(Math.Max(1, FallbackJwtExpiryTimeMinutes ?? DefaultFallbackJwtExpiryTimeMinutes)); + } + + private sealed class TokenEndpointResponse + { + [JsonPropertyName("access_token")] + public string? AccessToken { get; init; } + + [JsonPropertyName("expires_in")] + public int? ExpiresIn { get; init; } + } +} \ No newline at end of file diff --git a/src/RedShirt.Example.JobWorker.Core.Logic/JobLogicRunner.cs b/src/RedShirt.Example.JobWorker.Core.Logic/JobLogicRunner.cs index 42196d00..731e3c15 100644 --- a/src/RedShirt.Example.JobWorker.Core.Logic/JobLogicRunner.cs +++ b/src/RedShirt.Example.JobWorker.Core.Logic/JobLogicRunner.cs @@ -3,15 +3,24 @@ using RedShirt.Example.JobWorker.Common.Models; using RedShirt.Example.JobWorker.Common.Services.Abstractions; using RedShirt.Example.JobWorker.Common.Services.Utility; +using RedShirt.Example.JobWorker.Connectors.Bar.Core.Services; namespace RedShirt.Example.JobWorker.Core.Logic; -internal sealed class JobLogicRunner(ISleepService sleepService, ILogger logger) : IJobLogicRunner +internal sealed class JobLogicRunner( + IBarConnector barConnector, + ISleepService sleepService, + ILogger logger) : IJobLogicRunner { public async Task RunAsync(IJobModel job, CancellationToken cancellationToken = default) { logger.LogInformation("Sleeping for {DurationSeconds} seconds", job.Data.SleepDurationSeconds); await sleepService.DelayAsync(TimeSpan.FromSeconds(job.Data.SleepDurationSeconds), cancellationToken); + + var barId = Math.Max(1, job.Data.SleepDurationSeconds); + var barRecord = await barConnector.GetByIdAsync(barId, cancellationToken); + logger.LogInformation("Bar record {BarId} resolved to {BarName}", barRecord.Id, barRecord.Name); + return new JobLogicRunnerResponse { Result = JobResult.Success diff --git a/src/RedShirt.Example.JobWorker.Core.Logic/RedShirt.Example.JobWorker.Core.Logic.csproj b/src/RedShirt.Example.JobWorker.Core.Logic/RedShirt.Example.JobWorker.Core.Logic.csproj index 34af87c1..4399c1c3 100644 --- a/src/RedShirt.Example.JobWorker.Core.Logic/RedShirt.Example.JobWorker.Core.Logic.csproj +++ b/src/RedShirt.Example.JobWorker.Core.Logic/RedShirt.Example.JobWorker.Core.Logic.csproj @@ -12,6 +12,7 @@ + diff --git a/src/RedShirt.Example.JobWorker/Extensions/ServiceCollectionExtensions.cs b/src/RedShirt.Example.JobWorker/Extensions/ServiceCollectionExtensions.cs index c6978853..8e7acf06 100644 --- a/src/RedShirt.Example.JobWorker/Extensions/ServiceCollectionExtensions.cs +++ b/src/RedShirt.Example.JobWorker/Extensions/ServiceCollectionExtensions.cs @@ -4,6 +4,7 @@ using RedShirt.Example.JobWorker.Common.Azure.KeyVaultSecretManager.Extensions; using RedShirt.Example.JobWorker.Common.Distributed.Extensions; using RedShirt.Example.JobWorker.Common.SecretManagers.Core.Extensions; +using RedShirt.Example.JobWorker.Connectors.Bar.Implementation.Extensions; using RedShirt.Example.JobWorker.Core.Extensions; using RedShirt.Example.JobWorker.Core.Logic.Extensions; using RedShirt.Example.JobWorker.JobManagement.ActiveMq.Extensions; @@ -68,7 +69,9 @@ public static IServiceCollection ConfigureWorker(this IServiceCollection service // Core job handling .AddCoreJobManagement(configuration) // Implementation logic - .AddCoreLogic(configuration); + .AddCoreLogic(configuration) + // Bar connector (stand-in for an OAuth API client; see docs/bar-connector.md) + .AddBarConnector(configuration); /* * Template note: diff --git a/src/RedShirt.Example.JobWorker/RedShirt.Example.JobWorker.csproj b/src/RedShirt.Example.JobWorker/RedShirt.Example.JobWorker.csproj index 8d40ecdc..3cd15d03 100644 --- a/src/RedShirt.Example.JobWorker/RedShirt.Example.JobWorker.csproj +++ b/src/RedShirt.Example.JobWorker/RedShirt.Example.JobWorker.csproj @@ -18,6 +18,7 @@ + diff --git a/test/RedShirt.Example.JobWorker.Core.Logic.UnitTests/Tests/JobLogicRunnerTests.cs b/test/RedShirt.Example.JobWorker.Core.Logic.UnitTests/Tests/JobLogicRunnerTests.cs index 0c74112a..134c9dc9 100644 --- a/test/RedShirt.Example.JobWorker.Core.Logic.UnitTests/Tests/JobLogicRunnerTests.cs +++ b/test/RedShirt.Example.JobWorker.Core.Logic.UnitTests/Tests/JobLogicRunnerTests.cs @@ -2,6 +2,8 @@ using RedShirt.Example.JobWorker.Common.Enums; using RedShirt.Example.JobWorker.Common.Models; using RedShirt.Example.JobWorker.Common.Services.Utility; +using RedShirt.Example.JobWorker.Connectors.Bar.Core.Models; +using RedShirt.Example.JobWorker.Connectors.Bar.Core.Services; namespace RedShirt.Example.JobWorker.Core.Logic.UnitTests.Tests; @@ -15,7 +17,15 @@ public async Task Test_RunAsync() .Setup(s => s.DelayAsync(TimeSpan.Zero, TestContext.Current.CancellationToken)) .Returns(Task.CompletedTask); - var jobLogicRunner = new JobLogicRunner(sleepService.Object, new NullLogger()); + var barConnector = new Mock(MockBehavior.Strict); + barConnector + .Setup(b => b.GetByIdAsync(1, TestContext.Current.CancellationToken)) + .ReturnsAsync(new GetBarConnectorResponse {Id = 1, Name = "Bar-1"}); + + var jobLogicRunner = new JobLogicRunner( + barConnector.Object, + sleepService.Object, + new NullLogger()); var jobData = new Mock(MockBehavior.Strict); jobData.Setup(j => j.SleepDurationSeconds).Returns(0); @@ -27,5 +37,6 @@ public async Task Test_RunAsync() Assert.Equal(JobResult.Success, result.Result); sleepService.Verify(s => s.DelayAsync(TimeSpan.Zero, TestContext.Current.CancellationToken), Times.Once); + barConnector.Verify(b => b.GetByIdAsync(1, TestContext.Current.CancellationToken), Times.Once); } } \ No newline at end of file diff --git a/test/local/docker-compose.yaml b/test/local/docker-compose.yaml index ac34e82b..f79ed1ce 100644 --- a/test/local/docker-compose.yaml +++ b/test/local/docker-compose.yaml @@ -154,6 +154,19 @@ services: depends_on: - azure-service-bus-mssql + # Mocks Bar OAuth token + Bar HTTP API (POST /oauth/token, POST/GET /api/bar). + # Stubs live under ./wiremock/bar/mappings. Admin UI: http://localhost:9101/__admin/ + wiremock-bar: + image: wiremock/wiremock:3.13.1 + ports: + - "9101:8080" + volumes: + - ./wiremock/bar:/home/wiremock + entrypoint: + - /docker-entrypoint.sh + - --global-response-templating + - --verbose + google-pubsub-emulator: image: gcr.io/google.com/cloudsdktool/google-cloud-cli:emulators container_name: pubsub @@ -229,7 +242,7 @@ services: ## Default is the SSM parameter path. Override for Azure Key Vault (e.g. common-distributed-redis). COMMON__DISTRIBUTED__REDIS__CONNECTION_STRING_PATH: "${COMMON__DISTRIBUTED__REDIS__CONNECTION_STRING_PATH:-/common/redis}" ## Common Secret Management Library (In-Memory Cache) - COMMON__SECRETS__CACHE__FORCE_COOLDOWN_SECONDS: "${COMMON__SECRETS__CACHE__FORCE_COOLDOWN_SECONDS:-60}" + COMMON__SECRETS__CACHE__FORCE_COOLDOWN_SECONDS: "${COMMON__SECRETS__CACHE__FORCE_COOLDOWN_SECONDS:-1}" ## Common Secret Management Library (Azure Key Vault) COMMON__SECRETS__AZURE_KEY_VAULT__KEY_VAULT_URL: https://azure-key-vault-emulator.vault.azure.net:4997 # Tell the worker to generate a syntactically valid auth token for James Gould's key vault emulator @@ -330,4 +343,15 @@ services: JOB_SOURCE__GOOGLE_PUB_SUB__WAIT_TIME_SECONDS: ${JOB_SOURCE__GOOGLE_PUB_SUB__WAIT_TIME_SECONDS:-0} JOB_SOURCE__GOOGLE_PUB_SUB__DLQ_NOT_ENABLED: true JOB_SOURCE__GOOGLE_PUB_SUB__MAXIMUM_RECEIVES: 3 + ## Bar connector (external HTTP dependency + OAuth client credentials) + CONNECTORS__BAR__BASE_URL: "${CONNECTORS__BAR__BASE_URL-http://wiremock-bar:8080}" + CONNECTORS__BAR__TOKEN_URL: "${CONNECTORS__BAR__TOKEN_URL-http://wiremock-bar:8080/oauth/token}" + CONNECTORS__BAR__CLIENT_ID_PATH: "${CONNECTORS__BAR__CLIENT_ID_PATH-/bar/oauth/client-id}" + CONNECTORS__BAR__CLIENT_SECRET_PATH: "${CONNECTORS__BAR__CLIENT_SECRET_PATH-/bar/oauth/client-secret}" + CONNECTORS__BAR__SCOPE_LABEL: "${CONNECTORS__BAR__SCOPE_LABEL-audience}" + CONNECTORS__BAR__SCOPE_VALUE: "${CONNECTORS__BAR__SCOPE_VALUE-https://bar.local/api}" + CONNECTORS__BAR__TOKEN_REFRESH_COOLDOWN_SECONDS: "${CONNECTORS__BAR__TOKEN_REFRESH_COOLDOWN_SECONDS-30}" + CONNECTORS__BAR__RETRY_COUNT: "${CONNECTORS__BAR__RETRY_COUNT-3}" + CONNECTORS__BAR__FALLBACK_JWT_EXPIRY_TIME_MINUTES: "${CONNECTORS__BAR__FALLBACK_JWT_EXPIRY_TIME_MINUTES-30}" + CONNECTORS__BAR__REASON_TO_WAIT_FALLBACK_SECONDS: "${CONNECTORS__BAR__REASON_TO_WAIT_FALLBACK_SECONDS-15}" diff --git a/test/local/make-local-aws-resources.sh b/test/local/make-local-aws-resources.sh index bf06a006..d0aac234 100755 --- a/test/local/make-local-aws-resources.sh +++ b/test/local/make-local-aws-resources.sh @@ -36,3 +36,13 @@ awslocal ssm put-parameter --overwrite --type String --name /activemq/password - awslocal ssm put-parameter --overwrite --type String --name /nats/user --value admin awslocal ssm put-parameter --overwrite --type String --name /nats/password --value admin + +# Bar OAuth (WireMock bar connector) + +awslocal ssm put-parameter --overwrite --type String \ + --name /bar/oauth/client-id \ + --value "local-bar-client-id" + +awslocal ssm put-parameter --overwrite --type String \ + --name /bar/oauth/client-secret \ + --value "local-bar-client-secret" diff --git a/test/local/readme.md b/test/local/readme.md index f7117dfc..ce0e1d6b 100644 --- a/test/local/readme.md +++ b/test/local/readme.md @@ -57,7 +57,7 @@ docker compose up -d ministack redis ./send-sqs-message.py 12 ``` -4. Before starting the worker, make sure none of the `USE_` environment variables are set to **1**, and unset `COMMON__DISTRIBUTED__REDIS__CONNECTION_STRING_PATH` so compose uses the default SSM path (`/common/redis`): +4. Before starting the worker, make sure none of the `USE_` environment variables are set to **1**, unset `COMMON__DISTRIBUTED__REDIS__CONNECTION_STRING_PATH` so compose uses the default SSM path (`/common/redis`), and unset `CONNECTORS__BAR__CLIENT_ID_PATH` and `CONNECTORS__BAR__CLIENT_SECRET_PATH` so compose uses the default SSM Bar OAuth paths: ```bash export USE_ACTIVEMQ=0 @@ -69,9 +69,10 @@ docker compose up -d ministack redis export USE_NATS=0 export USE_REDIS_STREAMS=0 export USE_RABBITMQ=0 - export USE_RABBITMQ_SUBSCRIBE=0 export USE_GOOGLE_PUB_SUB=0 unset COMMON__DISTRIBUTED__REDIS__CONNECTION_STRING_PATH + unset CONNECTORS__BAR__CLIENT_ID_PATH + unset CONNECTORS__BAR__CLIENT_SECRET_PATH ``` 5. Bring up the worker: @@ -103,7 +104,7 @@ To initialize Kinesis and queue sample messages: ``` 4. Before starting the worker, make sure that neither the `USE_KINESIS` is set to `1` and that other `USE_` environment variables are not set to `1`. - Unset `COMMON__DISTRIBUTED__REDIS__CONNECTION_STRING_PATH` so compose uses the default SSM path (`/common/redis`): + Unset `COMMON__DISTRIBUTED__REDIS__CONNECTION_STRING_PATH` so compose uses the default SSM path (`/common/redis`), and unset `CONNECTORS__BAR__CLIENT_ID_PATH` and `CONNECTORS__BAR__CLIENT_SECRET_PATH` so compose uses the default SSM Bar OAuth paths: ```bash export USE_ACTIVEMQ=0 @@ -118,6 +119,8 @@ To initialize Kinesis and queue sample messages: export USE_RABBITMQ_SUBSCRIBE=0 export USE_GOOGLE_PUB_SUB=0 unset COMMON__DISTRIBUTED__REDIS__CONNECTION_STRING_PATH + unset CONNECTORS__BAR__CLIENT_ID_PATH + unset CONNECTORS__BAR__CLIENT_SECRET_PATH ``` 5. Bring up the worker: @@ -149,7 +152,7 @@ To initialize Kafka and queue sample messages: ``` 4. Before starting the worker, make sure that `USE_KAFKA` is set to `1` and that other `USE_` environment variables are not set to `1`. - Unset `COMMON__DISTRIBUTED__REDIS__CONNECTION_STRING_PATH` so compose uses the default SSM path (`/common/redis`):: + Unset `COMMON__DISTRIBUTED__REDIS__CONNECTION_STRING_PATH` so compose uses the default SSM path (`/common/redis`), and unset `CONNECTORS__BAR__CLIENT_ID_PATH` and `CONNECTORS__BAR__CLIENT_SECRET_PATH` so compose uses the default SSM Bar OAuth paths: ```bash export USE_KAFKA=1 @@ -164,6 +167,8 @@ To initialize Kafka and queue sample messages: export USE_RABBITMQ_SUBSCRIBE=0 export USE_GOOGLE_PUB_SUB=0 unset COMMON__DISTRIBUTED__REDIS__CONNECTION_STRING_PATH + unset CONNECTORS__BAR__CLIENT_ID_PATH + unset CONNECTORS__BAR__CLIENT_SECRET_PATH ``` 5. Bring up the worker: @@ -201,7 +206,7 @@ To initialize Apache Pulsar and queue sample messages: ``` 5. Before starting the worker, make sure that `USE_PULSAR` is set to `1` and that other `USE_` environment variables are not set to `1`. - Unset `COMMON__DISTRIBUTED__REDIS__CONNECTION_STRING_PATH` so compose uses the default SSM path (`/common/redis`): + Unset `COMMON__DISTRIBUTED__REDIS__CONNECTION_STRING_PATH` so compose uses the default SSM path (`/common/redis`), and unset `CONNECTORS__BAR__CLIENT_ID_PATH` and `CONNECTORS__BAR__CLIENT_SECRET_PATH` so compose uses the default SSM Bar OAuth paths: ```bash export USE_PULSAR=1 @@ -216,6 +221,8 @@ To initialize Apache Pulsar and queue sample messages: export USE_REDIS_STREAMS=0 export USE_GOOGLE_PUB_SUB=0 unset COMMON__DISTRIBUTED__REDIS__CONNECTION_STRING_PATH + unset CONNECTORS__BAR__CLIENT_ID_PATH + unset CONNECTORS__BAR__CLIENT_SECRET_PATH ``` 6. Bring up the worker: @@ -264,7 +271,7 @@ To initialize RabbitMQ and queue messages: ``` 6. Before starting the worker, make sure that the `USE_RABBITMQ` is set to `1` and that other `USE_` environment variables are not set to `1`. - Unset `COMMON__DISTRIBUTED__REDIS__CONNECTION_STRING_PATH` so compose uses the default SSM path (`/common/redis`): + Unset `COMMON__DISTRIBUTED__REDIS__CONNECTION_STRING_PATH` so compose uses the default SSM path (`/common/redis`), and unset `CONNECTORS__BAR__CLIENT_ID_PATH` and `CONNECTORS__BAR__CLIENT_SECRET_PATH` so compose uses the default SSM Bar OAuth paths: ```bash export USE_RABBITMQ=1 @@ -279,6 +286,8 @@ To initialize RabbitMQ and queue messages: export USE_REDIS_STREAMS=0 export USE_GOOGLE_PUB_SUB=0 unset COMMON__DISTRIBUTED__REDIS__CONNECTION_STRING_PATH + unset CONNECTORS__BAR__CLIENT_ID_PATH + unset CONNECTORS__BAR__CLIENT_SECRET_PATH ``` 7. By default, RabbitMQ uses a short polling strategy. If you want to have RabbitMQ instead subscribe to a queue, then set `JOB_SOURCE__RABBITMQ__SUBSCRIBE`: @@ -341,7 +350,7 @@ To initialize ActiveMQ and queue messages: ``` 6. Before starting the worker, make sure that `USE_ACTIVEMQ` is set to `1` and that other `USE_` environment variables are not set to `1`. - Unset `COMMON__DISTRIBUTED__REDIS__CONNECTION_STRING_PATH` so compose uses the default SSM path (`/common/redis`): + Unset `COMMON__DISTRIBUTED__REDIS__CONNECTION_STRING_PATH` so compose uses the default SSM path (`/common/redis`), and unset `CONNECTORS__BAR__CLIENT_ID_PATH` and `CONNECTORS__BAR__CLIENT_SECRET_PATH` so compose uses the default SSM Bar OAuth paths: ```bash export USE_ACTIVEMQ=1 @@ -355,6 +364,8 @@ To initialize ActiveMQ and queue messages: export USE_RABBITMQ=0 export USE_GOOGLE_PUB_SUB=0 unset COMMON__DISTRIBUTED__REDIS__CONNECTION_STRING_PATH + unset CONNECTORS__BAR__CLIENT_ID_PATH + unset CONNECTORS__BAR__CLIENT_SECRET_PATH ``` 7. By default, ActiveMQ uses short polling. To subscribe with an async listener instead, set `JOB_SOURCE__ACTIVEMQ__SUBSCRIBE`: @@ -442,7 +453,7 @@ To install the `nats` command: ``` 6. Before starting the worker, make sure that the `USE_NATS` is set to `1` and that other `USE_` environment variables are not set to `1`. - Unset `COMMON__DISTRIBUTED__REDIS__CONNECTION_STRING_PATH` so compose uses the default SSM path (`/common/redis`): + Unset `COMMON__DISTRIBUTED__REDIS__CONNECTION_STRING_PATH` so compose uses the default SSM path (`/common/redis`), and unset `CONNECTORS__BAR__CLIENT_ID_PATH` and `CONNECTORS__BAR__CLIENT_SECRET_PATH` so compose uses the default SSM Bar OAuth paths: ```bash export USE_NATS=1 @@ -457,6 +468,8 @@ To install the `nats` command: export USE_RABBITMQ_SUBSCRIBE=0 export USE_GOOGLE_PUB_SUB=0 unset COMMON__DISTRIBUTED__REDIS__CONNECTION_STRING_PATH + unset CONNECTORS__BAR__CLIENT_ID_PATH + unset CONNECTORS__BAR__CLIENT_SECRET_PATH ``` 7. By default, NATS uses short polling (`NextAsync` / `FetchNoWaitAsync`). To consume continuously via JetStream `ConsumeAsync` instead, set `JOB_SOURCE__NATS__SUBSCRIBE`: @@ -503,7 +516,7 @@ Redis Streams testing requires the `redis` Python module to be installed. ``` 5. Before starting the worker, make sure that `USE_REDIS_STREAMS` is set to `1` and that the other `USE_` environment variables are not set to `1`. - Unset `COMMON__DISTRIBUTED__REDIS__CONNECTION_STRING_PATH` so compose uses the default SSM path (`/common/redis`): + Unset `COMMON__DISTRIBUTED__REDIS__CONNECTION_STRING_PATH` so compose uses the default SSM path (`/common/redis`), and unset `CONNECTORS__BAR__CLIENT_ID_PATH` and `CONNECTORS__BAR__CLIENT_SECRET_PATH` so compose uses the default SSM Bar OAuth paths: ```bash export USE_REDIS_STREAMS=1 @@ -516,6 +529,8 @@ Redis Streams testing requires the `redis` Python module to be installed. export USE_RABBITMQ=0 export USE_RABBITMQ_SUBSCRIBE=0 unset COMMON__DISTRIBUTED__REDIS__CONNECTION_STRING_PATH + unset CONNECTORS__BAR__CLIENT_ID_PATH + unset CONNECTORS__BAR__CLIENT_SECRET_PATH ``` 6. Bring up the worker: @@ -583,7 +598,7 @@ VSCode automatically knows how to point to your local `azurite` server after the The script sends a plain UTF-8 JSON body (`{"SleepDurationSeconds": 12}`). If you instead add messages with Azure Storage Explorer, note that its Add menu **stores the message as a Base64-encoded string by default**. So far, this seems to be unique to Storage Explorer. Because of this, **this template does not go out of its way to account for Base64**. However, you may wish to consider it if you are adapting this into an application that uses Azure Queue Storage. Any messages added via Storage Explorer should be stored as **Plain UTF-8**. 6. Before starting the worker, make sure that the `USE_AZURE_QUEUE_STORAGE` is set to `1` and that other `USE_` environment variables are not set to `1`. - You will also point Redis at the Key Vault secret name created by `set-azure-key-vault-secrets.py`, as the compose file's default is to use the SSM path (Azure Key Vault key and SSM Parameter Store path formats are entirely incompatible with one another): + You will also point Redis at the Key Vault secret name created by `set-azure-key-vault-secrets.py`, as the compose file's default is to use the SSM path (Azure Key Vault key and SSM Parameter Store path formats are entirely incompatible with one another). Set `CONNECTORS__BAR__CLIENT_ID_PATH` and `CONNECTORS__BAR__CLIENT_SECRET_PATH` to the Bar OAuth Key Vault secret names from the same script: ```bash export USE_AZURE_QUEUE_STORAGE=1 @@ -598,6 +613,8 @@ VSCode automatically knows how to point to your local `azurite` server after the export USE_RABBITMQ_SUBSCRIBE=0 export USE_GOOGLE_PUB_SUB=0 export COMMON__DISTRIBUTED__REDIS__CONNECTION_STRING_PATH=common-distributed-redis + export CONNECTORS__BAR__CLIENT_ID_PATH=bar-oauth-client-id + export CONNECTORS__BAR__CLIENT_SECRET_PATH=bar-oauth-client-secret ``` 7. Bring up the worker: @@ -667,7 +684,7 @@ pip install azure.servicebus azure.identity azure.keyvault ``` 10. Before starting the worker, make sure that the `USE_AZURE_SERVICE_BUS` is set to `1` and that other `USE_` environment variables are not set to `1`. - You will also point Redis at the Key Vault secret name created by `set-azure-key-vault-secrets.py`, as the compose file's default is to use the SSM path (Azure Key Vault key and SSM Parameter Store path formats are entirely incompatible with one another): + You will also point Redis at the Key Vault secret name created by `set-azure-key-vault-secrets.py`, as the compose file's default is to use the SSM path (Azure Key Vault key and SSM Parameter Store path formats are entirely incompatible with one another). Set `CONNECTORS__BAR__CLIENT_ID_PATH` and `CONNECTORS__BAR__CLIENT_SECRET_PATH` to the Bar OAuth Key Vault secret names from the same script: ```bash export USE_AZURE_QUEUE_STORAGE=0 @@ -679,9 +696,10 @@ pip install azure.servicebus azure.identity azure.keyvault export USE_ACTIVEMQ=0 export USE_KINESIS=0 export USE_RABBITMQ=0 - export USE_RABBITMQ_SUBSCRIBE=0 export USE_GOOGLE_PUB_SUB=0 export COMMON__DISTRIBUTED__REDIS__CONNECTION_STRING_PATH=common-distributed-redis + export CONNECTORS__BAR__CLIENT_ID_PATH=bar-oauth-client-id + export CONNECTORS__BAR__CLIENT_SECRET_PATH=bar-oauth-client-secret ``` 11. Bring up the worker: @@ -690,7 +708,7 @@ pip install azure.servicebus azure.identity azure.keyvault docker compose up worker ``` -## Google Pub/Sub +### Google Pub/Sub To initialize Google Pub/Sub and queue sample messages: @@ -719,7 +737,7 @@ To initialize Google Pub/Sub and queue sample messages: ``` 5. Before starting the worker, make sure that `USE_GOOGLE_PUB_SUB` is set to `1` and that other `USE_` environment variables are not set to `1`. - Unset `COMMON__DISTRIBUTED__REDIS__CONNECTION_STRING_PATH` so compose uses the default SSM path (`/common/redis`): + Unset `COMMON__DISTRIBUTED__REDIS__CONNECTION_STRING_PATH` so compose uses the default SSM path (`/common/redis`), and unset `CONNECTORS__BAR__CLIENT_ID_PATH` and `CONNECTORS__BAR__CLIENT_SECRET_PATH` so compose uses the default SSM Bar OAuth paths: ```bash export USE_GOOGLE_PUB_SUB=1 @@ -730,8 +748,9 @@ To initialize Google Pub/Sub and queue sample messages: export USE_ACTIVEMQ=0 export USE_KINESIS=0 export USE_RABBITMQ=0 - export USE_RABBITMQ_SUBSCRIBE=0 unset COMMON__DISTRIBUTED__REDIS__CONNECTION_STRING_PATH + unset CONNECTORS__BAR__CLIENT_ID_PATH + unset CONNECTORS__BAR__CLIENT_SECRET_PATH ``` 6. Bring up the worker: @@ -739,3 +758,83 @@ To initialize Google Pub/Sub and queue sample messages: ```bash docker compose up worker ``` + +## Bar WireMock stubs + +`wiremock-bar` mocks the Bar OAuth token endpoint and Bar HTTP API used by +`RedShirt.Example.JobWorker.Connectors.Bar.Implementation` (`BarApiClient` + +`OAuthTokenSource`). Mapping files live under `wiremock/bar/mappings/`. See also +[`docs/bar-connector.md`](../../docs/bar-connector.md) for adapting this connector to a real API. + +Default credentials (from `make-local-aws-resources.sh`): + +* SSM `/bar/oauth/client-id` → `local-bar-client-id` +* SSM `/bar/oauth/client-secret` → `local-bar-client-secret` +* Access token returned by the token stub → `local-bar-access-token` + +Compose points the worker at `http://wiremock-bar:8080` for both `BaseUrl` and +`TokenUrl` (`…/oauth/token`), with scope form field `audience=https://bar.local/api`. +From the host use `http://localhost:9101`. + +| Method | Path | Auth / body | Result | +|--------|----------------------------|-------------------------------------------------------------------------|-----------------------------------------------------------| +| POST | `/oauth/token` | form: `grant_type=client_credentials`, valid client id/secret, audience | 200 with `access_token` + `expires_in` | +| POST | `/oauth/token` | anything else | 401 `invalid_client` | +| POST | `/api/bar` | `Authorization: Bearer local-bar-access-token` | 200 with `{ "Id": , "Name": }` | +| GET | `/api/bar/{id}` | valid Bearer | 200 with `{ "Id": {id}, "Name": "Bar-{id}" }` | +| GET | `/api/bar/404` | valid Bearer | 404 (exercises not-found handling) | +| any | `/api/bar` or `/api/bar/…` | missing/invalid Bearer | 401 | + +Bring up WireMock with ministack (for SSM) and Redis before running jobs that call Bar: + +```bash +docker compose up -d ministack redis wiremock-bar +./make-local-aws-resources.sh +``` + +### Secret paths: SSM vs Azure Key Vault + +By default, compose uses SSM Parameter Store paths (`/bar/oauth/client-id`, `/bar/oauth/client-secret`). +Azure Key Vault secret names cannot contain slashes; when testing with the Key Vault emulator instead of SSM, +set Key Vault–friendly paths before starting the worker: + +```bash +export CONNECTORS__BAR__CLIENT_ID_PATH=bar-oauth-client-id +export CONNECTORS__BAR__CLIENT_SECRET_PATH=bar-oauth-client-secret +``` + +Seed those secrets with `set-azure-key-vault-secrets.py` (which sets `bar-oauth-client-id` and +`bar-oauth-client-secret` alongside the Azure queue/service bus and Redis entries). To return to SSM-backed +local testing, **unset** those overrides so compose falls back to the default `/bar/oauth/…` paths: + +```bash +unset CONNECTORS__BAR__CLIENT_ID_PATH +unset CONNECTORS__BAR__CLIENT_SECRET_PATH +``` + +### Testing Unauthorized Behaviour + +To put an invalid client secret in SSM (token endpoint will 401 once credentials are refreshed): + +```bash +./scripts/wiremock-bar/bar-set-ssm-oauth-secret.sh 'bogus-secret-value-here' +``` + +Same caching caveat as other OAuth samples: a successfully obtained bearer token stays cached until it fails or expires. Setting a bad secret in SSM alone does not invalidate an already-cached token. To force WireMock to reject the current token (and exercise refresh), use the rotation script below so the worker's cached token no longer matches WireMock's Authorization matcher—or restart the worker after changing secrets. + +Local Compose defaults `COMMON__SECRETS__CACHE__FORCE_COOLDOWN_SECONDS` and +`CONNECTORS__BAR__TOKEN_REFRESH_COOLDOWN_SECONDS` to short values so credential rotation can +recover on the next request. The rotate script waits briefly for those windows to +elapse before returning. + +### Testing Credential / Token Rotations + +To update the client secret in SSM *and* WireMock's in-memory stubs (token bodyPatterns, returned `access_token`, and API `Authorization` matchers): + +```bash +./scripts/wiremock-bar/bar-rotate-oauth-credentials.sh +# or: +./scripts/wiremock-bar/bar-rotate-oauth-credentials.sh 'my-new-secret' 'my-new-access-token' +``` + +This only updates in-memory WireMock stubs. Restarting `wiremock-bar` restores the mapping files under `wiremock/bar/`. After the script finishes, process another job — the connector should 401 once with the old bearer, refresh client credentials + token, then succeed with the rotated bearer. diff --git a/test/local/scripts/wiremock-bar/bar-rotate-oauth-credentials.sh b/test/local/scripts/wiremock-bar/bar-rotate-oauth-credentials.sh new file mode 100755 index 00000000..76494278 --- /dev/null +++ b/test/local/scripts/wiremock-bar/bar-rotate-oauth-credentials.sh @@ -0,0 +1,104 @@ +#!/bin/bash +set -euo pipefail + +# Rotate the Bar OAuth client secret in ministack SSM and update WireMock's in-memory stubs +# (token bodyPatterns, access_token response, and Authorization bearer matchers). +# Does not rewrite files under wiremock/bar/mappings/ — a WireMock restart restores those defaults. +# +# Usage: +# ./scripts/wiremock-bar/bar-rotate-oauth-credentials.sh [new-secret] [new-access-token] +# +# Examples: +# ./scripts/wiremock-bar/bar-rotate-oauth-credentials.sh +# ./scripts/wiremock-bar/bar-rotate-oauth-credentials.sh my-new-secret my-new-token +# +# Pair with ./scripts/wiremock-bar/bar-set-ssm-oauth-secret.sh (SSM only → token 401) then this script (SSM + WireMock → recovery). + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +WIREMOCK_URL="${WIREMOCK_URL:-http://localhost:9101}" +SSM_PARAM_NAME="${BAR_CLIENT_SECRET_SSM_PATH:-/bar/oauth/client-secret}" +NEW_SECRET="${1:-rotated-bar-client-secret-$(date +%s)}" +NEW_TOKEN="${2:-rotated-bar-access-token-$(date +%s)}" + +if ! command -v jq >/dev/null 2>&1; then + echo "jq is required to update WireMock stubs via the Admin API." >&2 + exit 1 +fi + +if ! command -v awslocal >/dev/null 2>&1; then + echo "awslocal is required to update the SSM parameter." >&2 + exit 1 +fi + +if ! curl -sf "${WIREMOCK_URL}/__admin/health" >/dev/null \ + && ! curl -sf "${WIREMOCK_URL}/__admin/mappings" >/dev/null; then + echo "WireMock Admin API is not reachable at ${WIREMOCK_URL}." >&2 + echo "Start it with: (cd \"${ROOT_DIR}\" && docker compose up -d wiremock-bar)" >&2 + exit 1 +fi + +echo "Setting SSM ${SSM_PARAM_NAME} → ${NEW_SECRET}" +AWS_DEFAULT_REGION=us-east-1 awslocal ssm put-parameter --overwrite --type String \ + --name "${SSM_PARAM_NAME}" \ + --value "${NEW_SECRET}" >/dev/null + +echo "Updating WireMock stubs at ${WIREMOCK_URL} (in-memory only)" +updated_count=0 +while IFS= read -r stub; do + id="$(jq -r '.id' <<<"${stub}")" + changed=0 + + if jq -e ' + (.request.urlPath == "/oauth/token") + and (.request.bodyPatterns // [] | map(select(.contains? | type == "string" and startswith("client_secret="))) | length > 0) + ' <<<"${stub}" >/dev/null 2>&1; then + stub="$(jq --arg secret "${NEW_SECRET}" --arg token "${NEW_TOKEN}" ' + .request.bodyPatterns |= map( + if (.contains? | type == "string" and startswith("client_secret=")) + then .contains = ("client_secret=" + $secret) + else . + end + ) + | if .response.jsonBody.access_token? then .response.jsonBody.access_token = $token else . end + ' <<<"${stub}")" + changed=1 + fi + + if jq -e '.request.headers.Authorization.equalTo? | type == "string" and startswith("Bearer ")' \ + <<<"${stub}" >/dev/null 2>&1; then + stub="$(jq --arg token "${NEW_TOKEN}" \ + '.request.headers.Authorization.equalTo = ("Bearer " + $token)' <<<"${stub}")" + changed=1 + fi + + if [[ "${changed}" -eq 0 ]]; then + continue + fi + + curl -sf -X PUT \ + -H 'Content-Type: application/json' \ + -d "${stub}" \ + "${WIREMOCK_URL}/__admin/mappings/${id}" >/dev/null + + updated_count=$((updated_count + 1)) + echo " updated stub ${id}" +done < <(curl -sf "${WIREMOCK_URL}/__admin/mappings" | jq -c '.mappings[]') + +if [[ "${updated_count}" -eq 0 ]]; then + echo "No WireMock stubs with OAuth secret or Bearer Authorization matchers were found." >&2 + exit 1 +fi + +SECRET_FORCE_COOLDOWN_SECONDS="${SECRET_FORCE_COOLDOWN_SECONDS:-1}" +TOKEN_REFRESH_COOLDOWN_SECONDS="${TOKEN_REFRESH_COOLDOWN_SECONDS:-1}" +WAIT_SECONDS="${SECRET_FORCE_COOLDOWN_SECONDS}" +if [[ "${TOKEN_REFRESH_COOLDOWN_SECONDS}" -gt "${WAIT_SECONDS}" ]]; then + WAIT_SECONDS="${TOKEN_REFRESH_COOLDOWN_SECONDS}" +fi +WAIT_SECONDS=$((WAIT_SECONDS + 1)) +echo "Waiting ${WAIT_SECONDS}s for local secret/token refresh cooldowns…" +sleep "${WAIT_SECONDS}" + +echo "Done. Rotated ${updated_count} stub(s)." +echo " client secret: ${NEW_SECRET}" +echo " access token: ${NEW_TOKEN}" diff --git a/test/local/scripts/wiremock-bar/bar-set-ssm-oauth-secret.sh b/test/local/scripts/wiremock-bar/bar-set-ssm-oauth-secret.sh new file mode 100755 index 00000000..1fe6efb4 --- /dev/null +++ b/test/local/scripts/wiremock-bar/bar-set-ssm-oauth-secret.sh @@ -0,0 +1,21 @@ +#!/bin/bash +set -euo pipefail + +# Set the Bar OAuth client secret in ministack SSM (/bar/oauth/client-secret). +# Does not update WireMock stubs. +# Default value is intentionally invalid against the default WireMock mappings (local-bar-client-secret). +# To rotate a secret WireMock will also accept: ./scripts/wiremock-bar/bar-rotate-oauth-credentials.sh [new-secret] [new-token] +# +# Usage: +# ./scripts/wiremock-bar/bar-set-ssm-oauth-secret.sh [secret] +# +# Examples: +# ./scripts/wiremock-bar/bar-set-ssm-oauth-secret.sh +# ./scripts/wiremock-bar/bar-set-ssm-oauth-secret.sh 'bogus-secret-value-here' + +SECRET="${1:-bad-bar-client-secret}" + +echo "Setting SSM /bar/oauth/client-secret → ${SECRET}" +AWS_DEFAULT_REGION=us-east-1 awslocal ssm put-parameter --overwrite --type String \ + --name /bar/oauth/client-secret \ + --value "${SECRET}" diff --git a/test/local/set-azure-key-vault-secrets.py b/test/local/set-azure-key-vault-secrets.py index 268d91ae..4bcfbb5d 100755 --- a/test/local/set-azure-key-vault-secrets.py +++ b/test/local/set-azure-key-vault-secrets.py @@ -54,3 +54,5 @@ def set(secret_name, secret_value): "Endpoint=sb://azure-service-bus-emulator;SharedAccessKeyName=RootManageSharedAccessKey;SharedAccessKey=SAS_KEY_VALUE;UseDevelopmentEmulator=true;", ) set('common-distributed-redis', 'redis:6379') + set('bar-oauth-client-id', 'local-bar-client-id') + set('bar-oauth-client-secret', 'local-bar-client-secret') diff --git a/test/local/wiremock/mappings/get-api-bar-404.json b/test/local/wiremock/mappings/get-api-bar-404.json new file mode 100644 index 00000000..b7c580ab --- /dev/null +++ b/test/local/wiremock/mappings/get-api-bar-404.json @@ -0,0 +1,22 @@ +{ + "priority": 1, + "name": "Bar get by id — not found", + "request": { + "method": "GET", + "urlPath": "/api/bar/404", + "headers": { + "Authorization": { + "equalTo": "Bearer local-bar-access-token" + } + } + }, + "response": { + "status": 404, + "headers": { + "Content-Type": "application/json" + }, + "jsonBody": { + "message": "Bar not found" + } + } +} diff --git a/test/local/wiremock/mappings/get-api-bar-by-id.json b/test/local/wiremock/mappings/get-api-bar-by-id.json new file mode 100644 index 00000000..184cdac5 --- /dev/null +++ b/test/local/wiremock/mappings/get-api-bar-by-id.json @@ -0,0 +1,21 @@ +{ + "priority": 5, + "name": "Bar get by id", + "request": { + "method": "GET", + "urlPathPattern": "/api/bar/[0-9]+", + "headers": { + "Authorization": { + "equalTo": "Bearer local-bar-access-token" + } + } + }, + "response": { + "status": 200, + "headers": { + "Content-Type": "application/json" + }, + "transformers": ["response-template"], + "body": "{\"Id\":{{request.pathSegments.[2]}},\"Name\":\"Bar-{{request.pathSegments.[2]}}\"}" + } +} diff --git a/test/local/wiremock/mappings/post-api-bar.json b/test/local/wiremock/mappings/post-api-bar.json new file mode 100644 index 00000000..d6f761a0 --- /dev/null +++ b/test/local/wiremock/mappings/post-api-bar.json @@ -0,0 +1,26 @@ +{ + "priority": 1, + "name": "Bar create", + "request": { + "method": "POST", + "urlPath": "/api/bar", + "headers": { + "Authorization": { + "equalTo": "Bearer local-bar-access-token" + } + }, + "bodyPatterns": [ + { + "matchesJsonPath": "$.Name" + } + ] + }, + "response": { + "status": 200, + "headers": { + "Content-Type": "application/json" + }, + "transformers": ["response-template"], + "body": "{\"Id\":{{randomInt lower=1 upper=999999}},\"Name\":\"{{jsonPath request.body '$.Name'}}\"}" + } +} diff --git a/test/local/wiremock/mappings/post-oauth-token-unauthorized.json b/test/local/wiremock/mappings/post-oauth-token-unauthorized.json new file mode 100644 index 00000000..08f20c8f --- /dev/null +++ b/test/local/wiremock/mappings/post-oauth-token-unauthorized.json @@ -0,0 +1,18 @@ +{ + "priority": 10, + "name": "Bar OAuth token — unauthorized", + "request": { + "method": "POST", + "urlPath": "/oauth/token" + }, + "response": { + "status": 401, + "headers": { + "Content-Type": "application/json" + }, + "jsonBody": { + "error": "invalid_client", + "error_description": "Unauthorized" + } + } +} diff --git a/test/local/wiremock/mappings/post-oauth-token.json b/test/local/wiremock/mappings/post-oauth-token.json new file mode 100644 index 00000000..4c7bb190 --- /dev/null +++ b/test/local/wiremock/mappings/post-oauth-token.json @@ -0,0 +1,38 @@ +{ + "priority": 1, + "name": "Bar OAuth token — valid client credentials", + "request": { + "method": "POST", + "urlPath": "/oauth/token", + "headers": { + "Content-Type": { + "contains": "application/x-www-form-urlencoded" + } + }, + "bodyPatterns": [ + { + "contains": "grant_type=client_credentials" + }, + { + "contains": "client_id=local-bar-client-id" + }, + { + "contains": "client_secret=local-bar-client-secret" + }, + { + "contains": "audience=https%3A%2F%2Fbar.local%2Fapi" + } + ] + }, + "response": { + "status": 200, + "headers": { + "Content-Type": "application/json" + }, + "jsonBody": { + "access_token": "local-bar-access-token", + "token_type": "Bearer", + "expires_in": 3600 + } + } +} diff --git a/test/local/wiremock/mappings/unauthorized-api-bar.json b/test/local/wiremock/mappings/unauthorized-api-bar.json new file mode 100644 index 00000000..381bce37 --- /dev/null +++ b/test/local/wiremock/mappings/unauthorized-api-bar.json @@ -0,0 +1,17 @@ +{ + "priority": 10, + "name": "Bar API unauthorized without valid bearer token", + "request": { + "method": "ANY", + "urlPathPattern": "/api/bar(/.*)?" + }, + "response": { + "status": 401, + "headers": { + "Content-Type": "application/json" + }, + "jsonBody": { + "message": "Unauthorized" + } + } +} From a1fa16353b6324b846e34561e34d2aa1303d5bf6 Mon Sep 17 00:00:00 2001 From: Alan Deutscher Date: Thu, 3 Sep 2026 09:21:58 -0700 Subject: [PATCH 02/18] readme updates for wiremock --- test/local/readme.md | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/test/local/readme.md b/test/local/readme.md index ce0e1d6b..c68f384c 100644 --- a/test/local/readme.md +++ b/test/local/readme.md @@ -39,10 +39,10 @@ export JOBS__LOADER_MODE__MINIMUM_BATCH_SIZE=5 To initialize SQS and queue sample messages: -1. Bring up ministack and Redis: +1. Bring up ministack, Redis, and `wiremock-bar`: ```bash -docker compose up -d ministack redis +docker compose up -d ministack redis wiremock-bar ``` 2. Run the `make-local-aws-resources.sh` script: @@ -85,10 +85,10 @@ docker compose up -d ministack redis To initialize Kinesis and queue sample messages: -1. Bring up ministack and Redis: +1. Bring up ministack, Redis, and `wiremock-bar`: ```bash - docker compose up -d ministack redis + docker compose up -d ministack redis wiremock-bar ``` 2. Run the `make-local-aws-resources.sh` script: @@ -133,10 +133,10 @@ To initialize Kinesis and queue sample messages: To initialize Kafka and queue sample messages: -1. Bring up ministack, Kafka, and Redis: +1. Bring up ministack, Kafka, Redis, and `wiremock-bar`: ```bash - docker compose up -d ministack kafka redis + docker compose up -d ministack kafka redis wiremock-bar ``` 2. Run the `make-local-aws-resources.sh` script (creates the SQS queue used for Kafka job failures): @@ -181,10 +181,10 @@ To initialize Kafka and queue sample messages: To initialize Apache Pulsar and queue sample messages: -1. Bring up ministack, Pulsar, and Redis: +1. Bring up ministack, Pulsar, Redis, and `wiremock-bar`: ```bash - docker compose up -d ministack pulsar redis + docker compose up -d ministack pulsar redis wiremock-bar ``` 2. Run the `make-local-aws-resources.sh` script (creates shared local AWS resources such as Redis SSM params): @@ -239,10 +239,10 @@ RabbitMQ takes a few more steps to set up than the other input sources. To initialize RabbitMQ and queue messages: -1. Bring up ministack and Redis: +1. Bring up ministack, Redis, and `wiremock-bar`: ```bash - docker compose up -d ministack redis + docker compose up -d ministack redis wiremock-bar ``` 2. Run the `make-local-aws-resources.sh` script: @@ -315,10 +315,10 @@ ActiveMQ Artemis takes a few more steps to set up than the other input sources. To initialize ActiveMQ and queue messages: -1. Bring up ministack and Redis: +1. Bring up ministack, Redis, and `wiremock-bar`: ```bash - docker compose up -d ministack redis + docker compose up -d ministack redis wiremock-bar ``` 2. Run the `make-local-aws-resources.sh` script: @@ -424,10 +424,10 @@ To install the `nats` command: #### Testing Messages -1. Bring up ministack and Redis: +1. Bring up ministack, Redis, and `wiremock-bar`: ```bash - docker compose up -d ministack redis + docker compose up -d ministack redis wiremock-bar ``` 2. Run the `make-local-aws-resources.sh` script: @@ -490,10 +490,10 @@ Redis Streams testing requires the `redis` Python module to be installed. #### Testing Messages -1. Bring up ministack and Redis: +1. Bring up ministack, Redis, and `wiremock-bar`: ```bash - docker compose up -d ministack redis + docker compose up -d ministack redis wiremock-bar ``` 2. Run the `make-local-aws-resources.sh` script: @@ -571,10 +571,10 @@ VSCode automatically knows how to point to your local `azurite` server after the ./generate-azure-key-vault-cert.sh ``` -2. Bring up `azure-key-vault-emulator` (which shall be holding the connection string for Azure Queue Storage) and Redis: +2. Bring up `azure-key-vault-emulator` (which shall be holding the connection string for Azure Queue Storage), Redis, and `wiremock-bar`: ```bash - docker compose up -d azure-key-vault-emulator redis + docker compose up -d azure-key-vault-emulator redis wiremock-bar ``` 3. Run `set-azure-key-vault-secrets.py` to set the connection strings for Azure Queue Storage, Azure Service Bus, and Redis (`common-distributed-redis`) in the Azure Key Vault emulator: @@ -639,10 +639,10 @@ pip install azure.servicebus azure.identity azure.keyvault ./generate-azure-key-vault-cert.sh ``` -2. Bring up `azure-key-vault-emulator` (which shall be holding the connection string for Azure Service Bus) and Redis: +2. Bring up `azure-key-vault-emulator` (which shall be holding the connection string for Azure Service Bus), Redis, and `wiremock-bar`: ```bash - docker compose up -d azure-key-vault-emulator redis + docker compose up -d azure-key-vault-emulator redis wiremock-bar ``` 3. Run `set-azure-key-vault-secrets.py` to set the connection strings for Azure Queue Storage, Azure Service Bus, and Redis (`common-distributed-redis`) in the Azure Key Vault emulator: @@ -712,10 +712,10 @@ pip install azure.servicebus azure.identity azure.keyvault To initialize Google Pub/Sub and queue sample messages: -1. Bring up the Pub/Sub emulator, ministack, and Redis: +1. Bring up the Pub/Sub emulator, ministack, Redis, and `wiremock-bar`: ```bash - docker compose up -d google-pubsub-emulator ministack redis + docker compose up -d google-pubsub-emulator ministack redis wiremock-bar ``` 2. Run the `make-local-aws-resources.sh` script (creates the Redis SSM parameter used for idempotency): From 2e987d5fb9d5d9c180b715a45f6e40c7bb772f98 Mon Sep 17 00:00:00 2001 From: Alan Deutscher Date: Thu, 3 Sep 2026 09:24:13 -0700 Subject: [PATCH 03/18] readme header corrections --- test/local/readme.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/local/readme.md b/test/local/readme.md index c68f384c..a9257a59 100644 --- a/test/local/readme.md +++ b/test/local/readme.md @@ -129,7 +129,7 @@ To initialize Kinesis and queue sample messages: docker compose up worker ``` -## Kafka +### Kafka To initialize Kafka and queue sample messages: @@ -177,7 +177,7 @@ To initialize Kafka and queue sample messages: docker compose up worker ``` -## Apache Pulsar +### Apache Pulsar To initialize Apache Pulsar and queue sample messages: From 9f1c707626b80310bf3a9fec27f06309917c8dc0 Mon Sep 17 00:00:00 2001 From: Alan Deutscher Date: Thu, 3 Sep 2026 09:30:53 -0700 Subject: [PATCH 04/18] Standardize/cleanup USE_ mentions in local testing readme --- test/local/readme.md | 126 +++++++++++++++++++++++-------------------- 1 file changed, 69 insertions(+), 57 deletions(-) diff --git a/test/local/readme.md b/test/local/readme.md index a9257a59..8b876e53 100644 --- a/test/local/readme.md +++ b/test/local/readme.md @@ -35,6 +35,21 @@ export JOBS__LOADER_MODE__MINIMUM_BATCH_SIZE=5 ## Message Sources +Each section below includes a block that sets every job-source toggle defined in `test/local/docker-compose.yaml`: + +* `USE_ACTIVEMQ` +* `USE_AZURE_QUEUE_STORAGE` +* `USE_AZURE_SERVICE_BUS` +* `USE_GOOGLE_PUB_SUB` +* `USE_KAFKA` +* `USE_KINESIS` +* `USE_NATS` +* `USE_PULSAR` +* `USE_RABBITMQ` +* `USE_REDIS_STREAMS` + +SQS is the default when all of these are `0`, there is no `USE_SQS` variable. + ### SQS To initialize SQS and queue sample messages: @@ -61,15 +76,15 @@ docker compose up -d ministack redis wiremock-bar ```bash export USE_ACTIVEMQ=0 - export USE_KINESIS=0 - export USE_KAFKA=0 - export USE_PULSAR=0 export USE_AZURE_QUEUE_STORAGE=0 export USE_AZURE_SERVICE_BUS=0 + export USE_GOOGLE_PUB_SUB=0 + export USE_KAFKA=0 + export USE_KINESIS=0 export USE_NATS=0 - export USE_REDIS_STREAMS=0 + export USE_PULSAR=0 export USE_RABBITMQ=0 - export USE_GOOGLE_PUB_SUB=0 + export USE_REDIS_STREAMS=0 unset COMMON__DISTRIBUTED__REDIS__CONNECTION_STRING_PATH unset CONNECTORS__BAR__CLIENT_ID_PATH unset CONNECTORS__BAR__CLIENT_SECRET_PATH @@ -108,16 +123,15 @@ To initialize Kinesis and queue sample messages: ```bash export USE_ACTIVEMQ=0 - export USE_KINESIS=1 - export USE_KAFKA=0 - export USE_PULSAR=0 export USE_AZURE_QUEUE_STORAGE=0 export USE_AZURE_SERVICE_BUS=0 + export USE_GOOGLE_PUB_SUB=0 + export USE_KAFKA=0 + export USE_KINESIS=1 export USE_NATS=0 - export USE_REDIS_STREAMS=0 + export USE_PULSAR=0 export USE_RABBITMQ=0 - export USE_RABBITMQ_SUBSCRIBE=0 - export USE_GOOGLE_PUB_SUB=0 + export USE_REDIS_STREAMS=0 unset COMMON__DISTRIBUTED__REDIS__CONNECTION_STRING_PATH unset CONNECTORS__BAR__CLIENT_ID_PATH unset CONNECTORS__BAR__CLIENT_SECRET_PATH @@ -155,17 +169,16 @@ To initialize Kafka and queue sample messages: Unset `COMMON__DISTRIBUTED__REDIS__CONNECTION_STRING_PATH` so compose uses the default SSM path (`/common/redis`), and unset `CONNECTORS__BAR__CLIENT_ID_PATH` and `CONNECTORS__BAR__CLIENT_SECRET_PATH` so compose uses the default SSM Bar OAuth paths: ```bash - export USE_KAFKA=1 - export USE_PULSAR=0 export USE_ACTIVEMQ=0 - export USE_KINESIS=0 export USE_AZURE_QUEUE_STORAGE=0 export USE_AZURE_SERVICE_BUS=0 + export USE_GOOGLE_PUB_SUB=0 + export USE_KAFKA=1 + export USE_KINESIS=0 export USE_NATS=0 - export USE_REDIS_STREAMS=0 + export USE_PULSAR=0 export USE_RABBITMQ=0 - export USE_RABBITMQ_SUBSCRIBE=0 - export USE_GOOGLE_PUB_SUB=0 + export USE_REDIS_STREAMS=0 unset COMMON__DISTRIBUTED__REDIS__CONNECTION_STRING_PATH unset CONNECTORS__BAR__CLIENT_ID_PATH unset CONNECTORS__BAR__CLIENT_SECRET_PATH @@ -209,17 +222,16 @@ To initialize Apache Pulsar and queue sample messages: Unset `COMMON__DISTRIBUTED__REDIS__CONNECTION_STRING_PATH` so compose uses the default SSM path (`/common/redis`), and unset `CONNECTORS__BAR__CLIENT_ID_PATH` and `CONNECTORS__BAR__CLIENT_SECRET_PATH` so compose uses the default SSM Bar OAuth paths: ```bash - export USE_PULSAR=1 - export USE_KAFKA=0 export USE_ACTIVEMQ=0 - export USE_KINESIS=0 export USE_AZURE_QUEUE_STORAGE=0 export USE_AZURE_SERVICE_BUS=0 + export USE_GOOGLE_PUB_SUB=0 + export USE_KAFKA=0 + export USE_KINESIS=0 export USE_NATS=0 + export USE_PULSAR=1 export USE_RABBITMQ=0 - export USE_RABBITMQ_SUBSCRIBE=0 export USE_REDIS_STREAMS=0 - export USE_GOOGLE_PUB_SUB=0 unset COMMON__DISTRIBUTED__REDIS__CONNECTION_STRING_PATH unset CONNECTORS__BAR__CLIENT_ID_PATH unset CONNECTORS__BAR__CLIENT_SECRET_PATH @@ -274,17 +286,16 @@ To initialize RabbitMQ and queue messages: Unset `COMMON__DISTRIBUTED__REDIS__CONNECTION_STRING_PATH` so compose uses the default SSM path (`/common/redis`), and unset `CONNECTORS__BAR__CLIENT_ID_PATH` and `CONNECTORS__BAR__CLIENT_SECRET_PATH` so compose uses the default SSM Bar OAuth paths: ```bash - export USE_RABBITMQ=1 - export USE_RABBITMQ_SUBSCRIBE=false - export USE_KINESIS=0 - export USE_KAFKA=0 - export USE_PULSAR=0 export USE_ACTIVEMQ=0 export USE_AZURE_QUEUE_STORAGE=0 export USE_AZURE_SERVICE_BUS=0 + export USE_GOOGLE_PUB_SUB=0 + export USE_KAFKA=0 + export USE_KINESIS=0 export USE_NATS=0 + export USE_PULSAR=0 + export USE_RABBITMQ=1 export USE_REDIS_STREAMS=0 - export USE_GOOGLE_PUB_SUB=0 unset COMMON__DISTRIBUTED__REDIS__CONNECTION_STRING_PATH unset CONNECTORS__BAR__CLIENT_ID_PATH unset CONNECTORS__BAR__CLIENT_SECRET_PATH @@ -354,15 +365,15 @@ To initialize ActiveMQ and queue messages: ```bash export USE_ACTIVEMQ=1 - export USE_KINESIS=0 - export USE_KAFKA=0 - export USE_PULSAR=0 export USE_AZURE_QUEUE_STORAGE=0 export USE_AZURE_SERVICE_BUS=0 + export USE_GOOGLE_PUB_SUB=0 + export USE_KAFKA=0 + export USE_KINESIS=0 export USE_NATS=0 - export USE_REDIS_STREAMS=0 + export USE_PULSAR=0 export USE_RABBITMQ=0 - export USE_GOOGLE_PUB_SUB=0 + export USE_REDIS_STREAMS=0 unset COMMON__DISTRIBUTED__REDIS__CONNECTION_STRING_PATH unset CONNECTORS__BAR__CLIENT_ID_PATH unset CONNECTORS__BAR__CLIENT_SECRET_PATH @@ -456,17 +467,16 @@ To install the `nats` command: Unset `COMMON__DISTRIBUTED__REDIS__CONNECTION_STRING_PATH` so compose uses the default SSM path (`/common/redis`), and unset `CONNECTORS__BAR__CLIENT_ID_PATH` and `CONNECTORS__BAR__CLIENT_SECRET_PATH` so compose uses the default SSM Bar OAuth paths: ```bash - export USE_NATS=1 export USE_ACTIVEMQ=0 - export USE_KAFKA=0 - export USE_PULSAR=0 export USE_AZURE_QUEUE_STORAGE=0 export USE_AZURE_SERVICE_BUS=0 + export USE_GOOGLE_PUB_SUB=0 + export USE_KAFKA=0 export USE_KINESIS=0 - export USE_REDIS_STREAMS=0 + export USE_NATS=1 + export USE_PULSAR=0 export USE_RABBITMQ=0 - export USE_RABBITMQ_SUBSCRIBE=0 - export USE_GOOGLE_PUB_SUB=0 + export USE_REDIS_STREAMS=0 unset COMMON__DISTRIBUTED__REDIS__CONNECTION_STRING_PATH unset CONNECTORS__BAR__CLIENT_ID_PATH unset CONNECTORS__BAR__CLIENT_SECRET_PATH @@ -519,15 +529,16 @@ Redis Streams testing requires the `redis` Python module to be installed. Unset `COMMON__DISTRIBUTED__REDIS__CONNECTION_STRING_PATH` so compose uses the default SSM path (`/common/redis`), and unset `CONNECTORS__BAR__CLIENT_ID_PATH` and `CONNECTORS__BAR__CLIENT_SECRET_PATH` so compose uses the default SSM Bar OAuth paths: ```bash - export USE_REDIS_STREAMS=1 - export USE_NATS=0 export USE_ACTIVEMQ=0 - export USE_KAFKA=0 export USE_AZURE_QUEUE_STORAGE=0 export USE_AZURE_SERVICE_BUS=0 + export USE_GOOGLE_PUB_SUB=0 + export USE_KAFKA=0 export USE_KINESIS=0 + export USE_NATS=0 + export USE_PULSAR=0 export USE_RABBITMQ=0 - export USE_RABBITMQ_SUBSCRIBE=0 + export USE_REDIS_STREAMS=1 unset COMMON__DISTRIBUTED__REDIS__CONNECTION_STRING_PATH unset CONNECTORS__BAR__CLIENT_ID_PATH unset CONNECTORS__BAR__CLIENT_SECRET_PATH @@ -601,17 +612,16 @@ VSCode automatically knows how to point to your local `azurite` server after the You will also point Redis at the Key Vault secret name created by `set-azure-key-vault-secrets.py`, as the compose file's default is to use the SSM path (Azure Key Vault key and SSM Parameter Store path formats are entirely incompatible with one another). Set `CONNECTORS__BAR__CLIENT_ID_PATH` and `CONNECTORS__BAR__CLIENT_SECRET_PATH` to the Bar OAuth Key Vault secret names from the same script: ```bash + export USE_ACTIVEMQ=0 export USE_AZURE_QUEUE_STORAGE=1 export USE_AZURE_SERVICE_BUS=0 - export USE_NATS=0 - export USE_REDIS_STREAMS=0 + export USE_GOOGLE_PUB_SUB=0 export USE_KAFKA=0 - export USE_PULSAR=0 - export USE_ACTIVEMQ=0 export USE_KINESIS=0 + export USE_NATS=0 + export USE_PULSAR=0 export USE_RABBITMQ=0 - export USE_RABBITMQ_SUBSCRIBE=0 - export USE_GOOGLE_PUB_SUB=0 + export USE_REDIS_STREAMS=0 export COMMON__DISTRIBUTED__REDIS__CONNECTION_STRING_PATH=common-distributed-redis export CONNECTORS__BAR__CLIENT_ID_PATH=bar-oauth-client-id export CONNECTORS__BAR__CLIENT_SECRET_PATH=bar-oauth-client-secret @@ -687,16 +697,16 @@ pip install azure.servicebus azure.identity azure.keyvault You will also point Redis at the Key Vault secret name created by `set-azure-key-vault-secrets.py`, as the compose file's default is to use the SSM path (Azure Key Vault key and SSM Parameter Store path formats are entirely incompatible with one another). Set `CONNECTORS__BAR__CLIENT_ID_PATH` and `CONNECTORS__BAR__CLIENT_SECRET_PATH` to the Bar OAuth Key Vault secret names from the same script: ```bash + export USE_ACTIVEMQ=0 export USE_AZURE_QUEUE_STORAGE=0 export USE_AZURE_SERVICE_BUS=1 - export USE_NATS=0 - export USE_REDIS_STREAMS=0 + export USE_GOOGLE_PUB_SUB=0 export USE_KAFKA=0 - export USE_PULSAR=0 - export USE_ACTIVEMQ=0 export USE_KINESIS=0 + export USE_NATS=0 + export USE_PULSAR=0 export USE_RABBITMQ=0 - export USE_GOOGLE_PUB_SUB=0 + export USE_REDIS_STREAMS=0 export COMMON__DISTRIBUTED__REDIS__CONNECTION_STRING_PATH=common-distributed-redis export CONNECTORS__BAR__CLIENT_ID_PATH=bar-oauth-client-id export CONNECTORS__BAR__CLIENT_SECRET_PATH=bar-oauth-client-secret @@ -740,14 +750,16 @@ To initialize Google Pub/Sub and queue sample messages: Unset `COMMON__DISTRIBUTED__REDIS__CONNECTION_STRING_PATH` so compose uses the default SSM path (`/common/redis`), and unset `CONNECTORS__BAR__CLIENT_ID_PATH` and `CONNECTORS__BAR__CLIENT_SECRET_PATH` so compose uses the default SSM Bar OAuth paths: ```bash - export USE_GOOGLE_PUB_SUB=1 + export USE_ACTIVEMQ=0 export USE_AZURE_QUEUE_STORAGE=0 export USE_AZURE_SERVICE_BUS=0 - export USE_NATS=0 + export USE_GOOGLE_PUB_SUB=1 export USE_KAFKA=0 - export USE_ACTIVEMQ=0 export USE_KINESIS=0 + export USE_NATS=0 + export USE_PULSAR=0 export USE_RABBITMQ=0 + export USE_REDIS_STREAMS=0 unset COMMON__DISTRIBUTED__REDIS__CONNECTION_STRING_PATH unset CONNECTORS__BAR__CLIENT_ID_PATH unset CONNECTORS__BAR__CLIENT_SECRET_PATH From a90b3d969ad2668fc6316ef4d3e47c3a3f90a2ea Mon Sep 17 00:00:00 2001 From: Alan Deutscher Date: Thu, 3 Sep 2026 09:46:28 -0700 Subject: [PATCH 05/18] Update job logic test --- .../JobLogicRunner.cs | 8 ++-- .../Tests/JobLogicRunnerTests.cs | 37 ++++++++++++++++++- test/local/readme.md | 3 +- .../{ => bar}/mappings/get-api-bar-404.json | 0 .../bar/mappings/get-api-bar-429.json | 23 ++++++++++++ .../{ => bar}/mappings/get-api-bar-by-id.json | 0 .../{ => bar}/mappings/post-api-bar.json | 0 .../post-oauth-token-unauthorized.json | 0 .../{ => bar}/mappings/post-oauth-token.json | 0 .../mappings/unauthorized-api-bar.json | 0 10 files changed, 65 insertions(+), 6 deletions(-) rename test/local/wiremock/{ => bar}/mappings/get-api-bar-404.json (100%) create mode 100644 test/local/wiremock/bar/mappings/get-api-bar-429.json rename test/local/wiremock/{ => bar}/mappings/get-api-bar-by-id.json (100%) rename test/local/wiremock/{ => bar}/mappings/post-api-bar.json (100%) rename test/local/wiremock/{ => bar}/mappings/post-oauth-token-unauthorized.json (100%) rename test/local/wiremock/{ => bar}/mappings/post-oauth-token.json (100%) rename test/local/wiremock/{ => bar}/mappings/unauthorized-api-bar.json (100%) diff --git a/src/RedShirt.Example.JobWorker.Core.Logic/JobLogicRunner.cs b/src/RedShirt.Example.JobWorker.Core.Logic/JobLogicRunner.cs index 731e3c15..b3fe8590 100644 --- a/src/RedShirt.Example.JobWorker.Core.Logic/JobLogicRunner.cs +++ b/src/RedShirt.Example.JobWorker.Core.Logic/JobLogicRunner.cs @@ -14,10 +14,12 @@ internal sealed class JobLogicRunner( { public async Task RunAsync(IJobModel job, CancellationToken cancellationToken = default) { - logger.LogInformation("Sleeping for {DurationSeconds} seconds", job.Data.SleepDurationSeconds); - await sleepService.DelayAsync(TimeSpan.FromSeconds(job.Data.SleepDurationSeconds), cancellationToken); + var requestedSleepSeconds = job.Data.SleepDurationSeconds; + var effectiveSleepSeconds = requestedSleepSeconds is 404 or 429 ? 1 : requestedSleepSeconds; + logger.LogInformation("Sleeping for {DurationSeconds} seconds", effectiveSleepSeconds); + await sleepService.DelayAsync(TimeSpan.FromSeconds(effectiveSleepSeconds), cancellationToken); - var barId = Math.Max(1, job.Data.SleepDurationSeconds); + var barId = Math.Max(1, requestedSleepSeconds); var barRecord = await barConnector.GetByIdAsync(barId, cancellationToken); logger.LogInformation("Bar record {BarId} resolved to {BarName}", barRecord.Id, barRecord.Name); diff --git a/test/RedShirt.Example.JobWorker.Core.Logic.UnitTests/Tests/JobLogicRunnerTests.cs b/test/RedShirt.Example.JobWorker.Core.Logic.UnitTests/Tests/JobLogicRunnerTests.cs index 134c9dc9..0ff805b7 100644 --- a/test/RedShirt.Example.JobWorker.Core.Logic.UnitTests/Tests/JobLogicRunnerTests.cs +++ b/test/RedShirt.Example.JobWorker.Core.Logic.UnitTests/Tests/JobLogicRunnerTests.cs @@ -10,7 +10,7 @@ namespace RedShirt.Example.JobWorker.Core.Logic.UnitTests.Tests; public class JobLogicRunnerTests { [Fact] - public async Task Test_RunAsync() + public async Task RunAsync_WhenSleepDurationIsZero_SleepsZeroSecondsAndCallsBarWithIdOne() { var sleepService = new Mock(MockBehavior.Strict); sleepService @@ -39,4 +39,37 @@ public async Task Test_RunAsync() sleepService.Verify(s => s.DelayAsync(TimeSpan.Zero, TestContext.Current.CancellationToken), Times.Once); barConnector.Verify(b => b.GetByIdAsync(1, TestContext.Current.CancellationToken), Times.Once); } -} \ No newline at end of file + + [Theory] + [InlineData(404)] + [InlineData(429)] + public async Task RunAsync_WhenSleepDurationIsBarTestId_SleepsOneSecondAndCallsBarWithThatId(int barTestId) + { + var sleepService = new Mock(MockBehavior.Strict); + sleepService + .Setup(s => s.DelayAsync(TimeSpan.FromSeconds(1), TestContext.Current.CancellationToken)) + .Returns(Task.CompletedTask); + + var barConnector = new Mock(MockBehavior.Strict); + barConnector + .Setup(b => b.GetByIdAsync(barTestId, TestContext.Current.CancellationToken)) + .ReturnsAsync(new GetBarConnectorResponse {Id = barTestId, Name = $"Bar-{barTestId}"}); + + var jobLogicRunner = new JobLogicRunner( + barConnector.Object, + sleepService.Object, + new NullLogger()); + + var jobData = new Mock(MockBehavior.Strict); + jobData.Setup(j => j.SleepDurationSeconds).Returns(barTestId); + + var job = new Mock(MockBehavior.Strict); + job.Setup(j => j.Data).Returns(jobData.Object); + + var result = await jobLogicRunner.RunAsync(job.Object, TestContext.Current.CancellationToken); + + Assert.Equal(JobResult.Success, result.Result); + sleepService.Verify(s => s.DelayAsync(TimeSpan.FromSeconds(1), TestContext.Current.CancellationToken), Times.Once); + barConnector.Verify(b => b.GetByIdAsync(barTestId, TestContext.Current.CancellationToken), Times.Once); + } +} diff --git a/test/local/readme.md b/test/local/readme.md index 8b876e53..78ff9fa1 100644 --- a/test/local/readme.md +++ b/test/local/readme.md @@ -795,6 +795,7 @@ From the host use `http://localhost:9101`. | POST | `/api/bar` | `Authorization: Bearer local-bar-access-token` | 200 with `{ "Id": , "Name": }` | | GET | `/api/bar/{id}` | valid Bearer | 200 with `{ "Id": {id}, "Name": "Bar-{id}" }` | | GET | `/api/bar/404` | valid Bearer | 404 (exercises not-found handling) | +| GET | `/api/bar/429` | valid Bearer | 429 with `Retry-After: 1` (exercises rate-limit handling) | | any | `/api/bar` or `/api/bar/…` | missing/invalid Bearer | 401 | Bring up WireMock with ministack (for SSM) and Redis before running jobs that call Bar: @@ -849,4 +850,4 @@ To update the client secret in SSM *and* WireMock's in-memory stubs (token bodyP ./scripts/wiremock-bar/bar-rotate-oauth-credentials.sh 'my-new-secret' 'my-new-access-token' ``` -This only updates in-memory WireMock stubs. Restarting `wiremock-bar` restores the mapping files under `wiremock/bar/`. After the script finishes, process another job — the connector should 401 once with the old bearer, refresh client credentials + token, then succeed with the rotated bearer. +This only updates in-memory WireMock stubs. Restarting `wiremock-bar` restores the mapping files under `wiremock/bar/mappings/`. After the script finishes, process another job — the connector should 401 once with the old bearer, refresh client credentials + token, then succeed with the rotated bearer. diff --git a/test/local/wiremock/mappings/get-api-bar-404.json b/test/local/wiremock/bar/mappings/get-api-bar-404.json similarity index 100% rename from test/local/wiremock/mappings/get-api-bar-404.json rename to test/local/wiremock/bar/mappings/get-api-bar-404.json diff --git a/test/local/wiremock/bar/mappings/get-api-bar-429.json b/test/local/wiremock/bar/mappings/get-api-bar-429.json new file mode 100644 index 00000000..b4533fdd --- /dev/null +++ b/test/local/wiremock/bar/mappings/get-api-bar-429.json @@ -0,0 +1,23 @@ +{ + "priority": 1, + "name": "Bar get by id — rate limited", + "request": { + "method": "GET", + "urlPath": "/api/bar/429", + "headers": { + "Authorization": { + "equalTo": "Bearer local-bar-access-token" + } + } + }, + "response": { + "status": 429, + "headers": { + "Content-Type": "application/json", + "Retry-After": "1" + }, + "jsonBody": { + "message": "Rate limit exceeded" + } + } +} diff --git a/test/local/wiremock/mappings/get-api-bar-by-id.json b/test/local/wiremock/bar/mappings/get-api-bar-by-id.json similarity index 100% rename from test/local/wiremock/mappings/get-api-bar-by-id.json rename to test/local/wiremock/bar/mappings/get-api-bar-by-id.json diff --git a/test/local/wiremock/mappings/post-api-bar.json b/test/local/wiremock/bar/mappings/post-api-bar.json similarity index 100% rename from test/local/wiremock/mappings/post-api-bar.json rename to test/local/wiremock/bar/mappings/post-api-bar.json diff --git a/test/local/wiremock/mappings/post-oauth-token-unauthorized.json b/test/local/wiremock/bar/mappings/post-oauth-token-unauthorized.json similarity index 100% rename from test/local/wiremock/mappings/post-oauth-token-unauthorized.json rename to test/local/wiremock/bar/mappings/post-oauth-token-unauthorized.json diff --git a/test/local/wiremock/mappings/post-oauth-token.json b/test/local/wiremock/bar/mappings/post-oauth-token.json similarity index 100% rename from test/local/wiremock/mappings/post-oauth-token.json rename to test/local/wiremock/bar/mappings/post-oauth-token.json diff --git a/test/local/wiremock/mappings/unauthorized-api-bar.json b/test/local/wiremock/bar/mappings/unauthorized-api-bar.json similarity index 100% rename from test/local/wiremock/mappings/unauthorized-api-bar.json rename to test/local/wiremock/bar/mappings/unauthorized-api-bar.json From 5e6308b56785fe079e87acc7b769c31bdccf7dec Mon Sep 17 00:00:00 2001 From: Alan Deutscher Date: Thu, 3 Sep 2026 10:05:59 -0700 Subject: [PATCH 06/18] Make Bar access toggleable --- .../Extensions/ServiceCollectionExtensions.cs | 1 + .../JobLogicRunner.cs | 38 +++++- .../Tests/JobLogicRunnerTests.cs | 113 ++++++++++++++++-- 3 files changed, 143 insertions(+), 9 deletions(-) diff --git a/src/RedShirt.Example.JobWorker.Core.Logic/Extensions/ServiceCollectionExtensions.cs b/src/RedShirt.Example.JobWorker.Core.Logic/Extensions/ServiceCollectionExtensions.cs index 26611f1a..cbf1ce29 100644 --- a/src/RedShirt.Example.JobWorker.Core.Logic/Extensions/ServiceCollectionExtensions.cs +++ b/src/RedShirt.Example.JobWorker.Core.Logic/Extensions/ServiceCollectionExtensions.cs @@ -10,6 +10,7 @@ public static IServiceCollection AddCoreLogic(this IServiceCollection services, IConfigurationRoot configuration) { return services + .Configure(configuration.GetSection("Jobs:JobLogic")) .AddSingleton(); } } \ No newline at end of file diff --git a/src/RedShirt.Example.JobWorker.Core.Logic/JobLogicRunner.cs b/src/RedShirt.Example.JobWorker.Core.Logic/JobLogicRunner.cs index b3fe8590..3e119c17 100644 --- a/src/RedShirt.Example.JobWorker.Core.Logic/JobLogicRunner.cs +++ b/src/RedShirt.Example.JobWorker.Core.Logic/JobLogicRunner.cs @@ -1,4 +1,5 @@ using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using RedShirt.Example.JobWorker.Common.Enums; using RedShirt.Example.JobWorker.Common.Models; using RedShirt.Example.JobWorker.Common.Services.Abstractions; @@ -10,13 +11,31 @@ namespace RedShirt.Example.JobWorker.Core.Logic; internal sealed class JobLogicRunner( IBarConnector barConnector, ISleepService sleepService, + IOptions options, ILogger logger) : IJobLogicRunner { public async Task RunAsync(IJobModel job, CancellationToken cancellationToken = default) { var requestedSleepSeconds = job.Data.SleepDurationSeconds; + + if (!options.Value.EffectiveAccessBarEnabled) + { + // Bar access is not enabled, just do standard sleep + logger.LogInformation("Sleeping for {DurationSeconds} seconds", requestedSleepSeconds); + await sleepService.DelayAsync(TimeSpan.FromSeconds(requestedSleepSeconds), cancellationToken); + + return new JobLogicRunnerResponse + { + Result = JobResult.Success + }; + } + + // Bar access is enabled + + // A value of 404 or 429 suggests a special var effectiveSleepSeconds = requestedSleepSeconds is 404 or 429 ? 1 : requestedSleepSeconds; - logger.LogInformation("Sleeping for {DurationSeconds} seconds", effectiveSleepSeconds); + logger.LogInformation("Sleeping for {DurationSeconds} seconds before accessing Bar connector", + effectiveSleepSeconds); await sleepService.DelayAsync(TimeSpan.FromSeconds(effectiveSleepSeconds), cancellationToken); var barId = Math.Max(1, requestedSleepSeconds); @@ -28,4 +47,21 @@ public async Task RunAsync(IJobModel job, CancellationT Result = JobResult.Success }; } + + internal sealed class ConfigurationModel + { + public string? AccessBarEnabled { get; init; } + + /// + /// Parsing of . Values greater than zero or bool-parsed true are treated as + /// enabled. + /// + public bool EffectiveAccessBarEnabled => !string.IsNullOrWhiteSpace(AccessBarEnabled) + && ( + (int.TryParse(AccessBarEnabled, out var intResult) + && intResult > 0) + || (bool.TryParse(AccessBarEnabled, out var boolResult) + && boolResult) + ); + } } \ No newline at end of file diff --git a/test/RedShirt.Example.JobWorker.Core.Logic.UnitTests/Tests/JobLogicRunnerTests.cs b/test/RedShirt.Example.JobWorker.Core.Logic.UnitTests/Tests/JobLogicRunnerTests.cs index 0ff805b7..e87fb093 100644 --- a/test/RedShirt.Example.JobWorker.Core.Logic.UnitTests/Tests/JobLogicRunnerTests.cs +++ b/test/RedShirt.Example.JobWorker.Core.Logic.UnitTests/Tests/JobLogicRunnerTests.cs @@ -1,4 +1,5 @@ using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; using RedShirt.Example.JobWorker.Common.Enums; using RedShirt.Example.JobWorker.Common.Models; using RedShirt.Example.JobWorker.Common.Services.Utility; @@ -9,8 +10,71 @@ namespace RedShirt.Example.JobWorker.Core.Logic.UnitTests.Tests; public class JobLogicRunnerTests { + private static IOptions CreateOptions(string? accessBarEnabled = null) + { + return Options.Create(new JobLogicRunner.ConfigurationModel {AccessBarEnabled = accessBarEnabled}); + } + + [Theory] + [InlineData(null, false)] + [InlineData("", false)] + [InlineData(" ", false)] + [InlineData("0", false)] + [InlineData("-1", false)] + [InlineData("false", false)] + [InlineData("False", false)] + [InlineData("FALSE", false)] + [InlineData("not-a-value", false)] + [InlineData("1", true)] + [InlineData("2", true)] + [InlineData("true", true)] + [InlineData("True", true)] + [InlineData("TRUE", true)] + public void ConfigurationModel_EffectiveAccessBarEnabled(string? accessBarEnabled, bool expected) + { + var configuration = new JobLogicRunner.ConfigurationModel + { + AccessBarEnabled = accessBarEnabled + }; + + Assert.Equal(expected, configuration.EffectiveAccessBarEnabled); + } + + [Theory] + [InlineData(404)] + [InlineData(429)] + public async Task RunAsync_WhenAccessBarDisabled_SleepsFullDurationForBarTestIds(int sleepDurationSeconds) + { + var sleepService = new Mock(MockBehavior.Strict); + sleepService + .Setup(s => s.DelayAsync(TimeSpan.FromSeconds(sleepDurationSeconds), TestContext.Current.CancellationToken)) + .Returns(Task.CompletedTask); + + var barConnector = new Mock(MockBehavior.Strict); + + var jobLogicRunner = new JobLogicRunner( + barConnector.Object, + sleepService.Object, + CreateOptions(), + new NullLogger()); + + var jobData = new Mock(MockBehavior.Strict); + jobData.Setup(j => j.SleepDurationSeconds).Returns(sleepDurationSeconds); + + var job = new Mock(MockBehavior.Strict); + job.Setup(j => j.Data).Returns(jobData.Object); + + var result = await jobLogicRunner.RunAsync(job.Object, TestContext.Current.CancellationToken); + + Assert.Equal(JobResult.Success, result.Result); + sleepService.Verify( + s => s.DelayAsync(TimeSpan.FromSeconds(sleepDurationSeconds), TestContext.Current.CancellationToken), + Times.Once); + barConnector.VerifyNoOtherCalls(); + } + [Fact] - public async Task RunAsync_WhenSleepDurationIsZero_SleepsZeroSecondsAndCallsBarWithIdOne() + public async Task RunAsync_WhenAccessBarDisabled_SleepsRequestedDurationAndSkipsBarConnector() { var sleepService = new Mock(MockBehavior.Strict); sleepService @@ -18,13 +82,11 @@ public async Task RunAsync_WhenSleepDurationIsZero_SleepsZeroSecondsAndCallsBarW .Returns(Task.CompletedTask); var barConnector = new Mock(MockBehavior.Strict); - barConnector - .Setup(b => b.GetByIdAsync(1, TestContext.Current.CancellationToken)) - .ReturnsAsync(new GetBarConnectorResponse {Id = 1, Name = "Bar-1"}); var jobLogicRunner = new JobLogicRunner( barConnector.Object, sleepService.Object, + CreateOptions(), new NullLogger()); var jobData = new Mock(MockBehavior.Strict); @@ -37,13 +99,14 @@ public async Task RunAsync_WhenSleepDurationIsZero_SleepsZeroSecondsAndCallsBarW Assert.Equal(JobResult.Success, result.Result); sleepService.Verify(s => s.DelayAsync(TimeSpan.Zero, TestContext.Current.CancellationToken), Times.Once); - barConnector.Verify(b => b.GetByIdAsync(1, TestContext.Current.CancellationToken), Times.Once); + barConnector.VerifyNoOtherCalls(); } [Theory] [InlineData(404)] [InlineData(429)] - public async Task RunAsync_WhenSleepDurationIsBarTestId_SleepsOneSecondAndCallsBarWithThatId(int barTestId) + public async Task RunAsync_WhenAccessBarEnabledAndSleepDurationIsBarTestId_SleepsOneSecondAndCallsBarWithThatId( + int barTestId) { var sleepService = new Mock(MockBehavior.Strict); sleepService @@ -58,6 +121,7 @@ public async Task RunAsync_WhenSleepDurationIsBarTestId_SleepsOneSecondAndCallsB var jobLogicRunner = new JobLogicRunner( barConnector.Object, sleepService.Object, + CreateOptions("true"), new NullLogger()); var jobData = new Mock(MockBehavior.Strict); @@ -69,7 +133,40 @@ public async Task RunAsync_WhenSleepDurationIsBarTestId_SleepsOneSecondAndCallsB var result = await jobLogicRunner.RunAsync(job.Object, TestContext.Current.CancellationToken); Assert.Equal(JobResult.Success, result.Result); - sleepService.Verify(s => s.DelayAsync(TimeSpan.FromSeconds(1), TestContext.Current.CancellationToken), Times.Once); + sleepService.Verify(s => s.DelayAsync(TimeSpan.FromSeconds(1), TestContext.Current.CancellationToken), + Times.Once); barConnector.Verify(b => b.GetByIdAsync(barTestId, TestContext.Current.CancellationToken), Times.Once); } -} + + [Fact] + public async Task RunAsync_WhenAccessBarEnabledAndSleepDurationIsZero_SleepsZeroSecondsAndCallsBarWithIdOne() + { + var sleepService = new Mock(MockBehavior.Strict); + sleepService + .Setup(s => s.DelayAsync(TimeSpan.Zero, TestContext.Current.CancellationToken)) + .Returns(Task.CompletedTask); + + var barConnector = new Mock(MockBehavior.Strict); + barConnector + .Setup(b => b.GetByIdAsync(1, TestContext.Current.CancellationToken)) + .ReturnsAsync(new GetBarConnectorResponse {Id = 1, Name = "Bar-1"}); + + var jobLogicRunner = new JobLogicRunner( + barConnector.Object, + sleepService.Object, + CreateOptions("true"), + new NullLogger()); + + var jobData = new Mock(MockBehavior.Strict); + jobData.Setup(j => j.SleepDurationSeconds).Returns(0); + + var job = new Mock(MockBehavior.Strict); + job.Setup(j => j.Data).Returns(jobData.Object); + + var result = await jobLogicRunner.RunAsync(job.Object, TestContext.Current.CancellationToken); + + Assert.Equal(JobResult.Success, result.Result); + sleepService.Verify(s => s.DelayAsync(TimeSpan.Zero, TestContext.Current.CancellationToken), Times.Once); + barConnector.Verify(b => b.GetByIdAsync(1, TestContext.Current.CancellationToken), Times.Once); + } +} \ No newline at end of file From 361381b487fa302fd02cfe1abc83113163b47150 Mon Sep 17 00:00:00 2001 From: Alan Deutscher Date: Thu, 3 Sep 2026 10:17:08 -0700 Subject: [PATCH 07/18] Express JOBS__JOB_LOGIC__ACCESS_BAR_ENABLED in local-testing documentation --- docs/bar-connector.md | 2 +- test/local/docker-compose.yaml | 1 + test/local/readme.md | 89 +++++++++++++++++++++++++++++----- 3 files changed, 80 insertions(+), 12 deletions(-) diff --git a/docs/bar-connector.md b/docs/bar-connector.md index 9dc9b785..03397180 100644 --- a/docs/bar-connector.md +++ b/docs/bar-connector.md @@ -2,7 +2,7 @@ The Bar connector projects (`Connectors.Bar.Core` and `Connectors.Bar.Implementation`) are a placeholder for an OAuth-backed HTTP API, such as the [RedShirt.Example.Api](https://github.com/adeutscher/RedShirt.Example.Api) template. -They stand in for a real API client that a JobWorker uses to perform long-running or downstream work against another +They are a generic placeholder for a real API client that a JobWorker uses to perform long-running or downstream work against another service. ## OAuth by default diff --git a/test/local/docker-compose.yaml b/test/local/docker-compose.yaml index f79ed1ce..87741319 100644 --- a/test/local/docker-compose.yaml +++ b/test/local/docker-compose.yaml @@ -230,6 +230,7 @@ services: JOBS__WORKER_THREAD_COUNT: "${JOBS__WORKER_THREAD_COUNT:-2}" JOBS__LOADER_MODE__ENABLED: "${JOBS__LOADER_MODE__ENABLED:-false}" JOBS__LOADER_MODE__MINIMUM_BATCH_SIZE: "${JOBS__LOADER_MODE__MINIMUM_BATCH_SIZE:-5}" + JOBS__JOB_LOGIC__ACCESS_BAR_ENABLED: "${JOBS__JOB_LOGIC__ACCESS_BAR_ENABLED:-false}" JOBS__IDEMPOTENCY__ENABLED: "${JOBS__IDEMPOTENCY__ENABLED:-true}" JOBS__IDEMPOTENCY__RESULT_CACHE_DURATION_SECONDS: "${JOBS__IDEMPOTENCY__RESULT_CACHE_DURATION_SECONDS:-300}" JOBS__IDEMPOTENCY__MONITOR_INTERVAL_SECONDS: "${JOBS__IDEMPOTENCY__MONITOR_INTERVAL_SECONDS:-30}" diff --git a/test/local/readme.md b/test/local/readme.md index 78ff9fa1..2f354eb2 100644 --- a/test/local/readme.md +++ b/test/local/readme.md @@ -90,7 +90,13 @@ docker compose up -d ministack redis wiremock-bar unset CONNECTORS__BAR__CLIENT_SECRET_PATH ``` -5. Bring up the worker: +5. Optionally set `JOBS__JOB_LOGIC__ACCESS_BAR_ENABLED` to `true` to exercise the Bar connector after each job's sleep. Compose defaults to `false`, and when disabled jobs only sleep for the requested duration. When enabled, a requested sleep time of `404` or `429` sleeps for only 1 second so you can demonstrate special Bar API responses without having to wait for that many seconds: + + ```bash + export JOBS__JOB_LOGIC__ACCESS_BAR_ENABLED=true + ``` + +6. Bring up the worker: ```bash docker compose up worker @@ -137,7 +143,13 @@ To initialize Kinesis and queue sample messages: unset CONNECTORS__BAR__CLIENT_SECRET_PATH ``` -5. Bring up the worker: +5. Optionally set `JOBS__JOB_LOGIC__ACCESS_BAR_ENABLED` to `true` to exercise the Bar connector after each job's sleep. Compose defaults to `false`, and when disabled jobs only sleep for the requested duration. When enabled, a requested sleep time of `404` or `429` sleeps for only 1 second so you can demonstrate special Bar API responses without having to wait for that many seconds: + + ```bash + export JOBS__JOB_LOGIC__ACCESS_BAR_ENABLED=true + ``` + +6. Bring up the worker: ```bash docker compose up worker @@ -184,7 +196,13 @@ To initialize Kafka and queue sample messages: unset CONNECTORS__BAR__CLIENT_SECRET_PATH ``` -5. Bring up the worker: +5. Optionally set `JOBS__JOB_LOGIC__ACCESS_BAR_ENABLED` to `true` to exercise the Bar connector after each job's sleep. Compose defaults to `false`, and when disabled jobs only sleep for the requested duration. When enabled, a requested sleep time of `404` or `429` sleeps for only 1 second so you can demonstrate special Bar API responses without having to wait for that many seconds: + + ```bash + export JOBS__JOB_LOGIC__ACCESS_BAR_ENABLED=true + ``` + +6. Bring up the worker: ```bash docker compose up worker @@ -237,7 +255,13 @@ To initialize Apache Pulsar and queue sample messages: unset CONNECTORS__BAR__CLIENT_SECRET_PATH ``` -6. Bring up the worker: +6. Optionally set `JOBS__JOB_LOGIC__ACCESS_BAR_ENABLED` to `true` to exercise the Bar connector after each job's sleep. Compose defaults to `false`, and when disabled jobs only sleep for the requested duration. When enabled, a requested sleep time of `404` or `429` sleeps for only 1 second so you can demonstrate special Bar API responses without having to wait for that many seconds: + + ```bash + export JOBS__JOB_LOGIC__ACCESS_BAR_ENABLED=true + ``` + +7. Bring up the worker: ```bash docker compose up worker @@ -306,7 +330,14 @@ To initialize RabbitMQ and queue messages: ```bash export JOB_SOURCE__RABBITMQ__SUBSCRIBE=true ``` -8. Bring up the worker: + +8. Optionally set `JOBS__JOB_LOGIC__ACCESS_BAR_ENABLED` to `true` to exercise the Bar connector after each job's sleep. Compose defaults to `false`, and when disabled jobs only sleep for the requested duration. When enabled, a requested sleep time of `404` or `429` sleeps for only 1 second so you can demonstrate special Bar API responses without having to wait for that many seconds: + + ```bash + export JOBS__JOB_LOGIC__ACCESS_BAR_ENABLED=true + ``` + +9. Bring up the worker: ```bash docker compose up worker @@ -385,7 +416,13 @@ To initialize ActiveMQ and queue messages: export JOB_SOURCE__ACTIVEMQ__SUBSCRIBE=true ``` -8. Bring up the worker: +8. Optionally set `JOBS__JOB_LOGIC__ACCESS_BAR_ENABLED` to `true` to exercise the Bar connector after each job's sleep. Compose defaults to `false`, and when disabled jobs only sleep for the requested duration. When enabled, a requested sleep time of `404` or `429` sleeps for only 1 second so you can demonstrate special Bar API responses without having to wait for that many seconds: + + ```bash + export JOBS__JOB_LOGIC__ACCESS_BAR_ENABLED=true + ``` + +9. Bring up the worker: ```bash docker compose up worker @@ -488,7 +525,13 @@ To install the `nats` command: export JOB_SOURCE__NATS__SUBSCRIBE=true ``` -8. Bring up the worker: +8. Optionally set `JOBS__JOB_LOGIC__ACCESS_BAR_ENABLED` to `true` to exercise the Bar connector after each job's sleep. Compose defaults to `false`, and when disabled jobs only sleep for the requested duration. When enabled, a requested sleep time of `404` or `429` sleeps for only 1 second so you can demonstrate special Bar API responses without having to wait for that many seconds: + + ```bash + export JOBS__JOB_LOGIC__ACCESS_BAR_ENABLED=true + ``` + +9. Bring up the worker: ```bash docker compose up worker @@ -544,7 +587,13 @@ Redis Streams testing requires the `redis` Python module to be installed. unset CONNECTORS__BAR__CLIENT_SECRET_PATH ``` -6. Bring up the worker: +6. Optionally set `JOBS__JOB_LOGIC__ACCESS_BAR_ENABLED` to `true` to exercise the Bar connector after each job's sleep. Compose defaults to `false`, and when disabled jobs only sleep for the requested duration. When enabled, a requested sleep time of `404` or `429` sleeps for only 1 second so you can demonstrate special Bar API responses without having to wait for that many seconds: + + ```bash + export JOBS__JOB_LOGIC__ACCESS_BAR_ENABLED=true + ``` + +7. Bring up the worker: ```bash docker compose up worker @@ -627,7 +676,13 @@ VSCode automatically knows how to point to your local `azurite` server after the export CONNECTORS__BAR__CLIENT_SECRET_PATH=bar-oauth-client-secret ``` -7. Bring up the worker: +7. Optionally set `JOBS__JOB_LOGIC__ACCESS_BAR_ENABLED` to `true` to exercise the Bar connector after each job's sleep. Compose defaults to `false`, and when disabled jobs only sleep for the requested duration. When enabled, a requested sleep time of `404` or `429` sleeps for only 1 second so you can demonstrate special Bar API responses without having to wait for that many seconds: + + ```bash + export JOBS__JOB_LOGIC__ACCESS_BAR_ENABLED=true + ``` + +8. Bring up the worker: ```bash docker compose up worker @@ -712,7 +767,13 @@ pip install azure.servicebus azure.identity azure.keyvault export CONNECTORS__BAR__CLIENT_SECRET_PATH=bar-oauth-client-secret ``` -11. Bring up the worker: +11. Optionally set `JOBS__JOB_LOGIC__ACCESS_BAR_ENABLED` to `true` to exercise the Bar connector after each job's sleep. Compose defaults to `false`, and when disabled jobs only sleep for the requested duration. When enabled, a requested sleep time of `404` or `429` sleeps for only 1 second so you can demonstrate special Bar API responses without having to wait for that many seconds: + + ```bash + export JOBS__JOB_LOGIC__ACCESS_BAR_ENABLED=true + ``` + +12. Bring up the worker: ```bash docker compose up worker @@ -765,7 +826,13 @@ To initialize Google Pub/Sub and queue sample messages: unset CONNECTORS__BAR__CLIENT_SECRET_PATH ``` -6. Bring up the worker: +6. Optionally set `JOBS__JOB_LOGIC__ACCESS_BAR_ENABLED` to `true` to exercise the Bar connector after each job's sleep. Compose defaults to `false`, and when disabled jobs only sleep for the requested duration. When enabled, a requested sleep time of `404` or `429` sleeps for only 1 second so you can demonstrate special Bar API responses without having to wait for that many seconds: + + ```bash + export JOBS__JOB_LOGIC__ACCESS_BAR_ENABLED=true + ``` + +7. Bring up the worker: ```bash docker compose up worker From 1d3f20a1d25b9d7c1e2615eeceebc12a38e50775 Mon Sep 17 00:00:00 2001 From: Alan Deutscher Date: Thu, 3 Sep 2026 10:25:11 -0700 Subject: [PATCH 08/18] Update documentation, set Cursor rules for consistency with documentation habits --- .cursor/rules/markdown-list-markers.mdc | 34 +++++++++++++++++++++ .cursor/rules/prose-punctuation.mdc | 40 +++++++++++++++++++++++++ .gitignore | 1 - docs/bar-connector.md | 36 +++++++++++----------- 4 files changed, 93 insertions(+), 18 deletions(-) create mode 100644 .cursor/rules/markdown-list-markers.mdc create mode 100644 .cursor/rules/prose-punctuation.mdc diff --git a/.cursor/rules/markdown-list-markers.mdc b/.cursor/rules/markdown-list-markers.mdc new file mode 100644 index 00000000..0848b8af --- /dev/null +++ b/.cursor/rules/markdown-list-markers.mdc @@ -0,0 +1,34 @@ +--- +description: Use asterisk markers for Markdown unordered list bullets +globs: **/*.md +alwaysApply: false +--- + +# Markdown list bullets + +In Markdown files, mark **unordered** list items with asterisks (`*`), not hyphens (`-`) or plus signs (`+`). + +## Rules + +* Use `*` for every unordered list bullet, including nested items under numbered steps. +* Preserve existing indentation (typically four spaces per nesting level under numbered lists). +* Ordered lists (`1.`, `2.`, …) are unchanged. +* When editing or adding list items, convert any `-` or `+` bullets you touch to `*`. + +## Examples + +```markdown + +* First item +* Second item + * Nested item + +1. Step one: + * Detail A + * Detail B + + +- First item +- Second item + - Nested item +``` diff --git a/.cursor/rules/prose-punctuation.mdc b/.cursor/rules/prose-punctuation.mdc new file mode 100644 index 00000000..be752fb9 --- /dev/null +++ b/.cursor/rules/prose-punctuation.mdc @@ -0,0 +1,40 @@ +--- +description: Avoid semicolons and em dashes in prose and documentation +alwaysApply: true +--- + +# Prose punctuation + +In user-facing text (documentation, comments, commit messages, PR descriptions, and assistant replies), avoid semicolons and em dashes. + +## Semicolons + +Do not use `;` to join clauses in prose. Prefer separate sentences, a comma with a conjunction, or a short list. + +```markdown + +Compose defaults to `false`; when disabled, jobs only sleep for the requested duration. + + +Compose defaults to `false`. When disabled, jobs only sleep for the requested duration. +``` + +This rule does **not** apply to code syntax. Keep semicolons where the language requires them (for example C# statement terminators). + +## Em dashes + +Do not use em dashes (`—`) or spaced hyphen em-dash stand-ins (` - ` used as a break). Prefer periods, commas, parentheses, or a colon. + +```markdown + +The connector retries indefinitely—until the job is cancelled. + + +The connector retries indefinitely until the job is cancelled. +``` + +Hyphens in compound words (`client-credentials`) and en dashes in ranges (`1–5`) are fine. + +## When editing existing text + +When you touch documentation or comments, rewrite nearby semicolon- or em-dash-heavy phrasing into the preferred style. diff --git a/.gitignore b/.gitignore index 56affe0a..79b6b12a 100644 --- a/.gitignore +++ b/.gitignore @@ -58,7 +58,6 @@ BenchmarkDotNet.Artifacts/ # Editor .vscode/ -.cursor/ # Python __pycache__/ diff --git a/docs/bar-connector.md b/docs/bar-connector.md index 03397180..78697f36 100644 --- a/docs/bar-connector.md +++ b/docs/bar-connector.md @@ -2,27 +2,28 @@ The Bar connector projects (`Connectors.Bar.Core` and `Connectors.Bar.Implementation`) are a placeholder for an OAuth-backed HTTP API, such as the [RedShirt.Example.Api](https://github.com/adeutscher/RedShirt.Example.Api) template. -They are a generic placeholder for a real API client that a JobWorker uses to perform long-running or downstream work against another -service. +They are a generic placeholder for a real API client that a JobWorker uses to perform long-running or downstream work +against another service. ## OAuth by default This sample connector assumes OAuth 2.0 client-credentials authentication. Authentication in this JobWorker template was -made with OAuth in mind due to personal preference for projects and because it supports the authorization model that +made with OAuth in mind because it supports the authorization model that the [API Template](https://github.com/adeutscher/RedShirt.Example.Api) is set up for. -If you are planning to use this for an API that instead uses static keys, the request handler implementation should be -pivoted to be more like the `FooConnector` in the [API Template](https://github.com/adeutscher/RedShirt.Example.Api). +If you are planning to use this template with an API that instead uses static keys, the request handler implementation +should be pivoted to be more like the `FooConnector` in +the [API Template](https://github.com/adeutscher/RedShirt.Example.Api). ## Rate limits and reasons to wait -The client respects `BarReasonToWaitException` (including `BarRateLimitedException` and +The client respects `BarReasonToWaitException` (including its inheritors `BarRateLimitedException` and `BarTemporarilyUnavailableException`, both defined in `Connectors.Bar.Implementation`) **indefinitely**. When a call receives a reason to wait, the connector delays using `ISleepService` for the value of `RetryAfter` when present, or a configurable fallback (default **15 seconds** -when that fallback is null). It then retries until the operation succeeds or the job's cancellation token is triggered. -Cancellation and overall job lifetime are the Core job worker configuration's concern; the Bar connector's duty is to -keep trying respectfully when rate limited or otherwise told to wait. +when that fallback is itself `null`). It then retries until the operation succeeds or the job's cancellation token is +triggered. Cancellation and overall job lifetime are the Core job worker configuration's concern. The Bar connector's +duty is to keep trying respectfully when rate limited or otherwise told to wait. ## Last-mile instructions @@ -31,26 +32,27 @@ These steps cannot be fully prescribed by a general template. Adapt them for you ### If you are using the [API Template](https://github.com/adeutscher/RedShirt.Example.Api) 1. Adjust your API implementation to publish an interop package to a NuGet repository, for example: - - [Azure DevOps Artifacts](https://learn.microsoft.com/en-us/azure/devops/artifacts/nuget/publish) - - [GitHub Packages](https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-nuget-registry) - - [Sonatype Nexus Repository](https://help.sonatype.com/en/nuget-repositories.html) + * [Azure DevOps Artifacts](https://learn.microsoft.com/en-us/azure/devops/artifacts/nuget/publish) + * [GitHub Packages](https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-nuget-registry) + * [Sonatype Nexus Repository](https://help.sonatype.com/en/nuget-repositories.html) 2. Rename `Bar.Core` and `Bar.Implementation` projects as appropriate for your target API. 3. Reference the interop NuGet package in your renamed implementation project. 4. Write wrapper clients for the relevant clients from the interop package that translate thrown `SwaggerException` instances to a generic `BarException` (renamed for your API): - - `BarException` translation should judge the status code in the `SwaggerException` to set `CouldBeTransient`. + * `BarException` translation should judge the status code in the `SwaggerException` to set `CouldBeTransient`. Abstract this decision into an exception arbiter service with a method called `CouldSwaggerExceptionBeTransient`. - - The exception to this translation is HTTP **429**, which should specifically become a `BarRateLimitedException` - with the value of the `Retry-After` header. + * The exception to this translation is HTTP **429** (Too Many Requests), which should specifically become a + `BarRateLimitedException` with the value of the `Retry-After` header. 5. Adjust your client factory to return the wrapper client. +6. Rename classes whose names begin with `Bar` as appropriate for your target API. ### If you are not using the [API Template](https://github.com/adeutscher/RedShirt.Example.Api) 1. Rename `Bar.Core` and `Bar.Implementation` projects as appropriate for your target API. 2. Adjust HTTP clients for the subject API so non-successful status codes translate to a generic `BarException`: - - Judge the status code to set `CouldBeTransient`. Abstract this into an exception arbiter service with a method + * Judge the status code to set `CouldBeTransient`. Abstract this into an exception arbiter service with a method called `CouldSwaggerExceptionBeTransient` (or an equivalent name if you are not using Swagger/OpenAPI clients). - - HTTP **429** should specifically become a `BarRateLimitedException` with the value of the `Retry-After` header. + * HTTP **429** should specifically become a `BarRateLimitedException` with the value of the `Retry-After` header. 3. Rename classes whose names begin with `Bar` as appropriate for your target API. ## Local testing From e8343fa37f80e577f97eb73087df73f4fa673bdf Mon Sep 17 00:00:00 2001 From: Alan Deutscher Date: Thu, 3 Sep 2026 10:29:16 -0700 Subject: [PATCH 09/18] documentation nudge --- docs/bar-connector.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/bar-connector.md b/docs/bar-connector.md index 78697f36..711f18e9 100644 --- a/docs/bar-connector.md +++ b/docs/bar-connector.md @@ -39,8 +39,9 @@ These steps cannot be fully prescribed by a general template. Adapt them for you 3. Reference the interop NuGet package in your renamed implementation project. 4. Write wrapper clients for the relevant clients from the interop package that translate thrown `SwaggerException` instances to a generic `BarException` (renamed for your API): - * `BarException` translation should judge the status code in the `SwaggerException` to set `CouldBeTransient`. - Abstract this decision into an exception arbiter service with a method called `CouldSwaggerExceptionBeTransient`. + * `BarException` translation should judge the status code in the `SwaggerException` to set `CouldBeTransient`. It is + recommended to abstract this decision into an exception arbiter service with a method called + `CouldSwaggerExceptionBeTransient`. * The exception to this translation is HTTP **429** (Too Many Requests), which should specifically become a `BarRateLimitedException` with the value of the `Retry-After` header. 5. Adjust your client factory to return the wrapper client. @@ -50,8 +51,9 @@ These steps cannot be fully prescribed by a general template. Adapt them for you 1. Rename `Bar.Core` and `Bar.Implementation` projects as appropriate for your target API. 2. Adjust HTTP clients for the subject API so non-successful status codes translate to a generic `BarException`: - * Judge the status code to set `CouldBeTransient`. Abstract this into an exception arbiter service with a method - called `CouldSwaggerExceptionBeTransient` (or an equivalent name if you are not using Swagger/OpenAPI clients). + * Judge the status code to set `CouldBeTransient`. It is recommended to abstract this into an exception arbiter + service with a method called `CouldSwaggerExceptionBeTransient` (or an equivalent name if you are not using + Swagger/OpenAPI clients). * HTTP **429** should specifically become a `BarRateLimitedException` with the value of the `Retry-After` header. 3. Rename classes whose names begin with `Bar` as appropriate for your target API. From c280d3210a396719b507e4e1b31ee9988d63d36d Mon Sep 17 00:00:00 2001 From: Alan Deutscher Date: Thu, 3 Sep 2026 10:29:59 -0700 Subject: [PATCH 10/18] remove placeholder link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7b814b75..2cada2cc 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ Repo features: * Caches results to prevent re-running of a job if received non-concurrently * Container health probes. * Sample API connector. - * For more information, see TOMATO. + * For more information, see [`docs/bar-connector.md`](docs/bar-connector.md). * Documentation for local testing (see `test/local/`) # Core Architecture From 13b8c28f95af66829426b091750651dac8790d92 Mon Sep 17 00:00:00 2001 From: Alan Deutscher Date: Thu, 3 Sep 2026 10:32:12 -0700 Subject: [PATCH 11/18] Update WireMock 429 Retry-After to be 10 seconds. --- test/local/wiremock/bar/mappings/get-api-bar-429.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/local/wiremock/bar/mappings/get-api-bar-429.json b/test/local/wiremock/bar/mappings/get-api-bar-429.json index b4533fdd..06666dcf 100644 --- a/test/local/wiremock/bar/mappings/get-api-bar-429.json +++ b/test/local/wiremock/bar/mappings/get-api-bar-429.json @@ -14,7 +14,7 @@ "status": 429, "headers": { "Content-Type": "application/json", - "Retry-After": "1" + "Retry-After": "10" }, "jsonBody": { "message": "Rate limit exceeded" From 06b7026213e6283f56d9215b7d2e377c6c4d3cd6 Mon Sep 17 00:00:00 2001 From: Alan Deutscher Date: Thu, 3 Sep 2026 10:34:13 -0700 Subject: [PATCH 12/18] phrasing --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 2cada2cc..e2fd715f 100644 --- a/README.md +++ b/README.md @@ -43,9 +43,9 @@ Repo features: * Prevents simultaneous execution of the same message in the event of a dropped message * Caches results to prevent re-running of a job if received non-concurrently * Container health probes. -* Sample API connector. +* Sample API connector that respects rate limit responses. * For more information, see [`docs/bar-connector.md`](docs/bar-connector.md). -* Documentation for local testing (see `test/local/`) +* Documentation for local testing (see `test/local/`). # Core Architecture From ad72b8746a2a21bc199bc483a9d91917cc37a011 Mon Sep 17 00:00:00 2001 From: Alan Deutscher Date: Thu, 3 Sep 2026 10:40:38 -0700 Subject: [PATCH 13/18] solution cleanup sweep --- .../Exceptions/BarRateLimitedException.cs | 2 +- .../Exceptions/BarReasonToWaitException.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Exceptions/BarRateLimitedException.cs b/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Exceptions/BarRateLimitedException.cs index 7b7c16ab..33a07d5e 100644 --- a/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Exceptions/BarRateLimitedException.cs +++ b/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Exceptions/BarRateLimitedException.cs @@ -13,4 +13,4 @@ public BarRateLimitedException(TimeSpan? retryAfter) CouldBeTransient = true; CouldBeExternallySolvable = true; } -} +} \ No newline at end of file diff --git a/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Exceptions/BarReasonToWaitException.cs b/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Exceptions/BarReasonToWaitException.cs index 4f3c7269..052d620b 100644 --- a/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Exceptions/BarReasonToWaitException.cs +++ b/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Exceptions/BarReasonToWaitException.cs @@ -17,4 +17,4 @@ internal abstract class BarReasonToWaitException : BarException protected BarReasonToWaitException(string message) : base(message) { } -} +} \ No newline at end of file From 107b1e516a27738cffeb8e0d84048304012c953e Mon Sep 17 00:00:00 2001 From: Alan Deutscher Date: Thu, 3 Sep 2026 10:52:13 -0700 Subject: [PATCH 14/18] Add unit tests --- RedShirt.Example.JobWorker.slnx | 6 + .../Globals.cs | 3 +- ...orker.Connectors.Bar.Core.UnitTests.csproj | 31 +++ .../Tests/Exceptions/BarExceptionTests.cs | 34 +++ .../BarRecordNotFoundExceptionTests.cs | 17 ++ .../Globals.cs | 2 + ...ectors.Bar.Implementation.UnitTests.csproj | 34 +++ .../Tests/Clients/BarApiClientTests.cs | 98 +++++++++ .../Tests/Helpers/StubHttpMessageHandler.cs | 14 ++ .../Tests/Services/BarConnectorTests.cs | 152 ++++++++++++++ .../BarExceptionArbiterServiceTests.cs | 194 ++++++++++++++++++ .../Resilience/BarRetryWrapperServiceTests.cs | 167 +++++++++++++++ .../Globals.cs | 2 + ...er.Connectors.Common.Http.UnitTests.csproj | 34 +++ .../Tests/Helpers/StubHttpMessageHandler.cs | 14 ++ .../Tests/Services/OAuthTokenCacheTests.cs | 135 ++++++++++++ .../Tests/Services/OAuthTokenSourceTests.cs | 179 ++++++++++++++++ 17 files changed, 1115 insertions(+), 1 deletion(-) create mode 100644 test/RedShirt.Example.JobWorker.Connectors.Bar.Core.UnitTests/RedShirt.Example.JobWorker.Connectors.Bar.Core.UnitTests.csproj create mode 100644 test/RedShirt.Example.JobWorker.Connectors.Bar.Core.UnitTests/Tests/Exceptions/BarExceptionTests.cs create mode 100644 test/RedShirt.Example.JobWorker.Connectors.Bar.Core.UnitTests/Tests/Exceptions/BarRecordNotFoundExceptionTests.cs create mode 100644 test/RedShirt.Example.JobWorker.Connectors.Bar.Implementation.UnitTests/Globals.cs create mode 100644 test/RedShirt.Example.JobWorker.Connectors.Bar.Implementation.UnitTests/RedShirt.Example.JobWorker.Connectors.Bar.Implementation.UnitTests.csproj create mode 100644 test/RedShirt.Example.JobWorker.Connectors.Bar.Implementation.UnitTests/Tests/Clients/BarApiClientTests.cs create mode 100644 test/RedShirt.Example.JobWorker.Connectors.Bar.Implementation.UnitTests/Tests/Helpers/StubHttpMessageHandler.cs create mode 100644 test/RedShirt.Example.JobWorker.Connectors.Bar.Implementation.UnitTests/Tests/Services/BarConnectorTests.cs create mode 100644 test/RedShirt.Example.JobWorker.Connectors.Bar.Implementation.UnitTests/Tests/Services/Resilience/BarExceptionArbiterServiceTests.cs create mode 100644 test/RedShirt.Example.JobWorker.Connectors.Bar.Implementation.UnitTests/Tests/Services/Resilience/BarRetryWrapperServiceTests.cs create mode 100644 test/RedShirt.Example.JobWorker.Connectors.Common.Http.UnitTests/Globals.cs create mode 100644 test/RedShirt.Example.JobWorker.Connectors.Common.Http.UnitTests/RedShirt.Example.JobWorker.Connectors.Common.Http.UnitTests.csproj create mode 100644 test/RedShirt.Example.JobWorker.Connectors.Common.Http.UnitTests/Tests/Helpers/StubHttpMessageHandler.cs create mode 100644 test/RedShirt.Example.JobWorker.Connectors.Common.Http.UnitTests/Tests/Services/OAuthTokenCacheTests.cs create mode 100644 test/RedShirt.Example.JobWorker.Connectors.Common.Http.UnitTests/Tests/Services/OAuthTokenSourceTests.cs diff --git a/RedShirt.Example.JobWorker.slnx b/RedShirt.Example.JobWorker.slnx index 68c6cbe8..95a36c0f 100644 --- a/RedShirt.Example.JobWorker.slnx +++ b/RedShirt.Example.JobWorker.slnx @@ -81,6 +81,12 @@ Path="test/RedShirt.Example.JobWorker.Common.SecretManagers.Core.UnitTests/RedShirt.Example.JobWorker.Common.SecretManagers.Core.UnitTests.csproj"/> + + + + + + net10.0 + enable + enable + false + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + + diff --git a/test/RedShirt.Example.JobWorker.Connectors.Bar.Core.UnitTests/Tests/Exceptions/BarExceptionTests.cs b/test/RedShirt.Example.JobWorker.Connectors.Bar.Core.UnitTests/Tests/Exceptions/BarExceptionTests.cs new file mode 100644 index 00000000..7ead4b54 --- /dev/null +++ b/test/RedShirt.Example.JobWorker.Connectors.Bar.Core.UnitTests/Tests/Exceptions/BarExceptionTests.cs @@ -0,0 +1,34 @@ +using RedShirt.Example.JobWorker.Connectors.Bar.Core.Exceptions; + +namespace RedShirt.Example.JobWorker.Connectors.Bar.Core.UnitTests.Tests.Exceptions; + +public class BarExceptionTests +{ + [Fact] + public void Constructor_WithInnerException_PreservesMessageAndInner() + { + var inner = new InvalidOperationException("underlying failure"); + + var exception = new BarException(inner) + { + IsHandled = true, + CouldBeTransient = false, + CouldBeExternallySolvable = true + }; + + Assert.Equal("underlying failure", exception.Message); + Assert.Same(inner, exception.InnerException); + Assert.True(exception.IsHandled); + Assert.False(exception.CouldBeTransient); + Assert.True(exception.CouldBeExternallySolvable); + } + + [Fact] + public void Constructor_WithMessage_PreservesMessage() + { + var exception = new BarException("classified failure"); + + Assert.Equal("classified failure", exception.Message); + Assert.Null(exception.InnerException); + } +} \ No newline at end of file diff --git a/test/RedShirt.Example.JobWorker.Connectors.Bar.Core.UnitTests/Tests/Exceptions/BarRecordNotFoundExceptionTests.cs b/test/RedShirt.Example.JobWorker.Connectors.Bar.Core.UnitTests/Tests/Exceptions/BarRecordNotFoundExceptionTests.cs new file mode 100644 index 00000000..53fa67f7 --- /dev/null +++ b/test/RedShirt.Example.JobWorker.Connectors.Bar.Core.UnitTests/Tests/Exceptions/BarRecordNotFoundExceptionTests.cs @@ -0,0 +1,17 @@ +using RedShirt.Example.JobWorker.Connectors.Bar.Core.Exceptions; + +namespace RedShirt.Example.JobWorker.Connectors.Bar.Core.UnitTests.Tests.Exceptions; + +public class BarRecordNotFoundExceptionTests +{ + [Fact] + public void Constructor_SetsIdAndMessage() + { + const int barId = 404; + + var exception = new BarRecordNotFoundException(barId); + + Assert.Equal(barId, exception.Id); + Assert.Equal("Bar record 404 was not found.", exception.Message); + } +} \ No newline at end of file diff --git a/test/RedShirt.Example.JobWorker.Connectors.Bar.Implementation.UnitTests/Globals.cs b/test/RedShirt.Example.JobWorker.Connectors.Bar.Implementation.UnitTests/Globals.cs new file mode 100644 index 00000000..43eebfc5 --- /dev/null +++ b/test/RedShirt.Example.JobWorker.Connectors.Bar.Implementation.UnitTests/Globals.cs @@ -0,0 +1,2 @@ +global using Moq; +global using Xunit; \ No newline at end of file diff --git a/test/RedShirt.Example.JobWorker.Connectors.Bar.Implementation.UnitTests/RedShirt.Example.JobWorker.Connectors.Bar.Implementation.UnitTests.csproj b/test/RedShirt.Example.JobWorker.Connectors.Bar.Implementation.UnitTests/RedShirt.Example.JobWorker.Connectors.Bar.Implementation.UnitTests.csproj new file mode 100644 index 00000000..d8250a21 --- /dev/null +++ b/test/RedShirt.Example.JobWorker.Connectors.Bar.Implementation.UnitTests/RedShirt.Example.JobWorker.Connectors.Bar.Implementation.UnitTests.csproj @@ -0,0 +1,34 @@ + + + + net10.0 + enable + enable + false + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + + diff --git a/test/RedShirt.Example.JobWorker.Connectors.Bar.Implementation.UnitTests/Tests/Clients/BarApiClientTests.cs b/test/RedShirt.Example.JobWorker.Connectors.Bar.Implementation.UnitTests/Tests/Clients/BarApiClientTests.cs new file mode 100644 index 00000000..17613f6e --- /dev/null +++ b/test/RedShirt.Example.JobWorker.Connectors.Bar.Implementation.UnitTests/Tests/Clients/BarApiClientTests.cs @@ -0,0 +1,98 @@ +using RedShirt.Example.JobWorker.Connectors.Bar.Core.Exceptions; +using RedShirt.Example.JobWorker.Connectors.Bar.Core.Models; +using RedShirt.Example.JobWorker.Connectors.Bar.Implementation.Clients; +using RedShirt.Example.JobWorker.Connectors.Bar.Implementation.Exceptions; +using RedShirt.Example.JobWorker.Connectors.Bar.Implementation.UnitTests.Tests.Helpers; +using System.Net; +using System.Text; +using System.Text.Json; + +namespace RedShirt.Example.JobWorker.Connectors.Bar.Implementation.UnitTests.Tests.Clients; + +public class BarApiClientTests +{ + private static BarApiClient CreateClient(StubHttpMessageHandler handler) + { + return new BarApiClient(new HttpClient(handler), "https://bar.local"); + } + + [Fact] + public async Task CreateBarAsync_WhenBodyIsNull_ThrowsJsonException() + { + var handler = new StubHttpMessageHandler(_ => new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent("null", Encoding.UTF8, "application/json") + }); + var client = CreateClient(handler); + + await Assert.ThrowsAsync(() => + client.CreateBarAsync(new CreateBarConnectorRequest {Name = "Created"}, + TestContext.Current.CancellationToken)); + } + + [Fact] + public async Task CreateBarAsync_WhenSuccess_ReturnsMappedResponse() + { + var handler = new StubHttpMessageHandler(request => + { + Assert.Equal(HttpMethod.Post, request.Method); + Assert.Equal("https://bar.local/api/bar", request.RequestUri?.ToString()); + + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent("{\"Id\":99,\"Name\":\"Created\"}", Encoding.UTF8, "application/json") + }; + }); + var client = CreateClient(handler); + + var response = await client.CreateBarAsync(new CreateBarConnectorRequest {Name = "Created"}, + TestContext.Current.CancellationToken); + + Assert.Equal(99, response.Id); + Assert.Equal("Created", response.Name); + } + + [Fact] + public async Task GetBarByIdAsync_WhenNotFound_ThrowsBarRecordNotFoundException() + { + var handler = new StubHttpMessageHandler(_ => + new HttpResponseMessage(HttpStatusCode.NotFound)); + var client = CreateClient(handler); + + var thrown = await Assert.ThrowsAsync(() => + client.GetBarByIdAsync(404, TestContext.Current.CancellationToken)); + + Assert.Equal(404, thrown.Id); + } + + [Fact] + public async Task GetBarByIdAsync_WhenRateLimited_ThrowsBarRateLimitedExceptionWithRetryAfter() + { + var response = new HttpResponseMessage(HttpStatusCode.TooManyRequests); + response.Headers.TryAddWithoutValidation("Retry-After", "3"); + var handler = new StubHttpMessageHandler(_ => response); + var client = CreateClient(handler); + + var thrown = await Assert.ThrowsAsync(() => + client.GetBarByIdAsync(429, TestContext.Current.CancellationToken)); + + Assert.Equal(TimeSpan.FromSeconds(3), thrown.RetryAfter); + } + + [Fact] + public async Task GetBarByIdAsync_WhenSuccess_ReturnsMappedResponse() + { + var handler = new StubHttpMessageHandler(_ => new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent("{\"Id\":12,\"Name\":\"Bar-12\"}", Encoding.UTF8, "application/json") + }); + var client = CreateClient(handler); + + var response = await client.GetBarByIdAsync(12, TestContext.Current.CancellationToken); + + Assert.Equal(12, response.Id); + Assert.Equal("Bar-12", response.Name); + Assert.Equal(HttpMethod.Get, handler.Requests[0].Method); + Assert.Equal("https://bar.local/api/bar/12", handler.Requests[0].RequestUri?.ToString()); + } +} \ No newline at end of file diff --git a/test/RedShirt.Example.JobWorker.Connectors.Bar.Implementation.UnitTests/Tests/Helpers/StubHttpMessageHandler.cs b/test/RedShirt.Example.JobWorker.Connectors.Bar.Implementation.UnitTests/Tests/Helpers/StubHttpMessageHandler.cs new file mode 100644 index 00000000..6c2b9b79 --- /dev/null +++ b/test/RedShirt.Example.JobWorker.Connectors.Bar.Implementation.UnitTests/Tests/Helpers/StubHttpMessageHandler.cs @@ -0,0 +1,14 @@ +namespace RedShirt.Example.JobWorker.Connectors.Bar.Implementation.UnitTests.Tests.Helpers; + +internal sealed class StubHttpMessageHandler(Func responder) + : HttpMessageHandler +{ + public IList Requests { get; } = new List(); + + protected override Task SendAsync(HttpRequestMessage request, + CancellationToken cancellationToken) + { + Requests.Add(request); + return Task.FromResult(responder(request)); + } +} \ No newline at end of file diff --git a/test/RedShirt.Example.JobWorker.Connectors.Bar.Implementation.UnitTests/Tests/Services/BarConnectorTests.cs b/test/RedShirt.Example.JobWorker.Connectors.Bar.Implementation.UnitTests/Tests/Services/BarConnectorTests.cs new file mode 100644 index 00000000..47d4e0d0 --- /dev/null +++ b/test/RedShirt.Example.JobWorker.Connectors.Bar.Implementation.UnitTests/Tests/Services/BarConnectorTests.cs @@ -0,0 +1,152 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using RedShirt.Example.JobWorker.Common.Services.Utility; +using RedShirt.Example.JobWorker.Connectors.Bar.Core.Models; +using RedShirt.Example.JobWorker.Connectors.Bar.Implementation.Clients; +using RedShirt.Example.JobWorker.Connectors.Bar.Implementation.Exceptions; +using RedShirt.Example.JobWorker.Connectors.Bar.Implementation.Factories; +using RedShirt.Example.JobWorker.Connectors.Bar.Implementation.Services; +using RedShirt.Example.JobWorker.Connectors.Bar.Implementation.Services.Resilience; + +namespace RedShirt.Example.JobWorker.Connectors.Bar.Implementation.UnitTests.Tests.Services; + +public class BarConnectorTests +{ + private static IOptions CreateOptions(int? reasonToWaitFallbackSeconds = null) + { + return Options.Create(new BarConnector.ConfigurationModel + { + ReasonToWaitFallbackSeconds = reasonToWaitFallbackSeconds + }); + } + + private static BarConnector CreateConnector( + Mock apiClient, + IList? capturedDelays = null) + { + var factory = new Mock(MockBehavior.Strict); + factory.Setup(f => f.CreateBarApiClient()).Returns(apiClient.Object); + + var retryWrapper = new Mock(MockBehavior.Strict); + retryWrapper + .Setup(r => r.RunAsync(It.IsAny>>(), + It.IsAny())) + .Returns>, CancellationToken>((func, token) => + func(token)); + retryWrapper + .Setup(r => r.RunAsync(It.IsAny>>(), + It.IsAny())) + .Returns>, CancellationToken>((func, token) => + func(token)); + + var sleep = new Mock(MockBehavior.Strict); + sleep.Setup(s => s.DelayAsync(It.IsAny(), It.IsAny())) + .Returns((delay, _) => + { + capturedDelays?.Add(delay); + return Task.CompletedTask; + }); + + return new BarConnector( + factory.Object, + retryWrapper.Object, + sleep.Object, + NullLogger.Instance, + CreateOptions()); + } + + [Theory] + [InlineData(null, 15)] + [InlineData(0, 1)] + [InlineData(5, 5)] + public void ConfigurationModel_EffectiveReasonToWaitFallback(int? configuredSeconds, int expectedSeconds) + { + var model = new BarConnector.ConfigurationModel {ReasonToWaitFallbackSeconds = configuredSeconds}; + + Assert.Equal(TimeSpan.FromSeconds(expectedSeconds), model.EffectiveReasonToWaitFallback); + } + + [Fact] + public async Task CreateAsync_WhenRequestIsNull_ThrowsArgumentNullException() + { + var connector = CreateConnector(new Mock(MockBehavior.Strict)); + + await Assert.ThrowsAsync(() => + connector.CreateAsync(null!, TestContext.Current.CancellationToken)); + } + + [Fact] + public async Task GetByIdAsync_WhenRateLimitedThenSuccess_SleepsUsingRetryAfterAndReturnsRecord() + { + var attempts = 0; + var capturedDelays = new List(); + var apiClient = new Mock(MockBehavior.Strict); + apiClient + .Setup(c => c.GetBarByIdAsync(429, It.IsAny())) + .Returns(() => + { + if (++attempts == 1) + { + throw new BarRateLimitedException(TimeSpan.FromSeconds(2)); + } + + return Task.FromResult(new GetBarConnectorResponse {Id = 429, Name = "Bar-429"}); + }); + + var connector = CreateConnector(apiClient, capturedDelays); + + var response = await connector.GetByIdAsync(429, TestContext.Current.CancellationToken); + + Assert.Equal(429, response.Id); + Assert.Equal(2, attempts); + Assert.Equal([TimeSpan.FromSeconds(2)], capturedDelays); + } + + [Fact] + public async Task GetByIdAsync_WhenReasonToWaitHasNoRetryAfter_UsesConfiguredFallback() + { + var attempts = 0; + var capturedDelays = new List(); + var apiClient = new Mock(MockBehavior.Strict); + apiClient + .Setup(c => c.GetBarByIdAsync(1, It.IsAny())) + .Returns(() => + { + if (++attempts == 1) + { + throw new BarTemporarilyUnavailableException(); + } + + return Task.FromResult(new GetBarConnectorResponse {Id = 1, Name = "Bar-1"}); + }); + + var factory = new Mock(MockBehavior.Strict); + factory.Setup(f => f.CreateBarApiClient()).Returns(apiClient.Object); + + var retryWrapper = new Mock(MockBehavior.Strict); + retryWrapper + .Setup(r => r.RunAsync(It.IsAny>>(), + It.IsAny())) + .Returns>, CancellationToken>((func, token) => + func(token)); + + var sleep = new Mock(MockBehavior.Strict); + sleep.Setup(s => s.DelayAsync(It.IsAny(), It.IsAny())) + .Returns((delay, _) => + { + capturedDelays.Add(delay); + return Task.CompletedTask; + }); + + var connector = new BarConnector( + factory.Object, + retryWrapper.Object, + sleep.Object, + NullLogger.Instance, + Options.Create(new BarConnector.ConfigurationModel {ReasonToWaitFallbackSeconds = 10})); + + await connector.GetByIdAsync(1, TestContext.Current.CancellationToken); + + Assert.Equal([TimeSpan.FromSeconds(10)], capturedDelays); + } +} \ No newline at end of file diff --git a/test/RedShirt.Example.JobWorker.Connectors.Bar.Implementation.UnitTests/Tests/Services/Resilience/BarExceptionArbiterServiceTests.cs b/test/RedShirt.Example.JobWorker.Connectors.Bar.Implementation.UnitTests/Tests/Services/Resilience/BarExceptionArbiterServiceTests.cs new file mode 100644 index 00000000..080f2534 --- /dev/null +++ b/test/RedShirt.Example.JobWorker.Connectors.Bar.Implementation.UnitTests/Tests/Services/Resilience/BarExceptionArbiterServiceTests.cs @@ -0,0 +1,194 @@ +using RedShirt.Example.JobWorker.Common.SecretManagers.Core.Exceptions; +using RedShirt.Example.JobWorker.Connectors.Bar.Core.Exceptions; +using RedShirt.Example.JobWorker.Connectors.Bar.Implementation.Exceptions; +using RedShirt.Example.JobWorker.Connectors.Bar.Implementation.Services.Resilience; +using RedShirt.Example.JobWorker.Connectors.Common.Http.Exceptions; +using System.Net; +using System.Net.Sockets; +using System.Text.Json; + +namespace RedShirt.Example.JobWorker.Connectors.Bar.Implementation.UnitTests.Tests.Services.Resilience; + +public class BarExceptionArbiterServiceTests +{ + private readonly BarExceptionArbiterService _sut = new(); + + [Fact] + public void GetReport_BarRateLimitedException_IsExpectedButNotTransientForInnerRetry() + { + var report = _sut.GetReport(new BarRateLimitedException(TimeSpan.FromSeconds(1))); + + Assert.True(report.IsExpected); + Assert.False(report.CouldBeTransient); + Assert.True(report.CouldBeExternallySolvable); + Assert.False(report.AlreadyHandled); + } + + [Fact] + public void GetReport_BarRecordNotFoundException_IsExpectedAndNotTransient() + { + var report = _sut.GetReport(new BarRecordNotFoundException(404)); + + Assert.True(report.IsExpected); + Assert.False(report.CouldBeTransient); + Assert.False(report.CouldBeExternallySolvable); + } + + [Fact] + public void GetReport_BarUnauthorizedException_IsExpectedAndNotTransient() + { + var report = _sut.GetReport(new BarUnauthorizedException()); + + Assert.True(report.IsExpected); + Assert.False(report.CouldBeTransient); + Assert.True(report.CouldBeExternallySolvable); + } + + [Fact] + public void GetReport_HandledBarException_RespectsFlags() + { + var report = _sut.GetReport(new BarException(new InvalidOperationException("handled")) + { + IsHandled = true, + CouldBeTransient = true, + CouldBeExternallySolvable = false + }); + + Assert.True(report.AlreadyHandled); + Assert.True(report.IsExpected); + Assert.False(report.CouldBeTransient); + Assert.False(report.CouldBeExternallySolvable); + } + + [Theory] + [InlineData(HttpStatusCode.Unauthorized)] + [InlineData(HttpStatusCode.Forbidden)] + [InlineData(HttpStatusCode.NotFound)] + public void GetReport_HttpRequestException_WithClientError_IsNotTransient(HttpStatusCode statusCode) + { + var report = _sut.GetReport(new HttpRequestException("client error", null, statusCode)); + + Assert.True(report.IsExpected); + Assert.False(report.CouldBeTransient); + Assert.True(report.CouldBeExternallySolvable); + } + + [Theory] + [InlineData(HttpStatusCode.RequestTimeout)] + [InlineData(HttpStatusCode.TooManyRequests)] + [InlineData(HttpStatusCode.InternalServerError)] + [InlineData(HttpStatusCode.BadGateway)] + [InlineData(HttpStatusCode.ServiceUnavailable)] + [InlineData(HttpStatusCode.GatewayTimeout)] + public void GetReport_HttpRequestException_WithTransientStatus_IsTransient(HttpStatusCode statusCode) + { + var report = _sut.GetReport(new HttpRequestException("transient", null, statusCode)); + + Assert.True(report.IsExpected); + Assert.True(report.CouldBeTransient); + } + + [Fact] + public void GetReport_JsonException_IsNotTransient() + { + var report = _sut.GetReport(new JsonException("invalid json")); + + Assert.True(report.IsExpected); + Assert.False(report.CouldBeTransient); + } + + [Fact] + public void GetReport_OAuthRequestExceptionServerError_IsTransient() + { + var report = _sut.GetReport(new OAuthRequestException("server error") + { + StatusCode = HttpStatusCode.InternalServerError, + CredentialStorageProblem = false, + FreshCredentialCacheResult = true + }); + + Assert.True(report.IsExpected); + Assert.True(report.CouldBeTransient); + } + + [Fact] + public void GetReport_OAuthRequestExceptionUnauthorized_IsNotTransient() + { + var report = _sut.GetReport(new OAuthRequestException("unauthorized") + { + StatusCode = HttpStatusCode.Unauthorized, + CredentialStorageProblem = false, + FreshCredentialCacheResult = false + }); + + Assert.True(report.IsExpected); + Assert.False(report.CouldBeTransient); + } + + [Fact] + public void GetReport_OperationCanceledException_IsNotTransient() + { + var report = _sut.GetReport(new OperationCanceledException()); + + Assert.True(report.IsExpected); + Assert.False(report.CouldBeTransient); + } + + [Fact] + public void GetReport_SingleInnerAggregateException_UnwrapsInner() + { + var inner = new BarRecordNotFoundException(7); + var aggregate = new AggregateException(inner); + + var report = _sut.GetReport(aggregate); + + Assert.True(report.IsExpected); + Assert.False(report.CouldBeTransient); + } + + [Fact] + public void GetReport_SocketException_IsTransient() + { + var report = _sut.GetReport(new SocketException()); + + Assert.True(report.IsExpected); + Assert.True(report.CouldBeTransient); + } + + [Fact] + public void GetReport_UnhandledBarExceptionWithTransientFlag_IsTransient() + { + var report = _sut.GetReport(new BarException(new InvalidOperationException("unhandled")) + { + IsHandled = false, + CouldBeTransient = true, + CouldBeExternallySolvable = true + }); + + Assert.True(report.AlreadyHandled); + Assert.True(report.CouldBeTransient); + } + + [Fact] + public void GetReport_UnknownException_IsNotExpected() + { + var report = _sut.GetReport(new InvalidOperationException("unexpected")); + + Assert.False(report.IsExpected); + Assert.False(report.CouldBeTransient); + } + + [Fact] + public void GetReport_WorkerSecretManagerException_UsesHandledFlags() + { + var report = _sut.GetReport(new WorkerSecretManagerException("secret failure") + { + IsHandled = false, + CouldBeTransient = true, + CouldBeExternallySolvable = true + }); + + Assert.True(report.AlreadyHandled); + Assert.True(report.CouldBeTransient); + } +} \ No newline at end of file diff --git a/test/RedShirt.Example.JobWorker.Connectors.Bar.Implementation.UnitTests/Tests/Services/Resilience/BarRetryWrapperServiceTests.cs b/test/RedShirt.Example.JobWorker.Connectors.Bar.Implementation.UnitTests/Tests/Services/Resilience/BarRetryWrapperServiceTests.cs new file mode 100644 index 00000000..b9d6ce06 --- /dev/null +++ b/test/RedShirt.Example.JobWorker.Connectors.Bar.Implementation.UnitTests/Tests/Services/Resilience/BarRetryWrapperServiceTests.cs @@ -0,0 +1,167 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using RedShirt.Example.JobWorker.Common.Services.Utility; +using RedShirt.Example.JobWorker.Connectors.Bar.Core.Exceptions; +using RedShirt.Example.JobWorker.Connectors.Bar.Implementation.Exceptions; +using RedShirt.Example.JobWorker.Connectors.Bar.Implementation.Models; +using RedShirt.Example.JobWorker.Connectors.Bar.Implementation.Services.Resilience; +using System.Net; + +namespace RedShirt.Example.JobWorker.Connectors.Bar.Implementation.UnitTests.Tests.Services.Resilience; + +public class BarRetryWrapperServiceTests +{ + private static BarExceptionArbiterReport TransientReport() + { + return new BarExceptionArbiterReport + { + AlreadyHandled = false, + IsExpected = true, + CouldBeTransient = true, + CouldBeExternallySolvable = true + }; + } + + private static BarExceptionArbiterReport PermanentReport() + { + return new BarExceptionArbiterReport + { + AlreadyHandled = false, + IsExpected = true, + CouldBeTransient = false, + CouldBeExternallySolvable = false + }; + } + + private static Mock CreateSleepService(IList? capturedDelays = null) + { + var sleep = new Mock(MockBehavior.Strict); + sleep.Setup(s => s.DelayAsync(It.IsAny(), It.IsAny())) + .Returns((delay, _) => + { + capturedDelays?.Add(delay); + return Task.CompletedTask; + }); + return sleep; + } + + private static BarRetryWrapperService CreateSut( + Mock arbiter, + Mock sleep, + int retryCount = 3) + { + return new BarRetryWrapperService(arbiter.Object, NullLogger.Instance, sleep.Object, + Options.Create(new BarRetryWrapperService.ConfigurationModel {RetryCount = retryCount})); + } + + [Theory] + [InlineData(0, 0)] + [InlineData(-2, 0)] + [InlineData(5, 5)] + public void ConfigurationModel_EffectiveRetryCount(int configured, int expected) + { + var model = new BarRetryWrapperService.ConfigurationModel {RetryCount = configured}; + + Assert.Equal(expected, model.EffectiveRetryCount); + } + + [Fact] + public void ConfigurationModel_EffectiveRetryCount_WhenNull_UsesDefault() + { + var model = new BarRetryWrapperService.ConfigurationModel {RetryCount = null}; + + Assert.Equal(3, model.EffectiveRetryCount); + } + + [Fact] + public async Task RunAsync_WhenBarReasonToWaitException_PropagatesWithoutRetryOrWrapping() + { + var rateLimited = new BarRateLimitedException(TimeSpan.FromSeconds(2)); + var arbiter = new Mock(MockBehavior.Strict); + var sleep = CreateSleepService(); + var sut = CreateSut(arbiter, sleep, 3); + + var thrown = await Assert.ThrowsAsync(() => + sut.RunAsync(_ => throw rateLimited, TestContext.Current.CancellationToken)); + + Assert.Same(rateLimited, thrown); + arbiter.VerifyNoOtherCalls(); + sleep.VerifyNoOtherCalls(); + } + + [Fact] + public async Task RunAsync_WhenBarRecordNotFoundException_PropagatesWithoutWrapping() + { + var notFound = new BarRecordNotFoundException(404); + var arbiter = new Mock(MockBehavior.Strict); + arbiter.Setup(a => a.GetReport(notFound)).Returns(new BarExceptionArbiterReport + { + AlreadyHandled = false, + IsExpected = true, + CouldBeTransient = false, + CouldBeExternallySolvable = false + }); + var sleep = CreateSleepService(); + var sut = CreateSut(arbiter, sleep, 1); + + var thrown = await Assert.ThrowsAsync(() => + sut.RunAsync(_ => throw notFound, TestContext.Current.CancellationToken)); + + Assert.Same(notFound, thrown); + arbiter.Verify(a => a.GetReport(notFound), Times.Once); + } + + [Fact] + public async Task RunAsync_WhenFuncSucceeds_ReturnsResult() + { + var arbiter = new Mock(MockBehavior.Strict); + var sleep = CreateSleepService(); + var sut = CreateSut(arbiter, sleep); + + var result = await sut.RunAsync(_ => Task.FromResult(42), TestContext.Current.CancellationToken); + + Assert.Equal(42, result); + } + + [Fact] + public async Task RunAsync_WhenPermanentExpectedFailure_WrapsInBarException() + { + var inner = new HttpRequestException("forbidden", null, HttpStatusCode.Forbidden); + var arbiter = new Mock(MockBehavior.Strict); + arbiter.Setup(a => a.GetReport(inner)).Returns(PermanentReport()); + var sleep = CreateSleepService(); + var sut = CreateSut(arbiter, sleep, 1); + + var thrown = await Assert.ThrowsAsync(() => + sut.RunAsync(_ => throw inner, TestContext.Current.CancellationToken)); + + Assert.True(thrown.IsHandled); + Assert.False(thrown.CouldBeTransient); + Assert.Same(inner, thrown.InnerException); + } + + [Fact] + public async Task RunAsync_WhenTransientFailureThenSuccess_RetriesWithExponentialBackoff() + { + var attempts = 0; + var capturedDelays = new List(); + var arbiter = new Mock(MockBehavior.Strict); + arbiter.Setup(a => a.GetReport(It.IsAny())).Returns(TransientReport()); + var sleep = CreateSleepService(capturedDelays); + var sut = CreateSut(arbiter, sleep); + + var result = await sut.RunAsync(_ => + { + if (++attempts == 1) + { + throw new HttpRequestException("transient", null, HttpStatusCode.ServiceUnavailable); + } + + return Task.FromResult("ok"); + }, TestContext.Current.CancellationToken); + + Assert.Equal("ok", result); + Assert.Equal(2, attempts); + Assert.Equal([TimeSpan.FromSeconds(1)], capturedDelays); + } +} \ No newline at end of file diff --git a/test/RedShirt.Example.JobWorker.Connectors.Common.Http.UnitTests/Globals.cs b/test/RedShirt.Example.JobWorker.Connectors.Common.Http.UnitTests/Globals.cs new file mode 100644 index 00000000..43eebfc5 --- /dev/null +++ b/test/RedShirt.Example.JobWorker.Connectors.Common.Http.UnitTests/Globals.cs @@ -0,0 +1,2 @@ +global using Moq; +global using Xunit; \ No newline at end of file diff --git a/test/RedShirt.Example.JobWorker.Connectors.Common.Http.UnitTests/RedShirt.Example.JobWorker.Connectors.Common.Http.UnitTests.csproj b/test/RedShirt.Example.JobWorker.Connectors.Common.Http.UnitTests/RedShirt.Example.JobWorker.Connectors.Common.Http.UnitTests.csproj new file mode 100644 index 00000000..912a0c2b --- /dev/null +++ b/test/RedShirt.Example.JobWorker.Connectors.Common.Http.UnitTests/RedShirt.Example.JobWorker.Connectors.Common.Http.UnitTests.csproj @@ -0,0 +1,34 @@ + + + + net10.0 + enable + enable + false + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + + diff --git a/test/RedShirt.Example.JobWorker.Connectors.Common.Http.UnitTests/Tests/Helpers/StubHttpMessageHandler.cs b/test/RedShirt.Example.JobWorker.Connectors.Common.Http.UnitTests/Tests/Helpers/StubHttpMessageHandler.cs new file mode 100644 index 00000000..53aa69e7 --- /dev/null +++ b/test/RedShirt.Example.JobWorker.Connectors.Common.Http.UnitTests/Tests/Helpers/StubHttpMessageHandler.cs @@ -0,0 +1,14 @@ +namespace RedShirt.Example.JobWorker.Connectors.Common.Http.UnitTests.Tests.Helpers; + +internal sealed class StubHttpMessageHandler(Func responder) + : HttpMessageHandler +{ + public IList Requests { get; } = new List(); + + protected override Task SendAsync(HttpRequestMessage request, + CancellationToken cancellationToken) + { + Requests.Add(request); + return Task.FromResult(responder(request)); + } +} \ No newline at end of file diff --git a/test/RedShirt.Example.JobWorker.Connectors.Common.Http.UnitTests/Tests/Services/OAuthTokenCacheTests.cs b/test/RedShirt.Example.JobWorker.Connectors.Common.Http.UnitTests/Tests/Services/OAuthTokenCacheTests.cs new file mode 100644 index 00000000..b8246063 --- /dev/null +++ b/test/RedShirt.Example.JobWorker.Connectors.Common.Http.UnitTests/Tests/Services/OAuthTokenCacheTests.cs @@ -0,0 +1,135 @@ +using RedShirt.Example.JobWorker.Connectors.Common.Http.Enums; +using RedShirt.Example.JobWorker.Connectors.Common.Http.Models; +using RedShirt.Example.JobWorker.Connectors.Common.Http.Services; + +namespace RedShirt.Example.JobWorker.Connectors.Common.Http.UnitTests.Tests.Services; + +public class OAuthTokenCacheTests +{ + private static OAuthClientCredentialsRequest CreateRequest() + { + return new OAuthClientCredentialsRequest + { + TokenUrl = "https://auth.local/oauth/token", + ClientIdPath = "/client/id", + ClientSecretPath = "/client/secret", + ScopeLabel = "audience", + ScopeValue = "https://bar.local/api" + }; + } + + [Fact] + public async Task GetAsync_WhenCachedTokenExpired_RequestsFreshToken() + { + var request = CreateRequest(); + var tokenSource = new Mock(MockBehavior.Strict); + var cache = new OAuthTokenCache(tokenSource.Object); + var expiredToken = new OAuthTokenResponse + { + AccessToken = "expired-token", + ExpiresAtUtc = DateTimeOffset.UtcNow.AddSeconds(-1) + }; + var freshToken = new OAuthTokenResponse + { + AccessToken = "fresh-token", + ExpiresAtUtc = DateTimeOffset.UtcNow.AddMinutes(30) + }; + + tokenSource + .SetupSequence(s => s.GetTokenAsync(request, false, It.IsAny())) + .ReturnsAsync(expiredToken) + .ReturnsAsync(freshToken); + + await cache.GetAsync(request, false, false, + TestContext.Current.CancellationToken); + var response = await cache.GetAsync(request, false, false, + TestContext.Current.CancellationToken); + + Assert.Equal("fresh-token", response.AccessToken); + Assert.Equal(TokenCacheState.FreshToken, response.TokenCacheState); + } + + [Fact] + public async Task GetAsync_WhenCachedTokenIsValid_ReturnsCachedTokenWithoutCallingSource() + { + var request = CreateRequest(); + var tokenSource = new Mock(MockBehavior.Strict); + var cache = new OAuthTokenCache(tokenSource.Object); + var freshToken = new OAuthTokenResponse + { + AccessToken = "cached-token", + ExpiresAtUtc = DateTimeOffset.UtcNow.AddMinutes(30) + }; + + tokenSource + .Setup(s => s.GetTokenAsync(request, false, It.IsAny())) + .ReturnsAsync(freshToken); + + await cache.GetAsync(request, false, false, + TestContext.Current.CancellationToken); + var response = await cache.GetAsync(request, false, false, + TestContext.Current.CancellationToken); + + Assert.Equal("cached-token", response.AccessToken); + Assert.Equal(TokenCacheState.CachedToken, response.TokenCacheState); + tokenSource.Verify( + s => s.GetTokenAsync(request, false, It.IsAny()), + Times.Once); + } + + [Fact] + public async Task GetAsync_WhenForceFreshCredentials_PassesForceToSourceAndReportsState() + { + var request = CreateRequest(); + var tokenSource = new Mock(MockBehavior.Strict); + var cache = new OAuthTokenCache(tokenSource.Object); + var token = new OAuthTokenResponse + { + AccessToken = "forced-credentials-token", + ExpiresAtUtc = DateTimeOffset.UtcNow.AddMinutes(30) + }; + + tokenSource + .Setup(s => s.GetTokenAsync(request, true, It.IsAny())) + .ReturnsAsync(token); + + var response = await cache.GetAsync(request, false, true, + TestContext.Current.CancellationToken); + + Assert.Equal(TokenCacheState.ForcedCredentialRetrieval, response.TokenCacheState); + tokenSource.Verify( + s => s.GetTokenAsync(request, true, It.IsAny()), + Times.Once); + } + + [Fact] + public async Task GetAsync_WhenForceFreshToken_RequestsNewToken() + { + var request = CreateRequest(); + var tokenSource = new Mock(MockBehavior.Strict); + var cache = new OAuthTokenCache(tokenSource.Object); + var firstToken = new OAuthTokenResponse + { + AccessToken = "first-token", + ExpiresAtUtc = DateTimeOffset.UtcNow.AddMinutes(30) + }; + var secondToken = new OAuthTokenResponse + { + AccessToken = "second-token", + ExpiresAtUtc = DateTimeOffset.UtcNow.AddMinutes(30) + }; + + tokenSource + .SetupSequence(s => s.GetTokenAsync(request, false, It.IsAny())) + .ReturnsAsync(firstToken) + .ReturnsAsync(secondToken); + + await cache.GetAsync(request, false, false, + TestContext.Current.CancellationToken); + var response = await cache.GetAsync(request, true, false, + TestContext.Current.CancellationToken); + + Assert.Equal("second-token", response.AccessToken); + Assert.Equal(TokenCacheState.FreshToken, response.TokenCacheState); + } +} \ No newline at end of file diff --git a/test/RedShirt.Example.JobWorker.Connectors.Common.Http.UnitTests/Tests/Services/OAuthTokenSourceTests.cs b/test/RedShirt.Example.JobWorker.Connectors.Common.Http.UnitTests/Tests/Services/OAuthTokenSourceTests.cs new file mode 100644 index 00000000..d605deb9 --- /dev/null +++ b/test/RedShirt.Example.JobWorker.Connectors.Common.Http.UnitTests/Tests/Services/OAuthTokenSourceTests.cs @@ -0,0 +1,179 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using RedShirt.Example.JobWorker.Common.SecretManagers.Core.Exceptions; +using RedShirt.Example.JobWorker.Common.SecretManagers.Core.Models; +using RedShirt.Example.JobWorker.Common.SecretManagers.Core.Services; +using RedShirt.Example.JobWorker.Connectors.Common.Http.Exceptions; +using RedShirt.Example.JobWorker.Connectors.Common.Http.Models; +using RedShirt.Example.JobWorker.Connectors.Common.Http.Services; +using RedShirt.Example.JobWorker.Connectors.Common.Http.UnitTests.Tests.Helpers; +using System.Net; +using System.Text; + +namespace RedShirt.Example.JobWorker.Connectors.Common.Http.UnitTests.Tests.Services; + +public class OAuthTokenSourceTests +{ + private const string ClientIdPath = "/client/id"; + private const string ClientSecretPath = "/client/secret"; + + private static OAuthClientCredentialsRequest CreateRequest() + { + return new OAuthClientCredentialsRequest + { + TokenUrl = "https://auth.local/oauth/token", + ClientIdPath = ClientIdPath, + ClientSecretPath = ClientSecretPath, + ScopeLabel = null, + ScopeValue = null + }; + } + + private static OAuthTokenSource CreateSut( + StubHttpMessageHandler handler, + Mock secretManager, + int? fallbackJwtExpiryMinutes = null) + { + var httpClientFactory = new Mock(MockBehavior.Strict); + httpClientFactory + .Setup(f => f.CreateClient(nameof(OAuthTokenSource))) + .Returns(new HttpClient(handler)); + + return new OAuthTokenSource( + httpClientFactory.Object, + secretManager.Object, + NullLogger.Instance, + Options.Create(new OAuthTokenSource.ConfigurationModel + { + FallbackJwtExpiryTimeMinutes = fallbackJwtExpiryMinutes + })); + } + + private static void SetupSecrets(Mock secretManager, bool queriedSecretManager = false) + { + secretManager + .Setup(s => s.GetSecretsAsync( + It.Is>(paths => paths.Contains(ClientIdPath) && paths.Contains(ClientSecretPath)), + null, + It.IsAny(), + It.IsAny())) + .ReturnsAsync(new SecretManagerCacheSecretsResponse + { + Values = new Dictionary + { + [ClientIdPath] = "client-id", + [ClientSecretPath] = "client-secret" + }, + QueriedSecretManager = queriedSecretManager + }); + } + + [Theory] + [InlineData(0, 1)] + [InlineData(-5, 1)] + [InlineData(20, 20)] + public void ConfigurationModel_EffectiveFallbackJwtExpiryTimeMinutes(int configured, int expectedMinutes) + { + var model = new OAuthTokenSource.ConfigurationModel {FallbackJwtExpiryTimeMinutes = configured}; + + Assert.Equal(TimeSpan.FromMinutes(expectedMinutes), model.EffectiveFallbackJwtExpiryTimeMinutes); + } + + [Fact] + public async Task GetTokenAsync_WhenExpiresInMissing_UsesConfiguredFallbackLifetime() + { + var handler = new StubHttpMessageHandler(_ => new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent("{\"access_token\":\"access-token\"}", Encoding.UTF8, "application/json") + }); + var secretManager = new Mock(MockBehavior.Strict); + SetupSecrets(secretManager); + var sut = CreateSut(handler, secretManager, 10); + var before = DateTimeOffset.UtcNow; + + var response = + await sut.GetTokenAsync(CreateRequest(), cancellationToken: TestContext.Current.CancellationToken); + + Assert.InRange(response.ExpiresAtUtc, before.AddMinutes(9), before.AddMinutes(11)); + } + + [Fact] + public async Task GetTokenAsync_WhenResponseIsInvalidJson_ThrowsOAuthRequestJsonException() + { + var handler = new StubHttpMessageHandler(_ => new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent("not-json", Encoding.UTF8, "application/json") + }); + var secretManager = new Mock(MockBehavior.Strict); + SetupSecrets(secretManager); + var sut = CreateSut(handler, secretManager); + + await Assert.ThrowsAsync(() => + sut.GetTokenAsync(CreateRequest(), cancellationToken: TestContext.Current.CancellationToken)); + } + + [Fact] + public async Task GetTokenAsync_WhenSecretManagerFails_ThrowsOAuthRequestExceptionWithCredentialStorageProblem() + { + var handler = new StubHttpMessageHandler(_ => throw new InvalidOperationException("should not call http")); + var secretManager = new Mock(MockBehavior.Strict); + secretManager + .Setup(s => s.GetSecretsAsync( + It.IsAny>(), + null, + It.IsAny(), + It.IsAny())) + .ThrowsAsync(new WorkerSecretManagerException("secret store unavailable") + { + IsHandled = false, + CouldBeTransient = true, + CouldBeExternallySolvable = true + }); + var sut = CreateSut(handler, secretManager); + + var thrown = await Assert.ThrowsAsync(() => + sut.GetTokenAsync(CreateRequest(), cancellationToken: TestContext.Current.CancellationToken)); + + Assert.True(thrown.CredentialStorageProblem); + Assert.Null(thrown.StatusCode); + Assert.True(thrown.FreshCredentialCacheResult); + } + + [Fact] + public async Task GetTokenAsync_WhenTokenEndpointReturnsNonSuccess_ThrowsOAuthRequestException() + { + var handler = new StubHttpMessageHandler(_ => new HttpResponseMessage(HttpStatusCode.Unauthorized)); + var secretManager = new Mock(MockBehavior.Strict); + SetupSecrets(secretManager, true); + var sut = CreateSut(handler, secretManager); + + var thrown = await Assert.ThrowsAsync(() => + sut.GetTokenAsync(CreateRequest(), cancellationToken: TestContext.Current.CancellationToken)); + + Assert.Equal(HttpStatusCode.Unauthorized, thrown.StatusCode); + Assert.False(thrown.CredentialStorageProblem); + Assert.True(thrown.FreshCredentialCacheResult); + } + + [Fact] + public async Task GetTokenAsync_WhenTokenEndpointReturnsSuccess_UsesExpiresIn() + { + var handler = new StubHttpMessageHandler(_ => new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent( + "{\"access_token\":\"access-token\",\"expires_in\":120}", + Encoding.UTF8, + "application/json") + }); + var secretManager = new Mock(MockBehavior.Strict); + SetupSecrets(secretManager); + var sut = CreateSut(handler, secretManager); + var before = DateTimeOffset.UtcNow; + + var response = + await sut.GetTokenAsync(CreateRequest(), cancellationToken: TestContext.Current.CancellationToken); + + Assert.Equal("access-token", response.AccessToken); + Assert.InRange(response.ExpiresAtUtc, before.AddSeconds(119), before.AddSeconds(121)); + } +} \ No newline at end of file From 5a6069bc0857a01a0e59f24bd8aab0a812858677 Mon Sep 17 00:00:00 2001 From: Alan Deutscher Date: Thu, 3 Sep 2026 10:56:36 -0700 Subject: [PATCH 15/18] Rephrase BarConnector to make individual call methods simpler. --- .../Services/BarConnector.cs | 31 ++++++++++++------- 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Services/BarConnector.cs b/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Services/BarConnector.cs index 056c2fb7..56db4d0d 100644 --- a/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Services/BarConnector.cs +++ b/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Services/BarConnector.cs @@ -83,27 +83,34 @@ private Task ExecuteRespectingReasonToWaitAsync( cancellationToken).AsTask(); } + private Task ExecuteWithResilienceAsync( + Func> operation, + CancellationToken cancellationToken) + { + return ExecuteRespectingReasonToWaitAsync( + token => retryWrapperService.RunAsync(operation, token), + cancellationToken); + } + public Task CreateAsync(CreateBarConnectorRequest request, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(request); - return ExecuteRespectingReasonToWaitAsync(token => - retryWrapperService.RunAsync(async innerToken => - { - var client = barApiClientFactory.CreateBarApiClient(); - return await client.CreateBarAsync(request, innerToken); - }, token), cancellationToken); + return ExecuteWithResilienceAsync(async innerToken => + { + var client = barApiClientFactory.CreateBarApiClient(); + return await client.CreateBarAsync(request, innerToken); + }, cancellationToken); } public Task GetByIdAsync(int id, CancellationToken cancellationToken = default) { - return ExecuteRespectingReasonToWaitAsync(token => - retryWrapperService.RunAsync(innerToken => - { - var client = barApiClientFactory.CreateBarApiClient(); - return client.GetBarByIdAsync(id, innerToken); - }, token), cancellationToken); + return ExecuteWithResilienceAsync(innerToken => + { + var client = barApiClientFactory.CreateBarApiClient(); + return client.GetBarByIdAsync(id, innerToken); + }, cancellationToken); } internal sealed class ConfigurationModel From 175c93cbf24dc8961eaef6ff01727001db3199ff Mon Sep 17 00:00:00 2001 From: Alan Deutscher Date: Thu, 3 Sep 2026 10:59:47 -0700 Subject: [PATCH 16/18] comment nudge --- .../Services/BarConnector.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Services/BarConnector.cs b/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Services/BarConnector.cs index 56db4d0d..71ee9cc4 100644 --- a/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Services/BarConnector.cs +++ b/src/RedShirt.Example.JobWorker.Connectors.Bar.Implementation/Services/BarConnector.cs @@ -61,8 +61,8 @@ private ResiliencePipeline GetReasonToWaitPipeline() var baseDelay = reasonToWait.RetryAfter ?? options.Value.EffectiveReasonToWaitFallback; // Polly v8 AttemptNumber is zero-based (0 on the first retry). Add linear slack on top of - // RetryAfter/fallback because the API may need a little extra time to recognize that the - // rate-limiting window has passed (0s, then 1s, then 2s, etc). + // RetryAfter/fallback, assuming that the API may need a little extra time to recognize that + // the rate-limiting window has passed (0s, then 1s, then 2s, etc). var attemptBuffer = TimeSpan.FromSeconds(args.AttemptNumber); var delay = baseDelay + attemptBuffer; logger.LogWarning(reasonToWait, From 6c708759f3a1fed7bf9367f2472a382404ad5d8e Mon Sep 17 00:00:00 2001 From: Alan Deutscher Date: Thu, 3 Sep 2026 11:03:26 -0700 Subject: [PATCH 17/18] doc phrasing --- docs/bar-connector.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/bar-connector.md b/docs/bar-connector.md index 711f18e9..694aaee2 100644 --- a/docs/bar-connector.md +++ b/docs/bar-connector.md @@ -27,7 +27,7 @@ duty is to keep trying respectfully when rate limited or otherwise told to wait. ## Last-mile instructions -These steps cannot be fully prescribed by a general template. Adapt them for your target API. +These steps cannot be fully performed in a general template. Adapt them for your target API. ### If you are using the [API Template](https://github.com/adeutscher/RedShirt.Example.Api) From 608c158753a19eefcb5f586c45d2ce579c5d5126 Mon Sep 17 00:00:00 2001 From: Alan Deutscher Date: Thu, 3 Sep 2026 11:24:02 -0700 Subject: [PATCH 18/18] revise documentation --- README.md | 2 ++ docs/bar-connector.md | 25 ++++++++++++------------- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index e2fd715f..af5a2fd4 100644 --- a/README.md +++ b/README.md @@ -422,6 +422,8 @@ Below are the recommended steps for using this as a template: * The dependency injection setup in the root project assumes that the general template will be pruned down. * The dependency injection setup in the root project assumes that the chosen Secret Manager is SSM unless the chosen job source is explicitly Azure-based (see below for more details). +6. Consider revising/pruning the Markdown files such as this README or those located in the `docs/` directory. They + assume that they are speaking for a general template and not for an applied application. ## Cached Idempotency vs Database diff --git a/docs/bar-connector.md b/docs/bar-connector.md index 694aaee2..6710d0fe 100644 --- a/docs/bar-connector.md +++ b/docs/bar-connector.md @@ -37,25 +37,24 @@ These steps cannot be fully performed in a general template. Adapt them for your * [Sonatype Nexus Repository](https://help.sonatype.com/en/nuget-repositories.html) 2. Rename `Bar.Core` and `Bar.Implementation` projects as appropriate for your target API. 3. Reference the interop NuGet package in your renamed implementation project. -4. Write wrapper clients for the relevant clients from the interop package that translate thrown `SwaggerException` - instances to a generic `BarException` (renamed for your API): - * `BarException` translation should judge the status code in the `SwaggerException` to set `CouldBeTransient`. It is - recommended to abstract this decision into an exception arbiter service with a method called - `CouldSwaggerExceptionBeTransient`. - * The exception to this translation is HTTP **429** (Too Many Requests), which should specifically become a +4. Write wrapper clients for the relevant clients from the interop package that inspect thrown `SwaggerException` + instances: + * `SwaggerException` instances with an HTTP **429** status code (Too Many Requests) should specifically become a `BarRateLimitedException` with the value of the `Retry-After` header. + * Confirm that the exception arbiter (`BarExceptionArbiterService`) treats other status code values appropriately. 5. Adjust your client factory to return the wrapper client. 6. Rename classes whose names begin with `Bar` as appropriate for your target API. -### If you are not using the [API Template](https://github.com/adeutscher/RedShirt.Example.Api) +### If you are not using the [API Template](https://github.com/adeutscher/RedShirt.Example.Api) or other OpenAPI package + +If you are not using the API template or another flavour of OpenAPI/Swagger-generated package, then the existing Bar +connector example might already be closer to your needs: 1. Rename `Bar.Core` and `Bar.Implementation` projects as appropriate for your target API. -2. Adjust HTTP clients for the subject API so non-successful status codes translate to a generic `BarException`: - * Judge the status code to set `CouldBeTransient`. It is recommended to abstract this into an exception arbiter - service with a method called `CouldSwaggerExceptionBeTransient` (or an equivalent name if you are not using - Swagger/OpenAPI clients). - * HTTP **429** should specifically become a `BarRateLimitedException` with the value of the `Retry-After` header. -3. Rename classes whose names begin with `Bar` as appropriate for your target API. +2. Confirm that the exception arbiter treats the appropriate return codes appropriately. +3. In the client response handler, confirm that an HTTP **429** should specifically become a `BarRateLimitedException` + with the value of the `Retry-After` header. +4. Rename classes whose names begin with `Bar` as appropriate for your target API. ## Local testing