From e5c161e95da0daa6a401f4c96e6c530a88fc2963 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 18:19:48 -0700 Subject: [PATCH 01/47] feat(cqrs): compile #[CachedQuery] and #[InvalidatesQuery] into the handler manifest and derive cache keys from query value objects --- .../cli/src/Cache/ManifestCacheWriter.php | 4 +- packages/cqrs/src/Attributes/CachedQuery.php | 25 +++++ .../cqrs/src/Attributes/InvalidatesQuery.php | 36 ++++++ packages/cqrs/src/Cache/CachedQueryPolicy.php | 36 ++++++ packages/cqrs/src/Cache/InvalidationRule.php | 53 +++++++++ packages/cqrs/src/Cache/QueryCacheKey.php | 106 ++++++++++++++++++ .../cqrs/src/Cache/UnkeyableQueryValue.php | 16 +++ packages/cqrs/src/CqrsWiringProvider.php | 2 +- packages/cqrs/src/Handler/HandlerManifest.php | 50 ++++++++- .../src/Handler/HandlerManifestCompiler.php | 27 +++-- packages/cqrs/src/Scanner/HandlerScanner.php | 32 +++++- .../cqrs/tests/Cache/QueryCacheKeyTest.php | 67 +++++++++++ .../AmbiguousRuleEvent.php | 14 +++ .../CacheScannerBadFixtures/ZeroTtlQuery.php | 10 ++ .../CacheScannerFixtures/EntryPosted.php | 18 +++ .../CacheScannerFixtures/LedgerBalance.php | 14 +++ .../CacheScannerFixtures/LedgerTotals.php | 11 ++ .../tests/HandlerManifestFreshnessTest.php | 4 +- .../tests/Scanner/CacheAttributeScanTest.php | 81 +++++++++++++ .../cqrs/tests/Scanner/HandlerScannerTest.php | 4 +- 20 files changed, 593 insertions(+), 17 deletions(-) create mode 100644 packages/cqrs/src/Attributes/CachedQuery.php create mode 100644 packages/cqrs/src/Attributes/InvalidatesQuery.php create mode 100644 packages/cqrs/src/Cache/CachedQueryPolicy.php create mode 100644 packages/cqrs/src/Cache/InvalidationRule.php create mode 100644 packages/cqrs/src/Cache/QueryCacheKey.php create mode 100644 packages/cqrs/src/Cache/UnkeyableQueryValue.php create mode 100644 packages/cqrs/tests/Cache/QueryCacheKeyTest.php create mode 100644 packages/cqrs/tests/CacheScannerBadFixtures/AmbiguousRuleEvent.php create mode 100644 packages/cqrs/tests/CacheScannerBadFixtures/ZeroTtlQuery.php create mode 100644 packages/cqrs/tests/CacheScannerFixtures/EntryPosted.php create mode 100644 packages/cqrs/tests/CacheScannerFixtures/LedgerBalance.php create mode 100644 packages/cqrs/tests/CacheScannerFixtures/LedgerTotals.php create mode 100644 packages/cqrs/tests/Scanner/CacheAttributeScanTest.php diff --git a/packages/cli/src/Cache/ManifestCacheWriter.php b/packages/cli/src/Cache/ManifestCacheWriter.php index 99d5c6fd..5f68a038 100644 --- a/packages/cli/src/Cache/ManifestCacheWriter.php +++ b/packages/cli/src/Cache/ManifestCacheWriter.php @@ -161,12 +161,14 @@ public function writeManifests(array $psr4, string $dir): CacheReport $files[] = $dir.'/'.FireflyCachePaths::CONSTRAINTS, ); - // cqrs — 3-arg write (handlers + destinations + path) (SPECIAL). + // cqrs — handlers + destinations + path, then the query-cache policies and invalidation rules (SPECIAL). $handlers = (new HandlerScanner)->scan($psr4); (new HandlerManifestCompiler)->write( $handlers['handlers'], $handlers['destinations'], $files[] = $dir.'/'.FireflyCachePaths::HANDLERS, + $handlers['cachedQueries'], + $handlers['invalidations'], ); // eda diff --git a/packages/cqrs/src/Attributes/CachedQuery.php b/packages/cqrs/src/Attributes/CachedQuery.php new file mode 100644 index 00000000..b928ad80 --- /dev/null +++ b/packages/cqrs/src/Attributes/CachedQuery.php @@ -0,0 +1,25 @@ + 'eventProperty'] — evict the ONE entry whose derived key the query would get with + * those property values (the query's every public property must be mapped, with the same types); + * * $key: 'statement:{account}' — evict the entry a Cacheable query's own cacheKey() returns, `{name}` resolved + * from the event's public properties or from the EDA payload (dotted paths reach into nested arrays). + * + * An event published over EDA is matched by $eventType — default: the short class name, which is what + * DomainEvent::eventType() returns — so another process's envelope evicts this process's cache too. A placeholder + * the event cannot fill evicts the whole group instead: serving stale data is the failure worth preventing. + * PyFly's @cache_evict, moved to the event because the event is what knows what it makes stale. + */ +#[Attribute(Attribute::TARGET_CLASS | Attribute::IS_REPEATABLE)] +final class InvalidatesQuery +{ + /** + * @param class-string $query + * @param array $with query property => event property + */ + public function __construct( + public readonly string $query, + public readonly ?string $key = null, + public readonly array $with = [], + public readonly ?string $eventType = null, + ) {} +} diff --git a/packages/cqrs/src/Cache/CachedQueryPolicy.php b/packages/cqrs/src/Cache/CachedQueryPolicy.php new file mode 100644 index 00000000..2a2daa1a --- /dev/null +++ b/packages/cqrs/src/Cache/CachedQueryPolicy.php @@ -0,0 +1,36 @@ + $this->queryClass, 'ttl' => $this->ttl, 'group' => $this->group]; + } + + /** + * @param CachedQueryRow $row + */ + public static function fromArray(array $row): self + { + return new self($row['queryClass'], $row['ttl'], $row['group']); + } +} diff --git a/packages/cqrs/src/Cache/InvalidationRule.php b/packages/cqrs/src/Cache/InvalidationRule.php new file mode 100644 index 00000000..a8470c50 --- /dev/null +++ b/packages/cqrs/src/Cache/InvalidationRule.php @@ -0,0 +1,53 @@ +} + */ +final readonly class InvalidationRule +{ + /** + * @param array $with + */ + public function __construct( + public string $eventClass, + public string $eventType, + public string $queryClass, + public ?string $key = null, + public array $with = [], + ) {} + + public function evictsGroup(): bool + { + return $this->key === null && $this->with === []; + } + + /** + * @return InvalidationRow + */ + public function toArray(): array + { + return [ + 'eventClass' => $this->eventClass, + 'eventType' => $this->eventType, + 'queryClass' => $this->queryClass, + 'key' => $this->key, + 'with' => $this->with, + ]; + } + + /** + * @param InvalidationRow $row + */ + public static function fromArray(array $row): self + { + return new self($row['eventClass'], $row['eventType'], $row['queryClass'], $row['key'], $row['with']); + } +} diff --git a/packages/cqrs/src/Cache/QueryCacheKey.php b/packages/cqrs/src/Cache/QueryCacheKey.php new file mode 100644 index 00000000..3e8e87ac --- /dev/null +++ b/packages/cqrs/src/Cache/QueryCacheKey.php @@ -0,0 +1,106 @@ + $properties + */ + public static function fromProperties(string $queryClass, array $properties): ?string + { + try { + $canonical = json_encode( + [$queryClass, self::normalizeArray($properties, 0)], + JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE | JSON_PRESERVE_ZERO_FRACTION, + ); + } catch (UnkeyableQueryValue|JsonException) { + return null; + } + + return hash('sha256', $canonical); + } + + /** + * @return array + */ + private static function publicProperties(object $object): array + { + $properties = []; + foreach (get_object_vars($object) as $name => $value) { + $properties[(string) $name] = $value; + } + + return $properties; + } + + private static function normalize(mixed $value, int $depth): mixed + { + if ($depth > self::MAX_DEPTH) { + throw new UnkeyableQueryValue; + } + + return match (true) { + $value === null, is_bool($value), is_int($value), is_string($value) => $value, + is_float($value) => is_finite($value) ? $value : throw new UnkeyableQueryValue, + is_array($value) => self::normalizeArray($value, $depth + 1), + $value instanceof BackedEnum => ['enum' => $value::class, 'value' => $value->value], + $value instanceof UnitEnum => ['enum' => $value::class, 'name' => $value->name], + $value instanceof DateTimeInterface => ['instant' => $value->format('Y-m-d\TH:i:s.uP')], + $value instanceof Closure => throw new UnkeyableQueryValue, + $value instanceof JsonSerializable => ['json' => self::normalize($value->jsonSerialize(), $depth + 1)], + $value instanceof Stringable => ['string' => (string) $value], + is_object($value) => ['object' => $value::class, 'properties' => self::normalizeArray(self::publicProperties($value), $depth + 1)], + default => throw new UnkeyableQueryValue, + }; + } + + /** + * @param array $values + * @return array + */ + private static function normalizeArray(array $values, int $depth): array + { + if (! array_is_list($values)) { + ksort($values, SORT_STRING); + } + + $normalized = []; + foreach ($values as $key => $value) { + $normalized[$key] = self::normalize($value, $depth); + } + + return $normalized; + } +} diff --git a/packages/cqrs/src/Cache/UnkeyableQueryValue.php b/packages/cqrs/src/Cache/UnkeyableQueryValue.php new file mode 100644 index 00000000..97b8b8a7 --- /dev/null +++ b/packages/cqrs/src/Cache/UnkeyableQueryValue.php @@ -0,0 +1,16 @@ +scan($paths); - return new HandlerManifest($scanned['handlers'], $scanned['destinations']); + return new HandlerManifest($scanned['handlers'], $scanned['destinations'], $scanned['cachedQueries'], $scanned['invalidations']); }); } diff --git a/packages/cqrs/src/Handler/HandlerManifest.php b/packages/cqrs/src/Handler/HandlerManifest.php index bf271220..dfabada1 100644 --- a/packages/cqrs/src/Handler/HandlerManifest.php +++ b/packages/cqrs/src/Handler/HandlerManifest.php @@ -4,29 +4,39 @@ namespace Firefly\Cqrs\Handler; +use Firefly\Cqrs\Cache\CachedQueryPolicy; +use Firefly\Cqrs\Cache\InvalidationRule; use Firefly\Kernel\Exception\Framework\ConfigurationException; /** * The compiled, reflection-free handler source the CqrsHandlerWiringPass reads at boot (handler rows) and the * commandEventPublisher bean reads for routing (the #[PublishDomainEvent] destinations map). Loaded via require+map; * both halves are pure arrays so the whole manifest is a plain PHP array literal with two top-level keys - * (`handlers`, `destinations`). Mirrors the ScheduledManifest / EventListenerManifest idiom. + * (`handlers`, `destinations`), plus the two query-cache sections (`cachedQueries` — #[CachedQuery] policies keyed by + * query class — and `invalidations` — #[InvalidatesQuery] rules in scan order), both optional on load so a manifest + * compiled before they existed still loads. Mirrors the ScheduledManifest / EventListenerManifest idiom. * * @phpstan-import-type HandlerRow from HandlerDescriptor + * @phpstan-import-type CachedQueryRow from CachedQueryPolicy + * @phpstan-import-type InvalidationRow from InvalidationRule */ final class HandlerManifest { /** * @param list $handlers * @param array $destinations + * @param array $cachedQueries query class => policy (#[CachedQuery]) + * @param list $invalidations #[InvalidatesQuery] rules, in scan order */ public function __construct( private readonly array $handlers, private readonly array $destinations, + private readonly array $cachedQueries = [], + private readonly array $invalidations = [], ) {} /** - * @param array{handlers?: array, destinations?: array} $data + * @param array{handlers?: array, destinations?: array, cachedQueries?: array, invalidations?: array} $data */ public static function fromArray(array $data): self { @@ -35,7 +45,18 @@ public static function fromArray(array $data): self array_values($data['handlers'] ?? []), ); - return new self($handlers, $data['destinations'] ?? []); + $cachedQueries = []; + foreach ($data['cachedQueries'] ?? [] as $queryClass => $row) { + $cachedQueries[(string) $queryClass] = CachedQueryPolicy::fromArray($row); + } + + $invalidations = array_map( + static fn (array $row): InvalidationRule => InvalidationRule::fromArray($row), + array_values($data['invalidations'] ?? []), + ); + + // A handlers.php compiled before the cache sections existed has neither key: both default to empty. + return new self($handlers, $data['destinations'] ?? [], $cachedQueries, $invalidations); } public static function load(string $path): self @@ -50,7 +71,7 @@ public static function load(string $path): self throw new ConfigurationException("Handler manifest at {$path} did not return an array."); } - /** @var array{handlers?: array, destinations?: array} $data */ + /** @var array{handlers?: array, destinations?: array, cachedQueries?: array, invalidations?: array} $data */ return self::fromArray($data); } @@ -69,4 +90,25 @@ public function destinations(): array { return $this->destinations; } + + /** + * @return array + */ + public function cachedQueries(): array + { + return $this->cachedQueries; + } + + /** + * @return list + */ + public function invalidations(): array + { + return $this->invalidations; + } + + public function cachePolicyFor(string $queryClass): ?CachedQueryPolicy + { + return $this->cachedQueries[$queryClass] ?? null; + } } diff --git a/packages/cqrs/src/Handler/HandlerManifestCompiler.php b/packages/cqrs/src/Handler/HandlerManifestCompiler.php index ca28efe4..578f6764 100644 --- a/packages/cqrs/src/Handler/HandlerManifestCompiler.php +++ b/packages/cqrs/src/Handler/HandlerManifestCompiler.php @@ -4,52 +4,65 @@ namespace Firefly\Cqrs\Handler; +use Firefly\Cqrs\Cache\CachedQueryPolicy; +use Firefly\Cqrs\Cache\InvalidationRule; use Firefly\Kernel\Exception\Framework\ConfigurationException; /** * Mirrors the ScheduledManifestCompiler: var_export of the handler rows + the destinations map into a require-able - * PHP file with two top-level keys. Run Pint after any regen (the compiler emits raw var_export style). + * PHP file with two top-level keys. Run Pint after any regen (the compiler emits raw var_export style). The cache + * sections are trailing optional arguments so the 3-argument write() every existing caller uses keeps compiling. * * @phpstan-import-type HandlerRow from HandlerDescriptor + * @phpstan-import-type CachedQueryRow from CachedQueryPolicy + * @phpstan-import-type InvalidationRow from InvalidationRule */ final class HandlerManifestCompiler { /** * @param list $handlers * @param array $destinations + * @param array $cachedQueries + * @param list $invalidations */ - public function compile(array $handlers, array $destinations): string + public function compile(array $handlers, array $destinations, array $cachedQueries = [], array $invalidations = []): string { return "toArray($handlers, $destinations), true) + .var_export($this->toArray($handlers, $destinations, $cachedQueries, $invalidations), true) .";\n"; } /** * @param list $handlers * @param array $destinations - * @return array{handlers: list, destinations: array} + * @param array $cachedQueries + * @param list $invalidations + * @return array{handlers: list, destinations: array, cachedQueries: array, invalidations: list} */ - public function toArray(array $handlers, array $destinations): array + public function toArray(array $handlers, array $destinations, array $cachedQueries = [], array $invalidations = []): array { return [ 'handlers' => array_map(static fn (HandlerDescriptor $h): array => $h->toArray(), $handlers), 'destinations' => $destinations, + 'cachedQueries' => array_map(static fn (CachedQueryPolicy $p): array => $p->toArray(), $cachedQueries), + 'invalidations' => array_map(static fn (InvalidationRule $r): array => $r->toArray(), $invalidations), ]; } /** * @param list $handlers * @param array $destinations + * @param array $cachedQueries + * @param list $invalidations */ - public function write(array $handlers, array $destinations, string $path): void + public function write(array $handlers, array $destinations, string $path, array $cachedQueries = [], array $invalidations = []): void { $dir = dirname($path); if (! is_dir($dir) && ! mkdir($dir, 0o775, true) && ! is_dir($dir)) { throw new ConfigurationException("Could not create manifest directory {$dir}."); } - if (file_put_contents($path, $this->compile($handlers, $destinations)) === false) { + if (file_put_contents($path, $this->compile($handlers, $destinations, $cachedQueries, $invalidations)) === false) { throw new ConfigurationException("Could not write handler manifest to {$path}."); } } diff --git a/packages/cqrs/src/Scanner/HandlerScanner.php b/packages/cqrs/src/Scanner/HandlerScanner.php index 2ae6fc18..02d544a2 100644 --- a/packages/cqrs/src/Scanner/HandlerScanner.php +++ b/packages/cqrs/src/Scanner/HandlerScanner.php @@ -4,9 +4,13 @@ namespace Firefly\Cqrs\Scanner; +use Firefly\Cqrs\Attributes\CachedQuery; use Firefly\Cqrs\Attributes\CommandHandler; +use Firefly\Cqrs\Attributes\InvalidatesQuery; use Firefly\Cqrs\Attributes\PublishDomainEvent; use Firefly\Cqrs\Attributes\QueryHandler; +use Firefly\Cqrs\Cache\CachedQueryPolicy; +use Firefly\Cqrs\Cache\InvalidationRule; use Firefly\Cqrs\Exception\CqrsConfigurationException; use Firefly\Cqrs\Handler\HandlerDescriptor; use Firefly\Cqrs\Handler\HandlerKind; @@ -20,18 +24,23 @@ * arg else inferred from the sole handle() parameter, mirroring #[AsEventListener]'s param inference); and collects * #[PublishDomainEvent] off DomainEvent subclasses into a {eventClass => destination} map. Fails loud * (CqrsConfigurationException) on an unresolvable type or a missing/multi-param handle() — never silently skips. + * It also compiles #[CachedQuery] off query classes into per-query policies and #[InvalidatesQuery] off event classes + * into invalidation rules, refusing a rule with both a key template and a `with` map, a rule naming a class that does + * not exist, and a TTL below one second. * Runs only at cache time; production loads the compiled HandlerManifest via require+map. Mirrors ScheduledScanner. */ final class HandlerScanner { /** * @param array $psr4 namespace-prefix => absolute directory - * @return array{handlers: list, destinations: array} + * @return array{handlers: list, destinations: array, cachedQueries: array, invalidations: list} */ public function scan(array $psr4): array { $handlers = []; $destinations = []; + $cachedQueries = []; + $invalidations = []; foreach ($this->classes($psr4) as $class) { $reflection = new ReflectionClass($class); @@ -63,9 +72,28 @@ public function scan(array $psr4): array $destinations[$class] = $destination; } } + + foreach ($reflection->getAttributes(CachedQuery::class) as $attribute) { + $cached = $attribute->newInstance(); + if ($cached->ttl !== null && $cached->ttl < 1) { + throw new CqrsConfigurationException("#[CachedQuery] on [{$class}] declares ttl {$cached->ttl}; a TTL is at least one second (omit it to use firefly.cqrs.query.cache_ttl)."); + } + $cachedQueries[$class] = new CachedQueryPolicy($class, $cached->ttl, $cached->group ?? $class); + } + + foreach ($reflection->getAttributes(InvalidatesQuery::class) as $attribute) { + $rule = $attribute->newInstance(); + if (! class_exists($rule->query)) { + throw new CqrsConfigurationException("#[InvalidatesQuery] on [{$class}] names [{$rule->query}], which is not a class."); + } + if ($rule->key !== null && $rule->with !== []) { + throw new CqrsConfigurationException("#[InvalidatesQuery({$rule->query}::class)] on [{$class}] declares both key and with; a rule evicts by one or the other."); + } + $invalidations[] = new InvalidationRule($class, $rule->eventType ?? $reflection->getShortName(), $rule->query, $rule->key, $rule->with); + } } - return ['handlers' => $handlers, 'destinations' => $destinations]; + return ['handlers' => $handlers, 'destinations' => $destinations, 'cachedQueries' => $cachedQueries, 'invalidations' => $invalidations]; } /** diff --git a/packages/cqrs/tests/Cache/QueryCacheKeyTest.php b/packages/cqrs/tests/Cache/QueryCacheKeyTest.php new file mode 100644 index 00000000..50deb399 --- /dev/null +++ b/packages/cqrs/tests/Cache/QueryCacheKeyTest.php @@ -0,0 +1,67 @@ +toBe(QueryCacheKey::of(new LedgerBalance('a'))) + ->and(QueryCacheKey::of(new LedgerBalance('a')))->not->toBe(QueryCacheKey::of(new LedgerBalance('b'))) + ->and(QueryCacheKey::of(new LedgerBalance('a')))->toMatch('/^[0-9a-f]{64}$/'); +}); + +it('keys by the query class as well as its properties', function () { + $other = new class('a') + { + public function __construct(public string $account) {} + }; + + expect(QueryCacheKey::of(new LedgerBalance('a')))->not->toBe(QueryCacheKey::of($other)) + ->and(QueryCacheKey::of(new LedgerTotals))->toBe(QueryCacheKey::fromProperties(LedgerTotals::class, [])); +}); + +it('builds the key a value object would get from a property map, independent of key order', function () { + $query = new class('a', 3) + { + public function __construct(public string $account, public int $page) {} + }; + + expect(QueryCacheKey::fromProperties($query::class, ['page' => 3, 'account' => 'a']))->toBe(QueryCacheKey::of($query)) + ->and(QueryCacheKey::fromProperties($query::class, ['page' => '3', 'account' => 'a']))->not->toBe(QueryCacheKey::of($query)); +}); + +it('normalises enums, dates, nested objects and lists', function () { + $at = new DateTimeImmutable('2026-10-12T10:00:00+02:00'); + $a = new class(W4KeySide::Debit, $at, ['x', 'y'], new LedgerBalance('a')) + { + /** @param list $tags */ + public function __construct(public W4KeySide $side, public DateTimeImmutable $at, public array $tags, public LedgerBalance $nested) {} + }; + $b = new ($a::class)(W4KeySide::Debit, new DateTimeImmutable('2026-10-12T10:00:00+02:00'), ['x', 'y'], new LedgerBalance('a')); + $c = new ($a::class)(W4KeySide::Debit, $at, ['y', 'x'], new LedgerBalance('a')); + + expect(QueryCacheKey::of($a))->not->toBeNull() + ->and(QueryCacheKey::of($a))->toBe(QueryCacheKey::of($b)) + ->and(QueryCacheKey::of($a))->not->toBe(QueryCacheKey::of($c)); +}); + +it('refuses to key what cannot be part of a stable key', function () { + $closure = new class(static fn (): int => 1) + { + public function __construct(public Closure $callback) {} + }; + $nan = new class(NAN) + { + public function __construct(public float $value) {} + }; + + expect(QueryCacheKey::of($closure))->toBeNull() + ->and(QueryCacheKey::of($nan))->toBeNull(); +}); diff --git a/packages/cqrs/tests/CacheScannerBadFixtures/AmbiguousRuleEvent.php b/packages/cqrs/tests/CacheScannerBadFixtures/AmbiguousRuleEvent.php new file mode 100644 index 00000000..164fdc0a --- /dev/null +++ b/packages/cqrs/tests/CacheScannerBadFixtures/AmbiguousRuleEvent.php @@ -0,0 +1,14 @@ + 'account'])] +final class AmbiguousRuleEvent +{ + public function __construct(public string $account = 'a') {} +} diff --git a/packages/cqrs/tests/CacheScannerBadFixtures/ZeroTtlQuery.php b/packages/cqrs/tests/CacheScannerBadFixtures/ZeroTtlQuery.php new file mode 100644 index 00000000..e436d9c7 --- /dev/null +++ b/packages/cqrs/tests/CacheScannerBadFixtures/ZeroTtlQuery.php @@ -0,0 +1,10 @@ + 'account'])] +#[InvalidatesQuery(LedgerTotals::class, eventType: 'ledger.entry-posted')] +final readonly class EntryPosted extends DomainEvent +{ + public function __construct(public string $account, public int $amount) + { + parent::__construct(); + } +} diff --git a/packages/cqrs/tests/CacheScannerFixtures/LedgerBalance.php b/packages/cqrs/tests/CacheScannerFixtures/LedgerBalance.php new file mode 100644 index 00000000..80a32eeb --- /dev/null +++ b/packages/cqrs/tests/CacheScannerFixtures/LedgerBalance.php @@ -0,0 +1,14 @@ +handlers())->toBe($result['handlers']) // both [] - ->and($committedManifest->destinations())->toBe($result['destinations']); // both [] + ->and($committedManifest->destinations())->toBe($result['destinations']) // both [] + ->and($committedManifest->cachedQueries())->toBe($result['cachedQueries']) // both [] + ->and($committedManifest->invalidations())->toBe($result['invalidations']); // both [] }); diff --git a/packages/cqrs/tests/Scanner/CacheAttributeScanTest.php b/packages/cqrs/tests/Scanner/CacheAttributeScanTest.php new file mode 100644 index 00000000..bd1ae57e --- /dev/null +++ b/packages/cqrs/tests/Scanner/CacheAttributeScanTest.php @@ -0,0 +1,81 @@ +scan(['Firefly\\Cqrs\\Tests\\CacheScannerFixtures\\' => dirname(__DIR__).'/CacheScannerFixtures']); + + expect($result['cachedQueries'])->toEqual([ + LedgerBalance::class => new CachedQueryPolicy(LedgerBalance::class, 60, LedgerBalance::class), + LedgerTotals::class => new CachedQueryPolicy(LedgerTotals::class, null, 'ledger.totals'), + ]); +}); + +it('compiles #[InvalidatesQuery] into rules keyed by the event class and its event type', function () { + $result = (new HandlerScanner)->scan(['Firefly\\Cqrs\\Tests\\CacheScannerFixtures\\' => dirname(__DIR__).'/CacheScannerFixtures']); + + expect($result['invalidations'])->toEqual([ + new InvalidationRule(EntryPosted::class, 'EntryPosted', LedgerBalance::class, null, ['account' => 'account']), + new InvalidationRule(EntryPosted::class, 'ledger.entry-posted', LedgerTotals::class), + ]) + ->and($result['invalidations'][1]->evictsGroup())->toBeTrue() + ->and($result['invalidations'][0]->evictsGroup())->toBeFalse(); +}); + +it('refuses a rule that names both a key template and a with map, and a TTL below one second', function () { + $scan = static fn (): array => (new HandlerScanner)->scan(['Firefly\\Cqrs\\Tests\\CacheScannerBadFixtures\\' => dirname(__DIR__).'/CacheScannerBadFixtures']); + + expect($scan)->toThrow(CqrsConfigurationException::class); +}); + +// Isolates the TTL guard: the scanner names classes after the files it walks, so a directory holding only a +// ZeroTtlQuery.php placeholder reaches the real ZeroTtlQuery fixture (autoloaded from its own file) and nothing else — +// AmbiguousRuleEvent, which sorts first in CacheScannerBadFixtures, can no longer short the scan before the TTL check. +it('refuses a TTL below one second on its own', function () { + $dir = sys_get_temp_dir().'/firefly-cqrs-zero-ttl-'.bin2hex(random_bytes(6)); + mkdir($dir); + touch($dir.'/ZeroTtlQuery.php'); + + try { + expect(static fn (): array => (new HandlerScanner)->scan(['Firefly\\Cqrs\\Tests\\CacheScannerBadFixtures\\' => $dir])) + ->toThrow(CqrsConfigurationException::class, 'declares ttl 0'); + } finally { + @unlink($dir.'/ZeroTtlQuery.php'); + @rmdir($dir); + } +}); + +it('round-trips both sections through the compiled manifest file', function () { + $result = (new HandlerScanner)->scan(['Firefly\\Cqrs\\Tests\\CacheScannerFixtures\\' => dirname(__DIR__).'/CacheScannerFixtures']); + $path = sys_get_temp_dir().'/firefly-cqrs-cache-manifest-'.bin2hex(random_bytes(6)).'.php'; + + try { + (new HandlerManifestCompiler)->write($result['handlers'], $result['destinations'], $path, $result['cachedQueries'], $result['invalidations']); + $manifest = HandlerManifest::load($path); + + expect($manifest->cachedQueries())->toEqual($result['cachedQueries']) + ->and($manifest->invalidations())->toEqual($result['invalidations']) + ->and($manifest->cachePolicyFor(LedgerTotals::class)?->group)->toBe('ledger.totals') + ->and($manifest->cachePolicyFor(EntryPosted::class))->toBeNull(); + } finally { + @unlink($path); + } +}); + +it('still loads a handler manifest compiled before the cache sections existed', function () { + $manifest = HandlerManifest::fromArray(['handlers' => [], 'destinations' => ['App\\E' => 'x']]); + + expect($manifest->cachedQueries())->toBe([]) + ->and($manifest->invalidations())->toBe([]) + ->and($manifest->destinations())->toBe(['App\\E' => 'x']); +}); diff --git a/packages/cqrs/tests/Scanner/HandlerScannerTest.php b/packages/cqrs/tests/Scanner/HandlerScannerTest.php index 4d34efb8..ee1c61e7 100644 --- a/packages/cqrs/tests/Scanner/HandlerScannerTest.php +++ b/packages/cqrs/tests/Scanner/HandlerScannerTest.php @@ -2,6 +2,8 @@ declare(strict_types=1); +use Firefly\Cqrs\Cache\CachedQueryPolicy; +use Firefly\Cqrs\Cache\InvalidationRule; use Firefly\Cqrs\Exception\CqrsConfigurationException; use Firefly\Cqrs\Handler\HandlerDescriptor; use Firefly\Cqrs\Handler\HandlerKind; @@ -15,7 +17,7 @@ use Firefly\Cqrs\Tests\ScannerFixtures\WidgetCreated; /** - * @return array{handlers: list, destinations: array} + * @return array{handlers: list, destinations: array, cachedQueries: array, invalidations: list} */ function scanHandlerFixtures(string $subdir, string $namespace): array { From d045be2b35666674753d49b9f2ed880c32a6801a Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 18:25:35 -0700 Subject: [PATCH 02/47] fix(cqrs): refuse to key queries with non-public state and validate #[InvalidatesQuery] with maps at scan time --- book/src-es/13-cli-cache.md | 2 + book/src/13-cli-cache.md | 2 + .../cqrs/src/Attributes/InvalidatesQuery.php | 4 +- packages/cqrs/src/Cache/QueryCacheKey.php | 25 +++++++++-- .../cqrs/src/Cache/UnkeyableQueryValue.php | 7 +-- packages/cqrs/src/Scanner/HandlerScanner.php | 45 ++++++++++++++++++- .../cqrs/tests/Cache/QueryCacheKeyTest.php | 39 ++++++++++++++++ .../CacheableWithEvent.php | 13 ++++++ .../CacheScannerBadFixtures/KeyedLedger.php | 20 +++++++++ .../CacheScannerBadFixtures/PagedLedger.php | 14 ++++++ .../PartialWithEvent.php | 13 ++++++ .../CacheScannerBadFixtures/PrivateLedger.php | 19 ++++++++ .../PrivateWithEvent.php | 13 ++++++ .../CacheScannerBadFixtures/TypoWithEvent.php | 14 ++++++ .../tests/Scanner/CacheAttributeScanTest.php | 23 ++++++++++ 15 files changed, 244 insertions(+), 9 deletions(-) create mode 100644 packages/cqrs/tests/CacheScannerBadFixtures/CacheableWithEvent.php create mode 100644 packages/cqrs/tests/CacheScannerBadFixtures/KeyedLedger.php create mode 100644 packages/cqrs/tests/CacheScannerBadFixtures/PagedLedger.php create mode 100644 packages/cqrs/tests/CacheScannerBadFixtures/PartialWithEvent.php create mode 100644 packages/cqrs/tests/CacheScannerBadFixtures/PrivateLedger.php create mode 100644 packages/cqrs/tests/CacheScannerBadFixtures/PrivateWithEvent.php create mode 100644 packages/cqrs/tests/CacheScannerBadFixtures/TypoWithEvent.php diff --git a/book/src-es/13-cli-cache.md b/book/src-es/13-cli-cache.md index 535dfceb..4192a765 100644 --- a/book/src-es/13-cli-cache.md +++ b/book/src-es/13-cli-cache.md @@ -105,6 +105,8 @@ final class ManifestCacheWriter $handlers['handlers'], $handlers['destinations'], $files[] = $dir.'/'.FireflyCachePaths::HANDLERS, + $handlers['cachedQueries'], + $handlers['invalidations'], ); // eda diff --git a/book/src/13-cli-cache.md b/book/src/13-cli-cache.md index 6a625374..b6240756 100644 --- a/book/src/13-cli-cache.md +++ b/book/src/13-cli-cache.md @@ -105,6 +105,8 @@ final class ManifestCacheWriter $handlers['handlers'], $handlers['destinations'], $files[] = $dir.'/'.FireflyCachePaths::HANDLERS, + $handlers['cachedQueries'], + $handlers['invalidations'], ); // eda diff --git a/packages/cqrs/src/Attributes/InvalidatesQuery.php b/packages/cqrs/src/Attributes/InvalidatesQuery.php index 40b1ed53..444e12c7 100644 --- a/packages/cqrs/src/Attributes/InvalidatesQuery.php +++ b/packages/cqrs/src/Attributes/InvalidatesQuery.php @@ -11,7 +11,9 @@ * * * neither $key nor $with — evict the query's whole group (every cached result of it); * * $with: ['queryProperty' => 'eventProperty'] — evict the ONE entry whose derived key the query would get with - * those property values (the query's every public property must be mapped, with the same types); + * those property values. The query's every public property must be mapped, with the same types; HandlerScanner + * refuses a map that names an unknown property, leaves one out, or targets a Cacheable query or one holding + * private or protected state (the types are the author's to keep: the event value is only known at runtime); * * $key: 'statement:{account}' — evict the entry a Cacheable query's own cacheKey() returns, `{name}` resolved * from the event's public properties or from the EDA payload (dotted paths reach into nested arrays). * diff --git a/packages/cqrs/src/Cache/QueryCacheKey.php b/packages/cqrs/src/Cache/QueryCacheKey.php index 3e8e87ac..274f9c2d 100644 --- a/packages/cqrs/src/Cache/QueryCacheKey.php +++ b/packages/cqrs/src/Cache/QueryCacheKey.php @@ -16,12 +16,15 @@ * The cache key of a query VALUE OBJECT: sha256 over the query class and a canonical form of its public * properties, so two equal queries share an entry and two different ones never do. Reflection-free: public * properties are read with get_object_vars() from outside the object's scope, the idiom EdaCommandEventPublisher - * uses for event payloads. + * uses for event payloads. That read cannot see private or protected state, so an object that holds any — the query + * itself or a value object nested in it — is not keyable: comparing the (array) cast, which lists every initialized + * instance property, with the public view detects it, and the key is null. Keying such an object by its public + * view alone would give Q(private 'a') and Q(private 'b') one entry and serve one caller's result to another. * * The canonical form sorts associative keys (so property order never matters), keeps list order (so ['x','y'] and * ['y','x'] are different queries), writes enums by case, dates as an exact instant with offset, and walks nested * objects by their own public properties. Types are kept: 3 and '3' are different keys. Anything that cannot be - * written stably answers null — the bus then runs the handler uncached. + * written stably (non-public state included) answers null — the bus then runs the handler uncached. * * fromProperties() builds the key a value object WOULD get from a property map: it is how #[InvalidatesQuery(with: * [...])] evicts one entry without constructing the query. @@ -32,7 +35,13 @@ final class QueryCacheKey public static function of(object $query): ?string { - return self::fromProperties($query::class, self::publicProperties($query)); + try { + $properties = self::publicProperties($query); + } catch (UnkeyableQueryValue) { + return null; + } + + return self::fromProperties($query::class, $properties); } /** @@ -53,12 +62,20 @@ public static function fromProperties(string $queryClass, array $properties): ?s } /** + * The object's public properties, or UnkeyableQueryValue when it also holds private or protected instance state + * (which get_object_vars() from this scope would silently leave out of the key). + * * @return array */ private static function publicProperties(object $object): array { + $public = get_object_vars($object); + if (count((array) $object) !== count($public)) { + throw new UnkeyableQueryValue; + } + $properties = []; - foreach (get_object_vars($object) as $name => $value) { + foreach ($public as $name => $value) { $properties[(string) $name] = $value; } diff --git a/packages/cqrs/src/Cache/UnkeyableQueryValue.php b/packages/cqrs/src/Cache/UnkeyableQueryValue.php index 97b8b8a7..85327ce4 100644 --- a/packages/cqrs/src/Cache/UnkeyableQueryValue.php +++ b/packages/cqrs/src/Cache/UnkeyableQueryValue.php @@ -7,9 +7,10 @@ use RuntimeException; /** - * Internal to QueryCacheKey: a value that cannot be part of a stable cache key (a closure, a resource, a NaN, a - * structure deeper than QueryCacheKey::MAX_DEPTH). Never escapes QueryCacheKey, which answers null instead — - * "do not cache this call" — so an exotic query degrades to running its handler, never to an error. + * Internal to QueryCacheKey: a value that cannot be part of a stable cache key (a closure, a resource, a NaN, an + * object holding private or protected state, a structure deeper than QueryCacheKey::MAX_DEPTH). Never escapes + * QueryCacheKey, which answers null instead — "do not cache this call" — so an exotic query degrades to running its + * handler, never to an error. * * @internal */ diff --git a/packages/cqrs/src/Scanner/HandlerScanner.php b/packages/cqrs/src/Scanner/HandlerScanner.php index 02d544a2..17638a1a 100644 --- a/packages/cqrs/src/Scanner/HandlerScanner.php +++ b/packages/cqrs/src/Scanner/HandlerScanner.php @@ -9,6 +9,7 @@ use Firefly\Cqrs\Attributes\InvalidatesQuery; use Firefly\Cqrs\Attributes\PublishDomainEvent; use Firefly\Cqrs\Attributes\QueryHandler; +use Firefly\Cqrs\Cache\Cacheable; use Firefly\Cqrs\Cache\CachedQueryPolicy; use Firefly\Cqrs\Cache\InvalidationRule; use Firefly\Cqrs\Exception\CqrsConfigurationException; @@ -26,7 +27,8 @@ * (CqrsConfigurationException) on an unresolvable type or a missing/multi-param handle() — never silently skips. * It also compiles #[CachedQuery] off query classes into per-query policies and #[InvalidatesQuery] off event classes * into invalidation rules, refusing a rule with both a key template and a `with` map, a rule naming a class that does - * not exist, and a TTL below one second. + * not exist, a `with` map that does not name exactly the query's public properties (or targets a Cacheable query or + * one with non-public state), and a TTL below one second. * Runs only at cache time; production loads the compiled HandlerManifest via require+map. Mirrors ScheduledScanner. */ final class HandlerScanner @@ -89,6 +91,9 @@ public function scan(array $psr4): array if ($rule->key !== null && $rule->with !== []) { throw new CqrsConfigurationException("#[InvalidatesQuery({$rule->query}::class)] on [{$class}] declares both key and with; a rule evicts by one or the other."); } + if ($rule->with !== []) { + $this->assertWithMapsQuery($class, $rule->query, $rule->with); + } $invalidations[] = new InvalidationRule($class, $rule->eventType ?? $reflection->getShortName(), $rule->query, $rule->key, $rule->with); } } @@ -96,6 +101,44 @@ public function scan(array $psr4): array return ['handlers' => $handlers, 'destinations' => $destinations, 'cachedQueries' => $cachedQueries, 'invalidations' => $invalidations]; } + /** + * A `with` map names the query properties QueryCacheKey derives the key from, so it must name exactly the query's + * public instance properties: an unknown name or an unmapped property derives a key no cached entry has, and the + * eviction silently misses. A query holding private or protected state is never keyed by value (QueryCacheKey + * answers null), and a Cacheable query is keyed by its own cacheKey(), so `with` can match neither. + * + * @param class-string $queryClass + * @param array $with + */ + private function assertWithMapsQuery(string $eventClass, string $queryClass, array $with): void + { + $where = "#[InvalidatesQuery({$queryClass}::class)] on [{$eventClass}]"; + if (is_subclass_of($queryClass, Cacheable::class)) { + throw new CqrsConfigurationException("{$where} declares a with map, but the query is Cacheable and keyed by its own cacheKey(); use a key template instead."); + } + + $public = []; + foreach ((new ReflectionClass($queryClass))->getProperties() as $property) { + if ($property->isStatic()) { + continue; + } + if (! $property->isPublic()) { + throw new CqrsConfigurationException("{$where} declares a with map, but [{$queryClass}::\${$property->getName()}] is not public; a query keyed by value holds only public state."); + } + $public[] = $property->getName(); + } + + $unknown = array_diff(array_keys($with), $public); + if ($unknown !== []) { + throw new CqrsConfigurationException("{$where} maps [".implode(', ', $unknown)."], which [{$queryClass}] does not declare as public properties."); + } + + $unmapped = array_diff($public, array_keys($with)); + if ($unmapped !== []) { + throw new CqrsConfigurationException("{$where} leaves [".implode(', ', $unmapped).'] unmapped; a with map names every public property of the query.'); + } + } + /** * @param ReflectionClass $reflection */ diff --git a/packages/cqrs/tests/Cache/QueryCacheKeyTest.php b/packages/cqrs/tests/Cache/QueryCacheKeyTest.php index 50deb399..62714026 100644 --- a/packages/cqrs/tests/Cache/QueryCacheKeyTest.php +++ b/packages/cqrs/tests/Cache/QueryCacheKeyTest.php @@ -65,3 +65,42 @@ public function __construct(public float $value) {} expect(QueryCacheKey::of($closure))->toBeNull() ->and(QueryCacheKey::of($nan))->toBeNull(); }); + +it('refuses to key a query whose state is private or protected, instead of colliding on its public view', function () { + $query = new class('a') + { + public function __construct(private string $account) {} + + public function account(): string + { + return $this->account; + } + }; + $protected = new class('a') + { + public function __construct(protected string $account) {} + }; + + expect(QueryCacheKey::of($query))->toBeNull() + ->and(QueryCacheKey::of(new ($query::class)('b')))->toBeNull() + ->and(QueryCacheKey::of($protected))->toBeNull(); +}); + +it('refuses to key a public query that nests a value object with private state', function () { + $money = new class(100) + { + public function __construct(private int $cents) {} + + public function cents(): int + { + return $this->cents; + } + }; + $query = new class($money) + { + public function __construct(public object $amount) {} + }; + + expect(QueryCacheKey::of($query))->toBeNull() + ->and(QueryCacheKey::of(new ($query::class)(new ($money::class)(200))))->toBeNull(); +}); diff --git a/packages/cqrs/tests/CacheScannerBadFixtures/CacheableWithEvent.php b/packages/cqrs/tests/CacheScannerBadFixtures/CacheableWithEvent.php new file mode 100644 index 00000000..46a5ee2e --- /dev/null +++ b/packages/cqrs/tests/CacheScannerBadFixtures/CacheableWithEvent.php @@ -0,0 +1,13 @@ + 'account'])] +final class CacheableWithEvent +{ + public function __construct(public string $account = 'a') {} +} diff --git a/packages/cqrs/tests/CacheScannerBadFixtures/KeyedLedger.php b/packages/cqrs/tests/CacheScannerBadFixtures/KeyedLedger.php new file mode 100644 index 00000000..7aa248a5 --- /dev/null +++ b/packages/cqrs/tests/CacheScannerBadFixtures/KeyedLedger.php @@ -0,0 +1,20 @@ +account; + } +} diff --git a/packages/cqrs/tests/CacheScannerBadFixtures/PagedLedger.php b/packages/cqrs/tests/CacheScannerBadFixtures/PagedLedger.php new file mode 100644 index 00000000..9cd71c47 --- /dev/null +++ b/packages/cqrs/tests/CacheScannerBadFixtures/PagedLedger.php @@ -0,0 +1,14 @@ + 'account'])] +final class PartialWithEvent +{ + public function __construct(public string $account = 'a') {} +} diff --git a/packages/cqrs/tests/CacheScannerBadFixtures/PrivateLedger.php b/packages/cqrs/tests/CacheScannerBadFixtures/PrivateLedger.php new file mode 100644 index 00000000..bc86e573 --- /dev/null +++ b/packages/cqrs/tests/CacheScannerBadFixtures/PrivateLedger.php @@ -0,0 +1,19 @@ +account; + } +} diff --git a/packages/cqrs/tests/CacheScannerBadFixtures/PrivateWithEvent.php b/packages/cqrs/tests/CacheScannerBadFixtures/PrivateWithEvent.php new file mode 100644 index 00000000..09546a5f --- /dev/null +++ b/packages/cqrs/tests/CacheScannerBadFixtures/PrivateWithEvent.php @@ -0,0 +1,13 @@ + 'account'])] +final class PrivateWithEvent +{ + public function __construct(public string $account = 'a') {} +} diff --git a/packages/cqrs/tests/CacheScannerBadFixtures/TypoWithEvent.php b/packages/cqrs/tests/CacheScannerBadFixtures/TypoWithEvent.php new file mode 100644 index 00000000..e0988ec4 --- /dev/null +++ b/packages/cqrs/tests/CacheScannerBadFixtures/TypoWithEvent.php @@ -0,0 +1,14 @@ + 'account'])] +final class TypoWithEvent +{ + public function __construct(public string $account = 'a') {} +} diff --git a/packages/cqrs/tests/Scanner/CacheAttributeScanTest.php b/packages/cqrs/tests/Scanner/CacheAttributeScanTest.php index bd1ae57e..8a253de7 100644 --- a/packages/cqrs/tests/Scanner/CacheAttributeScanTest.php +++ b/packages/cqrs/tests/Scanner/CacheAttributeScanTest.php @@ -55,6 +55,29 @@ } }); +// Each `with` guard is isolated the same way: a directory holding only the event's placeholder file scans that one +// event, and its query fixture autoloads from CacheScannerBadFixtures without being scanned itself. +dataset('with map misconfigurations', [ + 'an unknown query property' => ['TypoWithEvent', 'maps [acount]'], + 'an unmapped query property' => ['PartialWithEvent', 'leaves [page] unmapped'], + 'a query with private state' => ['PrivateWithEvent', 'is not public'], + 'a Cacheable query' => ['CacheableWithEvent', 'use a key template instead'], +]); + +it('refuses a with map that does not name exactly the query\'s public properties', function (string $event, string $message) { + $dir = sys_get_temp_dir().'/firefly-cqrs-with-map-'.bin2hex(random_bytes(6)); + mkdir($dir); + touch($dir.'/'.$event.'.php'); + + try { + expect(static fn (): array => (new HandlerScanner)->scan(['Firefly\\Cqrs\\Tests\\CacheScannerBadFixtures\\' => $dir])) + ->toThrow(CqrsConfigurationException::class, $message); + } finally { + @unlink($dir.'/'.$event.'.php'); + @rmdir($dir); + } +})->with('with map misconfigurations'); + it('round-trips both sections through the compiled manifest file', function () { $result = (new HandlerScanner)->scan(['Firefly\\Cqrs\\Tests\\CacheScannerFixtures\\' => dirname(__DIR__).'/CacheScannerFixtures']); $path = sys_get_temp_dir().'/firefly-cqrs-cache-manifest-'.bin2hex(random_bytes(6)).'.php'; From 19227f5cae049655975bc7c67650abc5f3b74ffc Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 18:28:54 -0700 Subject: [PATCH 03/47] fix(cqrs): refuse an #[InvalidatesQuery] key template on a query that is keyed by value --- .../cqrs/src/Attributes/InvalidatesQuery.php | 2 ++ packages/cqrs/src/Scanner/HandlerScanner.php | 6 +++- .../CacheableTemplateEvent.php | 13 ++++++++ .../ValueKeyedTemplateEvent.php | 14 +++++++++ .../tests/Scanner/CacheAttributeScanTest.php | 30 ++++++++++++++++--- 5 files changed, 60 insertions(+), 5 deletions(-) create mode 100644 packages/cqrs/tests/CacheScannerBadFixtures/CacheableTemplateEvent.php create mode 100644 packages/cqrs/tests/CacheScannerBadFixtures/ValueKeyedTemplateEvent.php diff --git a/packages/cqrs/src/Attributes/InvalidatesQuery.php b/packages/cqrs/src/Attributes/InvalidatesQuery.php index 444e12c7..5e9e7d71 100644 --- a/packages/cqrs/src/Attributes/InvalidatesQuery.php +++ b/packages/cqrs/src/Attributes/InvalidatesQuery.php @@ -16,6 +16,8 @@ * private or protected state (the types are the author's to keep: the event value is only known at runtime); * * $key: 'statement:{account}' — evict the entry a Cacheable query's own cacheKey() returns, `{name}` resolved * from the event's public properties or from the EDA payload (dotted paths reach into nested arrays). + * HandlerScanner refuses a key template on a query that is not Cacheable: that query is keyed by value (a + * QueryCacheKey digest), so no template can name its entries and the eviction would silently miss. * * An event published over EDA is matched by $eventType — default: the short class name, which is what * DomainEvent::eventType() returns — so another process's envelope evicts this process's cache too. A placeholder diff --git a/packages/cqrs/src/Scanner/HandlerScanner.php b/packages/cqrs/src/Scanner/HandlerScanner.php index 17638a1a..564a737e 100644 --- a/packages/cqrs/src/Scanner/HandlerScanner.php +++ b/packages/cqrs/src/Scanner/HandlerScanner.php @@ -27,7 +27,8 @@ * (CqrsConfigurationException) on an unresolvable type or a missing/multi-param handle() — never silently skips. * It also compiles #[CachedQuery] off query classes into per-query policies and #[InvalidatesQuery] off event classes * into invalidation rules, refusing a rule with both a key template and a `with` map, a rule naming a class that does - * not exist, a `with` map that does not name exactly the query's public properties (or targets a Cacheable query or + * not exist, a key template on a query that is not Cacheable (its entries live under a QueryCacheKey digest no + * template can name, so the eviction would silently miss), a `with` map that does not name exactly the query's public properties (or targets a Cacheable query or * one with non-public state), and a TTL below one second. * Runs only at cache time; production loads the compiled HandlerManifest via require+map. Mirrors ScheduledScanner. */ @@ -91,6 +92,9 @@ public function scan(array $psr4): array if ($rule->key !== null && $rule->with !== []) { throw new CqrsConfigurationException("#[InvalidatesQuery({$rule->query}::class)] on [{$class}] declares both key and with; a rule evicts by one or the other."); } + if ($rule->key !== null && ! is_subclass_of($rule->query, Cacheable::class)) { + throw new CqrsConfigurationException("#[InvalidatesQuery({$rule->query}::class)] on [{$class}] declares a key template, but [{$rule->query}] is keyed by value (QueryCacheKey); use a with map or no key to evict its group."); + } if ($rule->with !== []) { $this->assertWithMapsQuery($class, $rule->query, $rule->with); } diff --git a/packages/cqrs/tests/CacheScannerBadFixtures/CacheableTemplateEvent.php b/packages/cqrs/tests/CacheScannerBadFixtures/CacheableTemplateEvent.php new file mode 100644 index 00000000..05703c6d --- /dev/null +++ b/packages/cqrs/tests/CacheScannerBadFixtures/CacheableTemplateEvent.php @@ -0,0 +1,13 @@ + ['TypoWithEvent', 'maps [acount]'], 'an unmapped query property' => ['PartialWithEvent', 'leaves [page] unmapped'], 'a query with private state' => ['PrivateWithEvent', 'is not public'], 'a Cacheable query' => ['CacheableWithEvent', 'use a key template instead'], + 'a key template on a value-keyed query' => ['ValueKeyedTemplateEvent', 'is keyed by value (QueryCacheKey)'], ]); -it('refuses a with map that does not name exactly the query\'s public properties', function (string $event, string $message) { +it('refuses a with map or key template that cannot name the query\'s cached entries', function (string $event, string $message) { $dir = sys_get_temp_dir().'/firefly-cqrs-with-map-'.bin2hex(random_bytes(6)); mkdir($dir); touch($dir.'/'.$event.'.php'); @@ -76,7 +79,26 @@ @unlink($dir.'/'.$event.'.php'); @rmdir($dir); } -})->with('with map misconfigurations'); +})->with('unmatchable eviction rules'); + +// The counterpart of the value-keyed refusal: a key template on a Cacheable query names the entries its own +// cacheKey() stores, so the scan compiles it into a key rule. +it('compiles a key template on a Cacheable query', function () { + $dir = sys_get_temp_dir().'/firefly-cqrs-key-template-'.bin2hex(random_bytes(6)); + mkdir($dir); + touch($dir.'/CacheableTemplateEvent.php'); + + try { + $result = (new HandlerScanner)->scan(['Firefly\\Cqrs\\Tests\\CacheScannerBadFixtures\\' => $dir]); + + expect($result['invalidations'])->toEqual([ + new InvalidationRule(CacheableTemplateEvent::class, 'CacheableTemplateEvent', KeyedLedger::class, 'ledger:{account}'), + ]); + } finally { + @unlink($dir.'/CacheableTemplateEvent.php'); + @rmdir($dir); + } +}); it('round-trips both sections through the compiled manifest file', function () { $result = (new HandlerScanner)->scan(['Firefly\\Cqrs\\Tests\\CacheScannerFixtures\\' => dirname(__DIR__).'/CacheScannerFixtures']); From 6f2f262f59d6a99d75e95c2c4ec70925ba994fd3 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 18:33:27 -0700 Subject: [PATCH 04/47] fix(cqrs): refuse #[CachedQuery] on a value-keyed query with non-public state and never key a Traversable by its empty view --- packages/cqrs/src/Cache/QueryCacheKey.php | 9 ++++ packages/cqrs/src/Scanner/HandlerScanner.php | 41 ++++++++++++++----- .../cqrs/tests/Cache/QueryCacheKeyTest.php | 41 +++++++++++++++++++ .../PrivateKeyedLedger.php | 20 +++++++++ .../tests/Scanner/CacheAttributeScanTest.php | 34 +++++++++++++++ 5 files changed, 134 insertions(+), 11 deletions(-) create mode 100644 packages/cqrs/tests/CacheScannerBadFixtures/PrivateKeyedLedger.php diff --git a/packages/cqrs/src/Cache/QueryCacheKey.php b/packages/cqrs/src/Cache/QueryCacheKey.php index 274f9c2d..5e0d3a20 100644 --- a/packages/cqrs/src/Cache/QueryCacheKey.php +++ b/packages/cqrs/src/Cache/QueryCacheKey.php @@ -10,6 +10,7 @@ use JsonException; use JsonSerializable; use Stringable; +use Traversable; use UnitEnum; /** @@ -21,6 +22,11 @@ * instance property, with the public view detects it, and the key is null. Keying such an object by its public * view alone would give Q(private 'a') and Q(private 'b') one entry and serve one caller's result to another. * + * A Traversable (a Generator, an Iterator, an IteratorAggregate) is refused the same way: its values live in internal + * state no property read can see, so two queries carrying generators that yield 1 and 2 would otherwise share the one + * key of an empty public view. A Traversable that is also JsonSerializable or Stringable (a collection) is keyed by + * that form instead, which does carry its values. + * * The canonical form sorts associative keys (so property order never matters), keeps list order (so ['x','y'] and * ['y','x'] are different queries), writes enums by case, dates as an exact instant with offset, and walks nested * objects by their own public properties. Types are kept: 3 and '3' are different keys. Anything that cannot be @@ -69,6 +75,9 @@ public static function fromProperties(string $queryClass, array $properties): ?s */ private static function publicProperties(object $object): array { + if ($object instanceof Traversable) { + throw new UnkeyableQueryValue; + } $public = get_object_vars($object); if (count((array) $object) !== count($public)) { throw new UnkeyableQueryValue; diff --git a/packages/cqrs/src/Scanner/HandlerScanner.php b/packages/cqrs/src/Scanner/HandlerScanner.php index 564a737e..d26e201f 100644 --- a/packages/cqrs/src/Scanner/HandlerScanner.php +++ b/packages/cqrs/src/Scanner/HandlerScanner.php @@ -26,7 +26,8 @@ * #[PublishDomainEvent] off DomainEvent subclasses into a {eventClass => destination} map. Fails loud * (CqrsConfigurationException) on an unresolvable type or a missing/multi-param handle() — never silently skips. * It also compiles #[CachedQuery] off query classes into per-query policies and #[InvalidatesQuery] off event classes - * into invalidation rules, refusing a rule with both a key template and a `with` map, a rule naming a class that does + * into invalidation rules, refusing a #[CachedQuery] on a query that is not Cacheable yet holds non-public state (QueryCacheKey + * could never key it, so the opt-in would silently do nothing), a rule with both a key template and a `with` map, a rule naming a class that does * not exist, a key template on a query that is not Cacheable (its entries live under a QueryCacheKey digest no * template can name, so the eviction would silently miss), a `with` map that does not name exactly the query's public properties (or targets a Cacheable query or * one with non-public state), and a TTL below one second. @@ -81,6 +82,9 @@ public function scan(array $psr4): array if ($cached->ttl !== null && $cached->ttl < 1) { throw new CqrsConfigurationException("#[CachedQuery] on [{$class}] declares ttl {$cached->ttl}; a TTL is at least one second (omit it to use firefly.cqrs.query.cache_ttl)."); } + if (! is_subclass_of($class, Cacheable::class)) { + $this->publicPropertiesOrFail($class, "#[CachedQuery] on [{$class}]", 'opts the query into caching'); + } $cachedQueries[$class] = new CachedQueryPolicy($class, $cached->ttl, $cached->group ?? $class); } @@ -121,16 +125,7 @@ private function assertWithMapsQuery(string $eventClass, string $queryClass, arr throw new CqrsConfigurationException("{$where} declares a with map, but the query is Cacheable and keyed by its own cacheKey(); use a key template instead."); } - $public = []; - foreach ((new ReflectionClass($queryClass))->getProperties() as $property) { - if ($property->isStatic()) { - continue; - } - if (! $property->isPublic()) { - throw new CqrsConfigurationException("{$where} declares a with map, but [{$queryClass}::\${$property->getName()}] is not public; a query keyed by value holds only public state."); - } - $public[] = $property->getName(); - } + $public = $this->publicPropertiesOrFail($queryClass, $where, 'declares a with map'); $unknown = array_diff(array_keys($with), $public); if ($unknown !== []) { @@ -143,6 +138,30 @@ private function assertWithMapsQuery(string $eventClass, string $queryClass, arr } } + /** + * The query's non-static property names, refusing any that is not public: QueryCacheKey reads only public state + * and answers null for an object holding more, so a value-keyed query with private or protected state would never + * be cached (#[CachedQuery] silently doing nothing) and no `with` map could name its entries. + * + * @param class-string $queryClass + * @return list + */ + private function publicPropertiesOrFail(string $queryClass, string $where, string $what): array + { + $public = []; + foreach ((new ReflectionClass($queryClass))->getProperties() as $property) { + if ($property->isStatic()) { + continue; + } + if (! $property->isPublic()) { + throw new CqrsConfigurationException("{$where} {$what}, but [{$queryClass}::\${$property->getName()}] is not public; a query keyed by value holds only public state (or implements Cacheable to key itself)."); + } + $public[] = $property->getName(); + } + + return $public; + } + /** * @param ReflectionClass $reflection */ diff --git a/packages/cqrs/tests/Cache/QueryCacheKeyTest.php b/packages/cqrs/tests/Cache/QueryCacheKeyTest.php index 62714026..8c46f189 100644 --- a/packages/cqrs/tests/Cache/QueryCacheKeyTest.php +++ b/packages/cqrs/tests/Cache/QueryCacheKeyTest.php @@ -104,3 +104,44 @@ public function __construct(public object $amount) {} expect(QueryCacheKey::of($query))->toBeNull() ->and(QueryCacheKey::of(new ($query::class)(new ($money::class)(200))))->toBeNull(); }); + +it('refuses to key a query that carries a generator or another Traversable, instead of colliding on its empty view', function () { + $query = new class((static fn () => yield 1)()) + { + /** @param iterable $ids */ + public function __construct(public iterable $ids) {} + }; + $other = new ($query::class)((static fn () => yield 2)()); + $iterator = new ($query::class)(new ArrayIterator([1])); + + expect(QueryCacheKey::of($query))->toBeNull() + ->and(QueryCacheKey::of($other))->toBeNull() + ->and(QueryCacheKey::of($iterator))->toBeNull() + ->and(QueryCacheKey::of(new ArrayIterator([1])))->toBeNull(); +}); + +it('keys a Traversable collection by its JSON form, which carries its values', function () { + $collection = static fn (array $items): object => new class($items) implements IteratorAggregate, JsonSerializable + { + /** @param list $items */ + public function __construct(private array $items) {} + + public function getIterator(): ArrayIterator + { + return new ArrayIterator($this->items); + } + + /** @return list */ + public function jsonSerialize(): array + { + return $this->items; + } + }; + $query = static fn (array $items): object => new class($collection($items)) + { + public function __construct(public object $ids) {} + }; + + expect(QueryCacheKey::of($query([1])))->not->toBeNull() + ->and(QueryCacheKey::of($query([1])))->not->toBe(QueryCacheKey::of($query([2]))); +}); diff --git a/packages/cqrs/tests/CacheScannerBadFixtures/PrivateKeyedLedger.php b/packages/cqrs/tests/CacheScannerBadFixtures/PrivateKeyedLedger.php new file mode 100644 index 00000000..3a4ec599 --- /dev/null +++ b/packages/cqrs/tests/CacheScannerBadFixtures/PrivateKeyedLedger.php @@ -0,0 +1,20 @@ +account; + } +} diff --git a/packages/cqrs/tests/Scanner/CacheAttributeScanTest.php b/packages/cqrs/tests/Scanner/CacheAttributeScanTest.php index daafa5ab..17df2ca7 100644 --- a/packages/cqrs/tests/Scanner/CacheAttributeScanTest.php +++ b/packages/cqrs/tests/Scanner/CacheAttributeScanTest.php @@ -10,6 +10,7 @@ use Firefly\Cqrs\Scanner\HandlerScanner; use Firefly\Cqrs\Tests\CacheScannerBadFixtures\CacheableTemplateEvent; use Firefly\Cqrs\Tests\CacheScannerBadFixtures\KeyedLedger; +use Firefly\Cqrs\Tests\CacheScannerBadFixtures\PrivateKeyedLedger; use Firefly\Cqrs\Tests\CacheScannerFixtures\EntryPosted; use Firefly\Cqrs\Tests\CacheScannerFixtures\LedgerBalance; use Firefly\Cqrs\Tests\CacheScannerFixtures\LedgerTotals; @@ -57,6 +58,39 @@ } }); +// A value-keyed #[CachedQuery] with private state could never be keyed (QueryCacheKey answers null), so the scan +// refuses it rather than compiling a policy that silently caches nothing; a Cacheable query keys itself and stays allowed. +it('refuses #[CachedQuery] on a value-keyed query with non-public state', function () { + $dir = sys_get_temp_dir().'/firefly-cqrs-private-cached-'.bin2hex(random_bytes(6)); + mkdir($dir); + touch($dir.'/PrivateLedger.php'); + + try { + expect(static fn (): array => (new HandlerScanner)->scan(['Firefly\\Cqrs\\Tests\\CacheScannerBadFixtures\\' => $dir])) + ->toThrow(CqrsConfigurationException::class, 'is not public'); + } finally { + @unlink($dir.'/PrivateLedger.php'); + @rmdir($dir); + } +}); + +it('compiles #[CachedQuery] on a Cacheable query whatever state it holds', function () { + $dir = sys_get_temp_dir().'/firefly-cqrs-keyed-cached-'.bin2hex(random_bytes(6)); + mkdir($dir); + touch($dir.'/PrivateKeyedLedger.php'); + + try { + $result = (new HandlerScanner)->scan(['Firefly\\Cqrs\\Tests\\CacheScannerBadFixtures\\' => $dir]); + + expect($result['cachedQueries'])->toEqual([ + PrivateKeyedLedger::class => new CachedQueryPolicy(PrivateKeyedLedger::class, null, PrivateKeyedLedger::class), + ]); + } finally { + @unlink($dir.'/PrivateKeyedLedger.php'); + @rmdir($dir); + } +}); + // Each `with`/key guard is isolated the same way: a directory holding only the event's placeholder file scans that one // event, and its query fixture autoloads from CacheScannerBadFixtures without being scanned itself. dataset('unmatchable eviction rules', [ From 7a7ebe76960833c4431ddc674210649ec82a04c5 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 18:36:57 -0700 Subject: [PATCH 05/47] test(cqrs): type the Traversable collection fixture so PHPStan max passes on the query cache key tests --- packages/cqrs/tests/Cache/QueryCacheKeyTest.php | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/cqrs/tests/Cache/QueryCacheKeyTest.php b/packages/cqrs/tests/Cache/QueryCacheKeyTest.php index 8c46f189..3059ccf3 100644 --- a/packages/cqrs/tests/Cache/QueryCacheKeyTest.php +++ b/packages/cqrs/tests/Cache/QueryCacheKeyTest.php @@ -123,15 +123,16 @@ public function __construct(public iterable $ids) {} it('keys a Traversable collection by its JSON form, which carries its values', function () { $collection = static fn (array $items): object => new class($items) implements IteratorAggregate, JsonSerializable { - /** @param list $items */ + /** @param array $items */ public function __construct(private array $items) {} + /** @return ArrayIterator */ public function getIterator(): ArrayIterator { return new ArrayIterator($this->items); } - /** @return list */ + /** @return array */ public function jsonSerialize(): array { return $this->items; From 913ee853922c545a4edba729367c7cfa01745580 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 18:36:57 -0700 Subject: [PATCH 06/47] feat(cqrs): a Laravel-cache query cache with generation-based group eviction that never fails a query --- packages/cqrs/src/Cache/CacheQueryCache.php | 161 +++++++++++++++ packages/cqrs/src/Cache/GroupedQueryCache.php | 25 +++ .../cqrs/tests/Cache/CacheQueryCacheTest.php | 192 ++++++++++++++++++ .../cqrs/tests/Support/RecordingLogger.php | 23 +++ 4 files changed, 401 insertions(+) create mode 100644 packages/cqrs/src/Cache/CacheQueryCache.php create mode 100644 packages/cqrs/src/Cache/GroupedQueryCache.php create mode 100644 packages/cqrs/tests/Cache/CacheQueryCacheTest.php create mode 100644 packages/cqrs/tests/Support/RecordingLogger.php diff --git a/packages/cqrs/src/Cache/CacheQueryCache.php b/packages/cqrs/src/Cache/CacheQueryCache.php new file mode 100644 index 00000000..69988d47 --- /dev/null +++ b/packages/cqrs/src/Cache/CacheQueryCache.php @@ -0,0 +1,161 @@ +g: => the group's current generation + * q: => one cached result + * + * evictGroup() forgets the generation; the next lookup starts a fresh random one, and no entry written under an + * older generation is reachable again — they expire on their own TTL (which is why the module guide tells you to + * set one). That also settles the classic read/evict race in the group's favour: a handler that read the database + * before a commit stores its result under the generation it looked up, which the commit's eviction has just made + * unreachable. A single-key eviction (evictKey) does not have that property; it is the cheaper, narrower tool. + * + * A generation is created with add() and a fixed TTL: Laravel's Repository::add() is only atomic when it is given a + * TTL, and two processes starting a generation at once must agree on one. Keys are hashed, so a group or key of any + * length fits every store's key limit (memcached: 250 bytes; the database store: 255). + * + * A CACHE FAILURE NEVER FAILS A QUERY: every operation catches, logs a warning and answers a miss — the handler + * runs, exactly as it would with no cache at all. null is never stored, because DefaultQueryBus reads null as a + * miss. + */ +final class CacheQueryCache implements GroupedQueryCache +{ + public const string DEFAULT_PREFIX = 'firefly:cqrs:'; + + /** Thirty days: long enough to outlive any entry TTL worth using; a generation that expires only costs misses. */ + public const int GENERATION_TTL = 2_592_000; + + public function __construct( + private readonly Repository $cache, + private readonly string $prefix = self::DEFAULT_PREFIX, + private readonly ?LoggerInterface $logger = null, + ) {} + + public function get(string $key): mixed + { + return $this->lookup('', $key); + } + + public function put(string $key, mixed $value, ?int $ttl): void + { + $this->store('', $key, $value, $ttl); + } + + public function evict(string $key): void + { + $this->evictKey('', $key); + } + + public function lookup(string $group, string $key): mixed + { + try { + $generation = $this->generation($group, true); + + return $generation === null ? null : $this->cache->get($this->entryKey($group, $generation, $key)); + } catch (Throwable $e) { + $this->report('lookup', $group, $e); + + return null; + } + } + + public function store(string $group, string $key, mixed $value, ?int $ttl): void + { + if ($value === null) { + return; + } + + try { + $generation = $this->generation($group, true); + if ($generation === null) { + return; + } + + $entry = $this->entryKey($group, $generation, $key); + if ($ttl === null) { + $this->cache->forever($entry, $value); + } else { + $this->cache->put($entry, $value, max(1, $ttl)); + } + } catch (Throwable $e) { + $this->report('store', $group, $e); + } + } + + public function evictKey(string $group, string $key): void + { + try { + $generation = $this->generation($group, false); + if ($generation !== null) { + $this->cache->forget($this->entryKey($group, $generation, $key)); + } + } catch (Throwable $e) { + $this->report('evictKey', $group, $e); + } + } + + public function evictGroup(string $group): void + { + try { + $this->cache->forget($this->generationKey($group)); + } catch (Throwable $e) { + $this->report('evictGroup', $group, $e); + } + } + + /** The group's generation; with $create, started now when it has none. null when none could be started. */ + private function generation(string $group, bool $create): ?string + { + $key = $this->generationKey($group); + $current = $this->cache->get($key); + if (is_string($current) && $current !== '') { + return $current; + } + + if (! $create) { + return null; + } + + $fresh = bin2hex(random_bytes(8)); + if ($this->cache->add($key, $fresh, self::GENERATION_TTL)) { + return $fresh; + } + + // Another process started one in between — use it; anything else means the store refused the write. + $current = $this->cache->get($key); + + return is_string($current) && $current !== '' ? $current : null; + } + + private function generationKey(string $group): string + { + return $this->prefix.'g:'.hash('sha256', $group); + } + + private function entryKey(string $group, string $generation, string $key): string + { + return $this->prefix.'q:'.hash('sha256', $group."\x1f".$generation."\x1f".$key); + } + + private function report(string $operation, string $group, Throwable $e): void + { + $this->logger?->warning( + "CQRS query cache {$operation} failed for group [{$group}]; the query runs uncached: {$e->getMessage()}", + ['exception' => $e], + ); + } +} diff --git a/packages/cqrs/src/Cache/GroupedQueryCache.php b/packages/cqrs/src/Cache/GroupedQueryCache.php new file mode 100644 index 00000000..e2122a6d --- /dev/null +++ b/packages/cqrs/src/Cache/GroupedQueryCache.php @@ -0,0 +1,25 @@ +addConnection(['driver' => 'sqlite', 'database' => ':memory:', 'prefix' => '']); + $connection = $capsule->getConnection(); + $connection->getSchemaBuilder()->create('cache', static function (Blueprint $table): void { + $table->string('key')->primary(); + $table->mediumText('value'); + $table->integer('expiration'); + }); + $connection->getSchemaBuilder()->create('cache_locks', static function (Blueprint $table): void { + $table->string('key')->primary(); + $table->string('owner'); + $table->integer('expiration'); + }); + + return new Repository(new DatabaseStore($connection, 'cache')); +} + +function w4QueryCacheRepository(string $store): Repository +{ + return $store === 'database' ? w4QueryCacheOnDatabase() : new Repository(new ArrayStore); +} + +dataset('w4 query cache stores', [ + 'array store' => ['array'], + 'database store' => ['database'], +]); + +afterEach(function () { + Carbon::setTestNow(); +}); + +it('is a grouped query cache that misses, then hits what it stored', function (string $store) { + $cache = new CacheQueryCache(w4QueryCacheRepository($store)); + + expect($cache)->toBeInstanceOf(GroupedQueryCache::class) + ->and($cache->lookup('balances', 'k1'))->toBeNull(); + + $cache->store('balances', 'k1', ['balance' => 10], 60); + + expect($cache->lookup('balances', 'k1'))->toBe(['balance' => 10]) + ->and($cache->lookup('balances', 'k2'))->toBeNull() + ->and($cache->lookup('other', 'k1'))->toBeNull(); +})->with('w4 query cache stores'); + +it('evicts one key without touching its neighbours', function (string $store) { + $cache = new CacheQueryCache(w4QueryCacheRepository($store)); + $cache->store('balances', 'k1', 1, 60); + $cache->store('balances', 'k2', 2, 60); + + $cache->evictKey('balances', 'k1'); + + expect($cache->lookup('balances', 'k1'))->toBeNull() + ->and($cache->lookup('balances', 'k2'))->toBe(2); +})->with('w4 query cache stores'); + +it('evicts a whole group by starting a new generation, and leaves other groups alone', function (string $store) { + $cache = new CacheQueryCache(w4QueryCacheRepository($store)); + $cache->store('balances', 'k1', 1, 60); + $cache->store('balances', 'k2', 2, 60); + $cache->store('totals', 'all', 3, 60); + + $cache->evictGroup('balances'); + + expect($cache->lookup('balances', 'k1'))->toBeNull() + ->and($cache->lookup('balances', 'k2'))->toBeNull() + ->and($cache->lookup('totals', 'all'))->toBe(3); + + $cache->store('balances', 'k1', 10, 60); + + expect($cache->lookup('balances', 'k1'))->toBe(10); +})->with('w4 query cache stores'); + +it('expires an entry after its TTL', function (string $store) { + Carbon::setTestNow('2026-10-12 10:00:00'); + $cache = new CacheQueryCache(w4QueryCacheRepository($store)); + $cache->store('balances', 'k1', 1, 60); + + Carbon::setTestNow('2026-10-12 10:01:01'); + + expect($cache->lookup('balances', 'k1'))->toBeNull(); +})->with('w4 query cache stores'); + +it('never stores null, so null stays a miss', function () { + $cache = new CacheQueryCache($store = new Repository(new ArrayStore)); + $cache->store('balances', 'k1', null, 60); + + expect($cache->lookup('balances', 'k1'))->toBeNull() + ->and($store->getStore())->toBeInstanceOf(ArrayStore::class); +}); + +it('serves the legacy QueryCache contract from the default group', function () { + $cache = new CacheQueryCache(new Repository(new ArrayStore)); + $cache->put('accounts:alice', 3, null); + + expect($cache->get('accounts:alice'))->toBe(3) + ->and($cache->lookup('', 'accounts:alice'))->toBe(3); + + $cache->evict('accounts:alice'); + + expect($cache->get('accounts:alice'))->toBeNull(); +}); + +it('logs a failing store and answers a miss instead of failing the query', function () { + $broken = new class implements Store + { + public function get($key): mixed + { + throw new RuntimeException('cache down'); + } + + /** + * @param array $keys + * @return array + */ + public function many(array $keys): array + { + throw new RuntimeException('cache down'); + } + + public function put($key, $value, $seconds): bool + { + throw new RuntimeException('cache down'); + } + + /** @param array $values */ + public function putMany(array $values, $seconds): bool + { + throw new RuntimeException('cache down'); + } + + public function increment($key, $value = 1): int|bool + { + throw new RuntimeException('cache down'); + } + + public function decrement($key, $value = 1): int|bool + { + throw new RuntimeException('cache down'); + } + + public function forever($key, $value): bool + { + throw new RuntimeException('cache down'); + } + + public function touch($key, $seconds): bool + { + throw new RuntimeException('cache down'); + } + + public function forget($key): bool + { + throw new RuntimeException('cache down'); + } + + public function flush(): bool + { + return false; + } + + public function getPrefix(): string + { + return ''; + } + }; + $logger = new RecordingLogger; + $cache = new CacheQueryCache(new Repository($broken), logger: $logger); + + $cache->store('balances', 'k1', 1, 60); + $cache->evictGroup('balances'); + + expect($cache->lookup('balances', 'k1'))->toBeNull() + ->and(array_column($logger->records, 'level'))->toBe(['warning', 'warning', 'warning']) + ->and($logger->records[0]['message'])->toContain('cache down'); +}); diff --git a/packages/cqrs/tests/Support/RecordingLogger.php b/packages/cqrs/tests/Support/RecordingLogger.php new file mode 100644 index 00000000..49697b9d --- /dev/null +++ b/packages/cqrs/tests/Support/RecordingLogger.php @@ -0,0 +1,23 @@ + */ + public array $records = []; + + /** + * @param array $context + */ + public function log($level, string|Stringable $message, array $context = []): void + { + $this->records[] = ['level' => is_string($level) ? $level : 'unknown', 'message' => (string) $message]; + } +} From c08e43a23f5720c9659e2ad6aedc87578e4add8a Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 18:39:18 -0700 Subject: [PATCH 07/47] fix(cqrs): drop a query cache store that lands after its group's eviction instead of caching the stale result --- packages/cqrs/src/Cache/CacheQueryCache.php | 63 +++++++++++++------ .../cqrs/tests/Cache/CacheQueryCacheTest.php | 29 +++++++++ 2 files changed, 74 insertions(+), 18 deletions(-) diff --git a/packages/cqrs/src/Cache/CacheQueryCache.php b/packages/cqrs/src/Cache/CacheQueryCache.php index 69988d47..23061a8d 100644 --- a/packages/cqrs/src/Cache/CacheQueryCache.php +++ b/packages/cqrs/src/Cache/CacheQueryCache.php @@ -18,14 +18,21 @@ * g: => the group's current generation * q: => one cached result * - * evictGroup() forgets the generation; the next lookup starts a fresh random one, and no entry written under an - * older generation is reachable again — they expire on their own TTL (which is why the module guide tells you to - * set one). That also settles the classic read/evict race in the group's favour: a handler that read the database - * before a commit stores its result under the generation it looked up, which the commit's eviction has just made - * unreachable. A single-key eviction (evictKey) does not have that property; it is the cheaper, narrower tool. + * evictGroup() replaces the generation with an eviction marker; the next lookup starts a fresh random one, and no + * entry written under an older generation is reachable again — they expire on their own TTL (which is why the module + * guide tells you to set one). The marker is what narrows the classic read/evict race: a handler that looked the + * group up before a commit and stores after the commit's eviction finds the marker, not a generation, and its store + * is dropped, so the stale result it read is never cached. Only store() is held back by the marker; a store into a + * group that never had a generation (a put() with no get() before it) starts one, as it always has. What remains is + * the window in which ANOTHER lookup starts a new generation between the eviction and the late store: the store + * then writes its stale result under the new generation. Closing that fully needs a token handed from lookup() to + * store(), which the QueryCache contract does not carry. A single-key eviction (evictKey) has none of this; it is + * the cheaper, narrower tool. * - * A generation is created with add() and a fixed TTL: Laravel's Repository::add() is only atomic when it is given a - * TTL, and two processes starting a generation at once must agree on one. Keys are hashed, so a group or key of any + * A first generation is created with add() and a fixed TTL: Laravel's Repository::add() is only atomic when it is + * given a TTL, and two processes starting a generation at once must agree on one. A generation that replaces an + * eviction marker is put() and then read back, so concurrent lookups settle on whichever write landed last; a + * lookup that lost that race only costs a miss. Keys are hashed, so a group or key of any * length fits every store's key limit (memcached: 250 bytes; the database store: 255). * * A CACHE FAILURE NEVER FAILS A QUERY: every operation catches, logs a warning and answers a miss — the handler @@ -39,6 +46,15 @@ final class CacheQueryCache implements GroupedQueryCache /** Thirty days: long enough to outlive any entry TTL worth using; a generation that expires only costs misses. */ public const int GENERATION_TTL = 2_592_000; + /** The value evictGroup() leaves in place of a generation; never a valid generation (those are 16 hex digits). */ + private const string EVICTED = '-'; + + private const int EXISTING = 0; + + private const int FOR_STORE = 1; + + private const int FOR_LOOKUP = 2; + public function __construct( private readonly Repository $cache, private readonly string $prefix = self::DEFAULT_PREFIX, @@ -63,7 +79,7 @@ public function evict(string $key): void public function lookup(string $group, string $key): mixed { try { - $generation = $this->generation($group, true); + $generation = $this->generation($group, self::FOR_LOOKUP); return $generation === null ? null : $this->cache->get($this->entryKey($group, $generation, $key)); } catch (Throwable $e) { @@ -80,7 +96,7 @@ public function store(string $group, string $key, mixed $value, ?int $ttl): void } try { - $generation = $this->generation($group, true); + $generation = $this->generation($group, self::FOR_STORE); if ($generation === null) { return; } @@ -99,7 +115,7 @@ public function store(string $group, string $key, mixed $value, ?int $ttl): void public function evictKey(string $group, string $key): void { try { - $generation = $this->generation($group, false); + $generation = $this->generation($group, self::EXISTING); if ($generation !== null) { $this->cache->forget($this->entryKey($group, $generation, $key)); } @@ -111,34 +127,45 @@ public function evictKey(string $group, string $key): void public function evictGroup(string $group): void { try { - $this->cache->forget($this->generationKey($group)); + $this->cache->put($this->generationKey($group), self::EVICTED, self::GENERATION_TTL); } catch (Throwable $e) { $this->report('evictGroup', $group, $e); } } - /** The group's generation; with $create, started now when it has none. null when none could be started. */ - private function generation(string $group, bool $create): ?string + /** + * The group's current generation, or null when there is none to use. + * + * EXISTING never starts one. FOR_STORE starts one only for a group that never had one, and answers null while + * an eviction marker stands. FOR_LOOKUP starts one in either case. + * + * @param self::EXISTING|self::FOR_STORE|self::FOR_LOOKUP $mode + */ + private function generation(string $group, int $mode): ?string { $key = $this->generationKey($group); $current = $this->cache->get($key); - if (is_string($current) && $current !== '') { + if (is_string($current) && $current !== '' && $current !== self::EVICTED) { return $current; } - if (! $create) { + $evicted = $current === self::EVICTED; + if ($mode === self::EXISTING || ($evicted && $mode === self::FOR_STORE)) { return null; } $fresh = bin2hex(random_bytes(8)); - if ($this->cache->add($key, $fresh, self::GENERATION_TTL)) { + if ($evicted) { + $this->cache->put($key, $fresh, self::GENERATION_TTL); + } elseif ($this->cache->add($key, $fresh, self::GENERATION_TTL)) { return $fresh; } - // Another process started one in between — use it; anything else means the store refused the write. + // Read back what landed: another process may have started a generation in between, or evicted again, or the + // store refused the write. Only a real generation is usable. $current = $this->cache->get($key); - return is_string($current) && $current !== '' ? $current : null; + return is_string($current) && $current !== '' && $current !== self::EVICTED ? $current : null; } private function generationKey(string $group): string diff --git a/packages/cqrs/tests/Cache/CacheQueryCacheTest.php b/packages/cqrs/tests/Cache/CacheQueryCacheTest.php index cf964e04..72a3a0b9 100644 --- a/packages/cqrs/tests/Cache/CacheQueryCacheTest.php +++ b/packages/cqrs/tests/Cache/CacheQueryCacheTest.php @@ -87,6 +87,35 @@ function w4QueryCacheRepository(string $store): Repository expect($cache->lookup('balances', 'k1'))->toBe(10); })->with('w4 query cache stores'); +it('drops a store that looked the group up before an eviction, so the stale result is never cached', function (string $store) { + $cache = new CacheQueryCache(w4QueryCacheRepository($store)); + + // A handler misses and reads the database; a commit evicts the group; the handler then stores what it read. + expect($cache->lookup('balances', 'k1'))->toBeNull(); + $cache->evictGroup('balances'); + $cache->store('balances', 'k1', 'STALE', 60); + + expect($cache->lookup('balances', 'k1'))->toBeNull(); + + // The lookup above started a fresh generation, so the next miss-then-store caches normally again. + $cache->store('balances', 'k1', 'FRESH', 60); + + expect($cache->lookup('balances', 'k1'))->toBe('FRESH'); +})->with('w4 query cache stores'); + +it('keeps a group evicted across stores until a lookup starts its next generation', function (string $store) { + $cache = new CacheQueryCache(w4QueryCacheRepository($store)); + $cache->store('balances', 'k1', 1, 60); + $cache->evictGroup('balances'); + + $cache->store('balances', 'k1', 'STALE', 60); + $cache->store('balances', 'k2', 'STALE', 60); + $cache->evictKey('balances', 'k1'); + + expect($cache->lookup('balances', 'k1'))->toBeNull() + ->and($cache->lookup('balances', 'k2'))->toBeNull(); +})->with('w4 query cache stores'); + it('expires an entry after its TTL', function (string $store) { Carbon::setTestNow('2026-10-12 10:00:00'); $cache = new CacheQueryCache(w4QueryCacheRepository($store)); From e64e3faef738aa1918d199e79ef5f414e90da104 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 18:41:19 -0700 Subject: [PATCH 08/47] fix(cqrs): bound a no-TTL query cache entry by its generation TTL so evicted groups leave no forever orphans --- packages/cqrs/src/Cache/CacheQueryCache.php | 17 ++++++++++------- packages/cqrs/src/Cache/GroupedQueryCache.php | 5 ++++- .../cqrs/tests/Cache/CacheQueryCacheTest.php | 19 +++++++++++++++++++ 3 files changed, 33 insertions(+), 8 deletions(-) diff --git a/packages/cqrs/src/Cache/CacheQueryCache.php b/packages/cqrs/src/Cache/CacheQueryCache.php index 23061a8d..0fb63817 100644 --- a/packages/cqrs/src/Cache/CacheQueryCache.php +++ b/packages/cqrs/src/Cache/CacheQueryCache.php @@ -20,7 +20,9 @@ * * evictGroup() replaces the generation with an eviction marker; the next lookup starts a fresh random one, and no * entry written under an older generation is reachable again — they expire on their own TTL (which is why the module - * guide tells you to set one). The marker is what narrows the classic read/evict race: a handler that looked the + * guide tells you to set one). An entry stored with no TTL is given GENERATION_TTL instead of living forever: it can + * never outlive the generation that makes it reachable, and forever entries orphaned by an eviction would pile up + * without bound on the file and database stores, which have no LRU to reclaim them. The marker is what narrows the classic read/evict race: a handler that looked the * group up before a commit and stores after the commit's eviction finds the marker, not a generation, and its store * is dropped, so the stale result it read is never cached. Only store() is held back by the marker; a store into a * group that never had a generation (a put() with no get() before it) starts one, as it always has. What remains is @@ -101,12 +103,13 @@ public function store(string $group, string $key, mixed $value, ?int $ttl): void return; } - $entry = $this->entryKey($group, $generation, $key); - if ($ttl === null) { - $this->cache->forever($entry, $value); - } else { - $this->cache->put($entry, $value, max(1, $ttl)); - } + // "No expiry" is bounded by the generation: an entry is unreachable once its generation is evicted or + // lapses, so a forever() entry would only be an orphan the file and database stores never reclaim. + $this->cache->put( + $this->entryKey($group, $generation, $key), + $value, + $ttl === null ? self::GENERATION_TTL : max(1, $ttl), + ); } catch (Throwable $e) { $this->report('store', $group, $e); } diff --git a/packages/cqrs/src/Cache/GroupedQueryCache.php b/packages/cqrs/src/Cache/GroupedQueryCache.php index e2122a6d..bea994a2 100644 --- a/packages/cqrs/src/Cache/GroupedQueryCache.php +++ b/packages/cqrs/src/Cache/GroupedQueryCache.php @@ -15,7 +15,10 @@ interface GroupedQueryCache extends QueryCache /** The cached value, or null on a miss. */ public function lookup(string $group, string $key): mixed; - /** Stores a non-null value for $ttl seconds (null = no expiry). */ + /** + * Stores a non-null value for $ttl seconds. null means no TTL of its own: the entry lives until its group is + * evicted, or for as long as the implementation keeps the group itself (CacheQueryCache: GENERATION_TTL). + */ public function store(string $group, string $key, mixed $value, ?int $ttl): void; public function evictKey(string $group, string $key): void; diff --git a/packages/cqrs/tests/Cache/CacheQueryCacheTest.php b/packages/cqrs/tests/Cache/CacheQueryCacheTest.php index 72a3a0b9..9a1d54ec 100644 --- a/packages/cqrs/tests/Cache/CacheQueryCacheTest.php +++ b/packages/cqrs/tests/Cache/CacheQueryCacheTest.php @@ -126,6 +126,25 @@ function w4QueryCacheRepository(string $store): Repository expect($cache->lookup('balances', 'k1'))->toBeNull(); })->with('w4 query cache stores'); +it('bounds an entry stored with no TTL by the generation TTL, so an evicted group leaves no forever orphans', function (string $store) { + Carbon::setTestNow('2026-10-12 10:00:00'); + $repository = w4QueryCacheRepository($store); + $cache = new CacheQueryCache($repository); + $cache->store('balances', 'k1', 1, null); + + $generation = $repository->get(CacheQueryCache::DEFAULT_PREFIX.'g:'.hash('sha256', 'balances')); + expect($generation)->toBeString(); + $entry = CacheQueryCache::DEFAULT_PREFIX.'q:'.hash('sha256', "balances\x1f".(is_string($generation) ? $generation : '')."\x1fk1"); + + // The eviction makes the entry unreachable; the raw row must still lapse instead of living forever. + $cache->evictGroup('balances'); + Carbon::setTestNow(Carbon::parse('2026-10-12 10:00:00')->addSeconds(CacheQueryCache::GENERATION_TTL - 1)); + expect($repository->get($entry))->toBe(1); + + Carbon::setTestNow(Carbon::parse('2026-10-12 10:00:00')->addSeconds(CacheQueryCache::GENERATION_TTL + 1)); + expect($repository->get($entry))->toBeNull(); +})->with('w4 query cache stores'); + it('never stores null, so null stays a miss', function () { $cache = new CacheQueryCache($store = new Repository(new ArrayStore)); $cache->store('balances', 'k1', null, 60); From a46fd68af514f31f0878c7f64d4b3925674d4c87 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 18:47:40 -0700 Subject: [PATCH 09/47] feat(cqrs): cache #[CachedQuery] results with per-query TTLs behind firefly.cqrs.query.cache.provider --- book/src-es/07-cqrs.md | 21 ++- book/src/07-cqrs.md | 21 ++- .../cqrs/cache/firefly-cqrs-components.php | 3 + .../cqrs/src/Cache/QueryCachePolicies.php | 35 +++++ packages/cqrs/src/CqrsAutoConfiguration.php | 47 ++++++- packages/cqrs/src/Query/DefaultQueryBus.php | 31 ++++- .../tests/Cache/QueryCacheProviderTest.php | 41 ++++++ .../CapstoneCqrsQueryCacheDisabledTest.php | 20 +++ .../cqrs/tests/CapstoneCqrsQueryCacheTest.php | 60 +++++++++ .../Ledger/AccountStatement.php | 21 +++ .../Ledger/AccountStatementHandler.php | 24 ++++ .../tests/CapstoneFixtures/Ledger/Entry.php | 35 +++++ .../CapstoneFixtures/Ledger/EntryPosted.php | 21 +++ .../Ledger/EntryRepository.php | 17 +++ .../CapstoneFixtures/Ledger/HandlerCalls.php | 27 ++++ .../CapstoneFixtures/Ledger/LedgerBalance.php | 14 ++ .../Ledger/LedgerBalanceHandler.php | 18 +++ .../Ledger/LedgerCqrsConfiguration.php | 25 ++++ .../CapstoneFixtures/Ledger/LedgerTotals.php | 11 ++ .../Ledger/LedgerTotalsHandler.php | 18 +++ .../CapstoneFixtures/Ledger/PostEntry.php | 12 ++ .../Ledger/PostEntryHandler.php | 24 ++++ .../CapstoneFixtures/Ledger/RejectEntry.php | 12 ++ .../Ledger/RejectEntryHandler.php | 26 ++++ .../tests/Query/QueryBusCachePolicyTest.php | 82 ++++++++++++ .../Support/CqrsCacheCapstoneTestCase.php | 126 ++++++++++++++++++ .../Support/CqrsNoCacheCapstoneTestCase.php | 14 ++ .../Support/LedgerComponentsProvider.php | 41 ++++++ 28 files changed, 829 insertions(+), 18 deletions(-) create mode 100644 packages/cqrs/src/Cache/QueryCachePolicies.php create mode 100644 packages/cqrs/tests/Cache/QueryCacheProviderTest.php create mode 100644 packages/cqrs/tests/CapstoneCqrsQueryCacheDisabledTest.php create mode 100644 packages/cqrs/tests/CapstoneCqrsQueryCacheTest.php create mode 100644 packages/cqrs/tests/CapstoneFixtures/Ledger/AccountStatement.php create mode 100644 packages/cqrs/tests/CapstoneFixtures/Ledger/AccountStatementHandler.php create mode 100644 packages/cqrs/tests/CapstoneFixtures/Ledger/Entry.php create mode 100644 packages/cqrs/tests/CapstoneFixtures/Ledger/EntryPosted.php create mode 100644 packages/cqrs/tests/CapstoneFixtures/Ledger/EntryRepository.php create mode 100644 packages/cqrs/tests/CapstoneFixtures/Ledger/HandlerCalls.php create mode 100644 packages/cqrs/tests/CapstoneFixtures/Ledger/LedgerBalance.php create mode 100644 packages/cqrs/tests/CapstoneFixtures/Ledger/LedgerBalanceHandler.php create mode 100644 packages/cqrs/tests/CapstoneFixtures/Ledger/LedgerCqrsConfiguration.php create mode 100644 packages/cqrs/tests/CapstoneFixtures/Ledger/LedgerTotals.php create mode 100644 packages/cqrs/tests/CapstoneFixtures/Ledger/LedgerTotalsHandler.php create mode 100644 packages/cqrs/tests/CapstoneFixtures/Ledger/PostEntry.php create mode 100644 packages/cqrs/tests/CapstoneFixtures/Ledger/PostEntryHandler.php create mode 100644 packages/cqrs/tests/CapstoneFixtures/Ledger/RejectEntry.php create mode 100644 packages/cqrs/tests/CapstoneFixtures/Ledger/RejectEntryHandler.php create mode 100644 packages/cqrs/tests/Query/QueryBusCachePolicyTest.php create mode 100644 packages/cqrs/tests/Support/CqrsCacheCapstoneTestCase.php create mode 100644 packages/cqrs/tests/Support/CqrsNoCacheCapstoneTestCase.php create mode 100644 packages/cqrs/tests/Support/LedgerComponentsProvider.php diff --git a/book/src-es/07-cqrs.md b/book/src-es/07-cqrs.md index 119f2f9e..c12fff57 100644 --- a/book/src-es/07-cqrs.md +++ b/book/src-es/07-cqrs.md @@ -328,10 +328,19 @@ final class DefaultQueryBus implements QueryBus $this->validator->validate($query); $this->authorizer->authorize($query); - $key = $query instanceof Cacheable ? $query->cacheKey() : null; + $policy = $this->policies?->policyFor($query::class); + $key = match (true) { + $query instanceof Cacheable => $query->cacheKey(), + $policy !== null => QueryCacheKey::of($query), + default => null, + }; + $group = $policy !== null ? $policy->group : ''; + $ttl = $policy->ttl ?? $this->cacheTtl; if ($key !== null) { - $cached = $this->cache->get($key); + $cached = $this->cache instanceof GroupedQueryCache + ? $this->cache->lookup($group, $key) + : $this->cache->get($key); if ($cached !== null) { return $cached; } @@ -341,7 +350,11 @@ final class DefaultQueryBus implements QueryBus $result = $handler($query); if ($key !== null) { - $this->cache->put($key, $result, $this->cacheTtl); + if ($this->cache instanceof GroupedQueryCache) { + $this->cache->store($group, $key, $result, $ttl); + } else { + $this->cache->put($key, $result, $ttl); + } } // … $this->metrics->recordQuerySuccess($query, microtime(true) - $startedAt); @@ -366,7 +379,7 @@ Todos los colaboradores excepto el propio `HandlerRegistry` son, por defecto, un | Autorización | `AllowAllAuthorizer` — permite todo | `SecurityCommandAuthorizer`/`SecurityQueryAuthorizer` de `firefly/security` (Capítulo 10) | | Correlación | Siempre activa — un id uuid4 por despacho, restaurado después | Nada que activar; se estampa en `x-correlation-id` cuando un evento de dominio llega al bus de eda (Capítulo 8) | | Métricas | `NoOpCqrsMetrics` | `firefly/observability` (un capítulo posterior) | -| Caché de consultas | `NoOpQueryCache` — siempre falla | Un futuro adaptador de `firefly/cache` | +| Caché de consultas | `NoOpQueryCache` — siempre falla | `firefly.cqrs.query.cache.provider=cache` (`CacheQueryCache` sobre un almacén de caché de Laravel) para las consultas `Cacheable` y `#[CachedQuery]` | Ninguno de los dos buses abre una transacción, y ninguno de los dos drena eventos de dominio. `#[Transactional]` sobre `OpenWalletHandler::handle()` de arriba hace ese trabajo enteramente por su cuenta, a través del proxy generado que el Capítulo 9 explica en detalle — el bus es completamente ajeno a él, y se comportaría idénticamente si `handle()` fuera un método plano, sin proxy, que llamara a `DB::beginTransaction()` a mano. diff --git a/book/src/07-cqrs.md b/book/src/07-cqrs.md index 957f321d..7ef84b6a 100644 --- a/book/src/07-cqrs.md +++ b/book/src/07-cqrs.md @@ -328,10 +328,19 @@ final class DefaultQueryBus implements QueryBus $this->validator->validate($query); $this->authorizer->authorize($query); - $key = $query instanceof Cacheable ? $query->cacheKey() : null; + $policy = $this->policies?->policyFor($query::class); + $key = match (true) { + $query instanceof Cacheable => $query->cacheKey(), + $policy !== null => QueryCacheKey::of($query), + default => null, + }; + $group = $policy !== null ? $policy->group : ''; + $ttl = $policy->ttl ?? $this->cacheTtl; if ($key !== null) { - $cached = $this->cache->get($key); + $cached = $this->cache instanceof GroupedQueryCache + ? $this->cache->lookup($group, $key) + : $this->cache->get($key); if ($cached !== null) { return $cached; } @@ -341,7 +350,11 @@ final class DefaultQueryBus implements QueryBus $result = $handler($query); if ($key !== null) { - $this->cache->put($key, $result, $this->cacheTtl); + if ($this->cache instanceof GroupedQueryCache) { + $this->cache->store($group, $key, $result, $ttl); + } else { + $this->cache->put($key, $result, $ttl); + } } // … $this->metrics->recordQuerySuccess($query, microtime(true) - $startedAt); @@ -366,7 +379,7 @@ Every collaborator except `HandlerRegistry` itself defaults to a **no-op**, so t | Authorization | `AllowAllAuthorizer` — allow everything | `firefly/security`'s `SecurityCommandAuthorizer`/`SecurityQueryAuthorizer` (Chapter 10) | | Correlation | Always on — a uuid4 id per dispatch, restored after | Nothing to enable; stamped into `x-correlation-id` when a domain event reaches the eda bus (Chapter 8) | | Metrics | `NoOpCqrsMetrics` | `firefly/observability` (a later chapter) | -| Query cache | `NoOpQueryCache` — always a miss | A future `firefly/cache` adapter | +| Query cache | `NoOpQueryCache` — always a miss | `firefly.cqrs.query.cache.provider=cache` (`CacheQueryCache` on a Laravel cache store) for `Cacheable` and `#[CachedQuery]` queries | Neither bus opens a transaction, and neither bus drains domain events. `#[Transactional]` on `OpenWalletHandler::handle()` above is doing that work entirely on its own, through the generated proxy Chapter 9 explains in full — the bus is completely oblivious to it, and would behave identically if `handle()` were a plain, unproxied method that happened to call `DB::beginTransaction()` by hand. diff --git a/packages/cqrs/cache/firefly-cqrs-components.php b/packages/cqrs/cache/firefly-cqrs-components.php index 58fd4d64..49df8aea 100644 --- a/packages/cqrs/cache/firefly-cqrs-components.php +++ b/packages/cqrs/cache/firefly-cqrs-components.php @@ -103,6 +103,8 @@ 'order' => 0, 'lazy' => false, 'dependencies' => [ + 0 => 'Firefly\\Config\\Config', + 1 => 'Illuminate\\Container\\Container', ], ], 8 => [ @@ -168,6 +170,7 @@ 5 => 'Firefly\\Cqrs\\Cache\\QueryCache', 6 => 'Firefly\\Config\\Config', 7 => 'Firefly\\Cqrs\\Tracing\\CqrsTracing', + 8 => 'Firefly\\Cqrs\\Handler\\HandlerManifest', ], ], ], diff --git a/packages/cqrs/src/Cache/QueryCachePolicies.php b/packages/cqrs/src/Cache/QueryCachePolicies.php new file mode 100644 index 00000000..17c25ff0 --- /dev/null +++ b/packages/cqrs/src/Cache/QueryCachePolicies.php @@ -0,0 +1,35 @@ + $policies + */ + public function __construct(private readonly array $policies = []) {} + + public static function fromManifest(HandlerManifest $manifest): self + { + return new self($manifest->cachedQueries()); + } + + public function policyFor(string $queryClass): ?CachedQueryPolicy + { + return $this->policies[$queryClass] ?? null; + } + + public function groupFor(string $queryClass): string + { + return $this->policyFor($queryClass)->group ?? ''; + } +} diff --git a/packages/cqrs/src/CqrsAutoConfiguration.php b/packages/cqrs/src/CqrsAutoConfiguration.php index 599b61a5..b63ec19a 100644 --- a/packages/cqrs/src/CqrsAutoConfiguration.php +++ b/packages/cqrs/src/CqrsAutoConfiguration.php @@ -9,8 +9,10 @@ use Firefly\Container\Attributes\Configuration; use Firefly\Container\Attributes\Order; use Firefly\Context\Condition\Attributes\ConditionalOnMissingBean; +use Firefly\Cqrs\Cache\CacheQueryCache; use Firefly\Cqrs\Cache\NoOpQueryCache; use Firefly\Cqrs\Cache\QueryCache; +use Firefly\Cqrs\Cache\QueryCachePolicies; use Firefly\Cqrs\Command\CommandBus; use Firefly\Cqrs\Command\DefaultCommandBus; use Firefly\Cqrs\Correlation\CorrelationContext; @@ -19,6 +21,7 @@ use Firefly\Cqrs\Event\EdaCommandEventPublisher; use Firefly\Cqrs\Event\EventFailureStrategy; use Firefly\Cqrs\Event\NoOpEventPublisher; +use Firefly\Cqrs\Exception\CqrsConfigurationException; use Firefly\Cqrs\Handler\HandlerManifest; use Firefly\Cqrs\Handler\HandlerRegistry; use Firefly\Cqrs\Metrics\CqrsMetrics; @@ -34,6 +37,7 @@ use Firefly\Eda\EventPublisher; use Firefly\Validation\Validator; use Illuminate\Container\Container; +use Illuminate\Contracts\Cache\Factory as CacheFactory; use Psr\Log\LoggerInterface; /** @@ -110,11 +114,34 @@ public function cqrsTracing(): CqrsTracing return new NoOpCqrsTracing; } + /** + * firefly.cqrs.query.cache.provider: `none` (the default — the inert NoOpQueryCache) or `cache`, the + * CacheQueryCache on the Laravel cache store firefly.cqrs.query.cache.store names ('' = the default store), + * under the key prefix firefly.cqrs.query.cache.prefix. Anything else refuses the boot rather than silently + * running uncached. An application that binds its own QueryCache wins, as before. + */ #[Bean] #[ConditionalOnMissingBean(QueryCache::class)] - public function queryCache(): QueryCache + public function queryCache(Config $config, Container $container): QueryCache { - return new NoOpQueryCache; + $provider = $config->string('firefly.cqrs.query.cache.provider', 'none'); + if ($provider === 'none') { + return new NoOpQueryCache; + } + + if ($provider !== 'cache') { + throw new CqrsConfigurationException("firefly.cqrs.query.cache.provider must be 'none' or 'cache', got '{$provider}'."); + } + + /** @var CacheFactory $caches */ + $caches = $container->make('cache'); + $store = $config->string('firefly.cqrs.query.cache.store', ''); + + return new CacheQueryCache( + $caches->store($store === '' ? null : $store), + $config->string('firefly.cqrs.query.cache.prefix', CacheQueryCache::DEFAULT_PREFIX), + $this->logger($container), + ); } #[Bean] @@ -141,7 +168,7 @@ public function commandEventPublisher(Container $container, Config $config, Hand public function domainEventBridge(CommandEventPublisher $publisher, Config $config, Container $container): DomainEventBridge { $strategy = EventFailureStrategy::tryFrom($config->string('firefly.cqrs.event_failure_strategy', 'log')) ?? EventFailureStrategy::Log; - $logger = $container->bound(LoggerInterface::class) ? $container->make(LoggerInterface::class) : null; + $logger = $this->logger($container); return new DomainEventBridge($publisher, $strategy, $logger); } @@ -155,10 +182,20 @@ public function commandBus(HandlerRegistry $registry, MessageValidator $validato #[Bean] #[ConditionalOnMissingBean(QueryBus::class)] - public function queryBus(HandlerRegistry $registry, MessageValidator $validator, QueryAuthorizer $authorizer, CorrelationContext $correlation, CqrsMetrics $metrics, QueryCache $cache, Config $config, ?CqrsTracing $tracing = null): QueryBus + public function queryBus(HandlerRegistry $registry, MessageValidator $validator, QueryAuthorizer $authorizer, CorrelationContext $correlation, CqrsMetrics $metrics, QueryCache $cache, Config $config, ?CqrsTracing $tracing = null, ?HandlerManifest $manifest = null): QueryBus { $ttl = $config->has('firefly.cqrs.query.cache_ttl') ? $config->int('firefly.cqrs.query.cache_ttl') : null; + $policies = $manifest !== null ? QueryCachePolicies::fromManifest($manifest) : null; + + return new DefaultQueryBus($registry, $validator, $authorizer, $correlation, $metrics, $cache, $ttl, $tracing, $policies); + } + + private function logger(Container $container): ?LoggerInterface + { + if (! $container->bound(LoggerInterface::class)) { + return null; + } - return new DefaultQueryBus($registry, $validator, $authorizer, $correlation, $metrics, $cache, $ttl, $tracing); + return $container->make(LoggerInterface::class); } } diff --git a/packages/cqrs/src/Query/DefaultQueryBus.php b/packages/cqrs/src/Query/DefaultQueryBus.php index 27df0d05..ca9d3b2a 100644 --- a/packages/cqrs/src/Query/DefaultQueryBus.php +++ b/packages/cqrs/src/Query/DefaultQueryBus.php @@ -5,7 +5,10 @@ namespace Firefly\Cqrs\Query; use Firefly\Cqrs\Cache\Cacheable; +use Firefly\Cqrs\Cache\GroupedQueryCache; use Firefly\Cqrs\Cache\QueryCache; +use Firefly\Cqrs\Cache\QueryCacheKey; +use Firefly\Cqrs\Cache\QueryCachePolicies; use Firefly\Cqrs\Correlation\CorrelationContext; use Firefly\Cqrs\Exception\QueryProcessingException; use Firefly\Cqrs\Handler\HandlerRegistry; @@ -19,8 +22,12 @@ /** * The synchronous in-process query mediator. ask() runs: correlate -> validate -> authorize -> cache-get (a Cacheable * query yielding a non-null key is looked up; a non-null hit short-circuits straight to metrics + return) -> resolve - * the single registered handler and invoke it -> cache-put -> metrics. The QueryCache default is NoOpQueryCache - * (always-miss), so caching is an inert seam until firefly/cache lands. Any handler/stage throwable is metrics-recorded + * the single registered handler and invoke it -> cache-put -> metrics. The key is the query's own cacheKey() when it + * is Cacheable, else the key QueryCacheKey derives from the value object when the query carries #[CachedQuery], else + * there is no caching. A GroupedQueryCache is addressed by the query's group and its policy TTL (falling back to + * firefly.cqrs.query.cache_ttl); a plain QueryCache keeps receiving get()/put() with the flat key, so an + * application's own cache keeps working unchanged. The default is still NoOpQueryCache — + * firefly.cqrs.query.cache.provider=cache binds CacheQueryCache. Any handler/stage throwable is metrics-recorded * and re-thrown WRAPPED in a category-preserving QueryProcessingException (unless already one). Correlation is begun * and restored in finally, exactly like the command bus. The whole validate → authorize → cache-get → resolve → * invoke → cache-put sequence runs inside the CqrsTracing seam (an INTERNAL span when observability is on), so a @@ -40,6 +47,7 @@ public function __construct( private readonly QueryCache $cache, private readonly ?int $cacheTtl = null, ?CqrsTracing $tracing = null, + private readonly ?QueryCachePolicies $policies = null, ) { $this->tracing = $tracing ?? new NoOpCqrsTracing; } @@ -54,10 +62,19 @@ public function ask(object $query): mixed $this->validator->validate($query); $this->authorizer->authorize($query); - $key = $query instanceof Cacheable ? $query->cacheKey() : null; + $policy = $this->policies?->policyFor($query::class); + $key = match (true) { + $query instanceof Cacheable => $query->cacheKey(), + $policy !== null => QueryCacheKey::of($query), + default => null, + }; + $group = $policy !== null ? $policy->group : ''; + $ttl = $policy->ttl ?? $this->cacheTtl; if ($key !== null) { - $cached = $this->cache->get($key); + $cached = $this->cache instanceof GroupedQueryCache + ? $this->cache->lookup($group, $key) + : $this->cache->get($key); if ($cached !== null) { return $cached; } @@ -67,7 +84,11 @@ public function ask(object $query): mixed $result = $handler($query); if ($key !== null) { - $this->cache->put($key, $result, $this->cacheTtl); + if ($this->cache instanceof GroupedQueryCache) { + $this->cache->store($group, $key, $result, $ttl); + } else { + $this->cache->put($key, $result, $ttl); + } } return $result; diff --git a/packages/cqrs/tests/Cache/QueryCacheProviderTest.php b/packages/cqrs/tests/Cache/QueryCacheProviderTest.php new file mode 100644 index 00000000..fad867be --- /dev/null +++ b/packages/cqrs/tests/Cache/QueryCacheProviderTest.php @@ -0,0 +1,41 @@ +instance('config', new Repository(['cache' => ['default' => 'array', 'stores' => ['array' => ['driver' => 'array'], 'second' => ['driver' => 'array']]]])); + $container->instance('cache', new CacheManager($container)); + + return $container; +} + +it('defaults to the inert NoOp cache', function () { + $cache = (new CqrsAutoConfiguration)->queryCache(new Config(new Repository(['firefly' => []])), w4QueryCacheContainer()); + + expect($cache)->toBeInstanceOf(NoOpQueryCache::class); +}); + +it('builds the Laravel-cache adapter on the named store for provider=cache', function () { + $config = new Config(new Repository(['firefly' => ['cqrs' => ['query' => ['cache' => ['provider' => 'cache', 'store' => 'second']]]]])); + + expect((new CqrsAutoConfiguration)->queryCache($config, w4QueryCacheContainer()))->toBeInstanceOf(CacheQueryCache::class); +}); + +it('refuses an unknown provider at boot, naming the key', function () { + $config = new Config(new Repository(['firefly' => ['cqrs' => ['query' => ['cache' => ['provider' => 'redis']]]]])); + + expect(fn () => (new CqrsAutoConfiguration)->queryCache($config, w4QueryCacheContainer())) + ->toThrow(CqrsConfigurationException::class, 'firefly.cqrs.query.cache.provider'); +}); diff --git a/packages/cqrs/tests/CapstoneCqrsQueryCacheDisabledTest.php b/packages/cqrs/tests/CapstoneCqrsQueryCacheDisabledTest.php new file mode 100644 index 00000000..274e98df --- /dev/null +++ b/packages/cqrs/tests/CapstoneCqrsQueryCacheDisabledTest.php @@ -0,0 +1,20 @@ +queryBus()->ask(new LedgerBalance('alice')); + $this->queryBus()->ask(new LedgerBalance('alice')); + + expect($this->app()->make(QueryCache::class))->toBeInstanceOf(NoOpQueryCache::class) + ->and(HandlerCalls::of('balance'))->toBe(2); +}); diff --git a/packages/cqrs/tests/CapstoneCqrsQueryCacheTest.php b/packages/cqrs/tests/CapstoneCqrsQueryCacheTest.php new file mode 100644 index 00000000..45273765 --- /dev/null +++ b/packages/cqrs/tests/CapstoneCqrsQueryCacheTest.php @@ -0,0 +1,60 @@ +app()->make(QueryCache::class))->toBeInstanceOf(CacheQueryCache::class); +}); + +it('answers the second ask of a #[CachedQuery] from the cache', function () { + /** @var CqrsCacheCapstoneTestCase $this */ + $this->commandBus()->send(new PostEntry('alice', 10)); + + expect($this->queryBus()->ask(new LedgerBalance('alice')))->toBe(10) + ->and($this->queryBus()->ask(new LedgerBalance('alice')))->toBe(10) + ->and(HandlerCalls::of('balance'))->toBe(1); +}); + +it('keys each value object separately', function () { + /** @var CqrsCacheCapstoneTestCase $this */ + $this->queryBus()->ask(new LedgerBalance('alice')); + $this->queryBus()->ask(new LedgerBalance('bob')); + $this->queryBus()->ask(new LedgerBalance('alice')); + + expect(HandlerCalls::of('balance'))->toBe(2); +}); + +it('honours the per-query TTL', function () { + /** @var CqrsCacheCapstoneTestCase $this */ + Carbon::setTestNow('2026-10-12 10:00:00'); + $this->queryBus()->ask(new LedgerBalance('alice')); + + Carbon::setTestNow('2026-10-12 10:00:59'); + $this->queryBus()->ask(new LedgerBalance('alice')); + expect(HandlerCalls::of('balance'))->toBe(1); + + Carbon::setTestNow('2026-10-12 10:01:01'); + $this->queryBus()->ask(new LedgerBalance('alice')); + expect(HandlerCalls::of('balance'))->toBe(2); +}); + +it('caches a Cacheable query under its own key', function () { + /** @var CqrsCacheCapstoneTestCase $this */ + $this->commandBus()->send(new PostEntry('alice', 4)); + + expect($this->queryBus()->ask(new AccountStatement('alice')))->toBe([4]) + ->and($this->queryBus()->ask(new AccountStatement('alice')))->toBe([4]) + ->and(HandlerCalls::of('statement'))->toBe(1); +}); diff --git a/packages/cqrs/tests/CapstoneFixtures/Ledger/AccountStatement.php b/packages/cqrs/tests/CapstoneFixtures/Ledger/AccountStatement.php new file mode 100644 index 00000000..ed15772d --- /dev/null +++ b/packages/cqrs/tests/CapstoneFixtures/Ledger/AccountStatement.php @@ -0,0 +1,21 @@ +account; + } +} diff --git a/packages/cqrs/tests/CapstoneFixtures/Ledger/AccountStatementHandler.php b/packages/cqrs/tests/CapstoneFixtures/Ledger/AccountStatementHandler.php new file mode 100644 index 00000000..1f7cf933 --- /dev/null +++ b/packages/cqrs/tests/CapstoneFixtures/Ledger/AccountStatementHandler.php @@ -0,0 +1,24 @@ + */ + public function handle(AccountStatement $query): array + { + HandlerCalls::hit('statement'); + + $amounts = []; + foreach (Entry::query()->where('account', $query->account)->orderBy('id')->get() as $entry) { + $amounts[] = $entry->amount; + } + + return $amounts; + } +} diff --git a/packages/cqrs/tests/CapstoneFixtures/Ledger/Entry.php b/packages/cqrs/tests/CapstoneFixtures/Ledger/Entry.php new file mode 100644 index 00000000..cc4b7448 --- /dev/null +++ b/packages/cqrs/tests/CapstoneFixtures/Ledger/Entry.php @@ -0,0 +1,35 @@ + 'integer']; + + public function post(): void + { + $this->raiseEvent(new EntryPosted((string) $this->account, (int) $this->amount)); + } +} diff --git a/packages/cqrs/tests/CapstoneFixtures/Ledger/EntryPosted.php b/packages/cqrs/tests/CapstoneFixtures/Ledger/EntryPosted.php new file mode 100644 index 00000000..71554936 --- /dev/null +++ b/packages/cqrs/tests/CapstoneFixtures/Ledger/EntryPosted.php @@ -0,0 +1,21 @@ + 'account'])] +#[InvalidatesQuery(LedgerTotals::class)] +#[InvalidatesQuery(AccountStatement::class, key: 'statement:{account}')] +final readonly class EntryPosted extends DomainEvent +{ + public function __construct(public string $account, public int $amount) + { + parent::__construct(); + } +} diff --git a/packages/cqrs/tests/CapstoneFixtures/Ledger/EntryRepository.php b/packages/cqrs/tests/CapstoneFixtures/Ledger/EntryRepository.php new file mode 100644 index 00000000..f43ca39c --- /dev/null +++ b/packages/cqrs/tests/CapstoneFixtures/Ledger/EntryRepository.php @@ -0,0 +1,17 @@ + + */ +#[Repository] +class EntryRepository extends EloquentRepository +{ + protected string $model = Entry::class; +} diff --git a/packages/cqrs/tests/CapstoneFixtures/Ledger/HandlerCalls.php b/packages/cqrs/tests/CapstoneFixtures/Ledger/HandlerCalls.php new file mode 100644 index 00000000..931d78fa --- /dev/null +++ b/packages/cqrs/tests/CapstoneFixtures/Ledger/HandlerCalls.php @@ -0,0 +1,27 @@ + */ + public static array $counts = []; + + public static function hit(string $handler): void + { + self::$counts[$handler] = (self::$counts[$handler] ?? 0) + 1; + } + + public static function of(string $handler): int + { + return self::$counts[$handler] ?? 0; + } + + public static function reset(): void + { + self::$counts = []; + } +} diff --git a/packages/cqrs/tests/CapstoneFixtures/Ledger/LedgerBalance.php b/packages/cqrs/tests/CapstoneFixtures/Ledger/LedgerBalance.php new file mode 100644 index 00000000..0c03ae36 --- /dev/null +++ b/packages/cqrs/tests/CapstoneFixtures/Ledger/LedgerBalance.php @@ -0,0 +1,14 @@ +where('account', $query->account)->sum('amount'); + } +} diff --git a/packages/cqrs/tests/CapstoneFixtures/Ledger/LedgerCqrsConfiguration.php b/packages/cqrs/tests/CapstoneFixtures/Ledger/LedgerCqrsConfiguration.php new file mode 100644 index 00000000..1af638cd --- /dev/null +++ b/packages/cqrs/tests/CapstoneFixtures/Ledger/LedgerCqrsConfiguration.php @@ -0,0 +1,25 @@ +sum('amount'); + } +} diff --git a/packages/cqrs/tests/CapstoneFixtures/Ledger/PostEntry.php b/packages/cqrs/tests/CapstoneFixtures/Ledger/PostEntry.php new file mode 100644 index 00000000..366233a9 --- /dev/null +++ b/packages/cqrs/tests/CapstoneFixtures/Ledger/PostEntry.php @@ -0,0 +1,12 @@ + $command->account, 'amount' => $command->amount]); + $entry->post(); + $this->entries->save($entry); + + return (int) $entry->id; + } +} diff --git a/packages/cqrs/tests/CapstoneFixtures/Ledger/RejectEntry.php b/packages/cqrs/tests/CapstoneFixtures/Ledger/RejectEntry.php new file mode 100644 index 00000000..4d5f4ffe --- /dev/null +++ b/packages/cqrs/tests/CapstoneFixtures/Ledger/RejectEntry.php @@ -0,0 +1,12 @@ + $command->account, 'amount' => $command->amount]); + $entry->post(); + $this->entries->save($entry); + + throw new RuntimeException('rejected after the write'); + } +} diff --git a/packages/cqrs/tests/Query/QueryBusCachePolicyTest.php b/packages/cqrs/tests/Query/QueryBusCachePolicyTest.php new file mode 100644 index 00000000..8d700ca3 --- /dev/null +++ b/packages/cqrs/tests/Query/QueryBusCachePolicyTest.php @@ -0,0 +1,82 @@ +registerQueryHandler(LedgerBalance::class, function (object $query) use (&$calls): int { + $calls++; + + return $query instanceof LedgerBalance ? strlen($query->account) : 0; + }); + $cache = new CacheQueryCache(new Repository(new ArrayStore)); + $bus = w4PolicyBus($registry, $cache, new QueryCachePolicies([LedgerBalance::class => new CachedQueryPolicy(LedgerBalance::class, 60, 'balances')])); + + expect($bus->ask(new LedgerBalance('alice')))->toBe(5) + ->and($bus->ask(new LedgerBalance('alice')))->toBe(5) + ->and($calls)->toBe(1) + ->and($cache->lookup('balances', (string) QueryCacheKey::of(new LedgerBalance('alice'))))->toBe(5); + + $cache->evictGroup('balances'); + + expect($bus->ask(new LedgerBalance('alice')))->toBe(5)->and($calls)->toBe(2); +}); + +it('does not cache a plain query without a policy, exactly as before', function () { + $registry = new HandlerRegistry; + $calls = 0; + $registry->registerQueryHandler(LedgerBalance::class, function () use (&$calls): int { + return ++$calls; + }); + $bus = w4PolicyBus($registry, new CacheQueryCache(new Repository(new ArrayStore)), new QueryCachePolicies); + + expect($bus->ask(new LedgerBalance('a')))->toBe(1)->and($bus->ask(new LedgerBalance('a')))->toBe(2); +}); + +it('keeps calling get()/put() with the flat key on a QueryCache that has no groups', function () { + $registry = new HandlerRegistry; + $registry->registerQueryHandler(LedgerBalance::class, static fn (): int => 7); + $legacy = new class implements QueryCache + { + /** @var list */ + public array $puts = []; + + public function get(string $key): mixed + { + return null; + } + + public function put(string $key, mixed $value, ?int $ttl): void + { + $this->puts[] = ['key' => $key, 'ttl' => $ttl]; + } + + public function evict(string $key): void {} + }; + $bus = w4PolicyBus($registry, $legacy, new QueryCachePolicies([LedgerBalance::class => new CachedQueryPolicy(LedgerBalance::class, null, LedgerBalance::class)]), 30); + + $bus->ask(new LedgerBalance('a')); + + expect($legacy->puts)->toBe([['key' => (string) QueryCacheKey::of(new LedgerBalance('a')), 'ttl' => 30]]); +}); diff --git a/packages/cqrs/tests/Support/CqrsCacheCapstoneTestCase.php b/packages/cqrs/tests/Support/CqrsCacheCapstoneTestCase.php new file mode 100644 index 00000000..92297a79 --- /dev/null +++ b/packages/cqrs/tests/Support/CqrsCacheCapstoneTestCase.php @@ -0,0 +1,126 @@ + */ + protected function fireflyProviders(): array + { + return [ + DataServiceProvider::class, + CqrsServiceProvider::class, + CqrsWiringProvider::class, + LedgerComponentsProvider::class, + ]; + } + + protected function cacheProvider(): string + { + return 'cache'; + } + + /** @return array */ + protected function configOverrides(): array + { + return [ + 'cache.default' => 'array', + 'firefly.cqrs.query.cache.provider' => $this->cacheProvider(), + 'firefly.cqrs.query.cache.invalidation' => ['ledger.closed' => [LedgerTotals::class]], + ]; + } + + protected function defineFireflyEnvironment(Application $app): void + { + $this->events = new InMemoryEventBus(new SubscriberRegistry); + $app->instance(EventPublisher::class, $this->events); + + $psr4 = ['Firefly\\Cqrs\\Tests\\CapstoneFixtures\\Ledger\\' => dirname(__DIR__).'/CapstoneFixtures/Ledger']; + + $scanner = new TransactionalScanner; + $generator = new ProxyClassGenerator; + foreach ($scanner->scanProxyMethods($psr4) as $target => $methods) { + $generator->load($target, $methods); + } + (new TransactionalManifestCompiler)->write($scanner->scan($psr4), LedgerCqrsConfiguration::manifestPath()); + + $handlers = (new HandlerScanner)->scan($psr4); + $app->instance(HandlerManifest::class, new HandlerManifest( + $handlers['handlers'], + $handlers['destinations'], + $handlers['cachedQueries'], + $handlers['invalidations'], + )); + } + + protected function setUp(): void + { + parent::setUp(); + + Schema::create('ledger_entries', function (Blueprint $table): void { + $table->increments('id'); + $table->string('account'); + $table->integer('amount'); + }); + + HandlerCalls::reset(); + } + + protected function tearDown(): void + { + Carbon::setTestNow(); + + parent::tearDown(); + } + + public function commandBus(): CommandBus + { + /** @var CommandBus $bus */ + $bus = $this->app()->make(ApplicationContext::class)->get(CommandBus::class); + + return $bus; + } + + public function queryBus(): QueryBus + { + /** @var QueryBus $bus */ + $bus = $this->app()->make(ApplicationContext::class)->get(QueryBus::class); + + return $bus; + } +} diff --git a/packages/cqrs/tests/Support/CqrsNoCacheCapstoneTestCase.php b/packages/cqrs/tests/Support/CqrsNoCacheCapstoneTestCase.php new file mode 100644 index 00000000..2bc22a51 --- /dev/null +++ b/packages/cqrs/tests/Support/CqrsNoCacheCapstoneTestCase.php @@ -0,0 +1,14 @@ + dirname(__DIR__).'/CapstoneFixtures/Ledger']; + (new AutoConfigManifestCompiler)->write($psr4, $components, $context); + $paths = [$components, $context]; + } + + return $paths; + } +} From 6cd92a132e34be32f666d448227ed9ec8c1fee22 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 18:51:00 -0700 Subject: [PATCH 10/47] docs(config): document firefly.cqrs.query.cache provider, store and prefix in the skeleton --- skeleton/config/firefly.php | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/skeleton/config/firefly.php b/skeleton/config/firefly.php index d312c1c7..2163b237 100644 --- a/skeleton/config/firefly.php +++ b/skeleton/config/firefly.php @@ -2315,13 +2315,24 @@ 'event_failure_strategy' => 'log', /* - | Default TTL in seconds for #[Cacheable] query results that do not declare their own. UNSET - | means "no default TTL" — not zero — so leave it commented out unless you want one. + | The query cache. | - | Default: unset. + | `provider` 'none' (the default) keeps every query uncached; 'cache' stores the results of queries + | marked #[CachedQuery] (and of Cacheable queries) on the Laravel cache store `store` names ('' = the + | default store), under `prefix`. Any other provider refuses the boot. A #[CachedQuery] query's key is + | derived from its public properties, so equal value objects share an entry; `ttl:` on the attribute + | is its lifetime, falling back to `cache_ttl` (UNSET means "no TTL" — not zero — so leave it + | commented out unless you want one). + | + | Defaults: provider 'none', store '', prefix 'firefly:cqrs:', cache_ttl unset. */ // 'query' => [ // 'cache_ttl' => 60, + // 'cache' => [ + // 'provider' => env('FIREFLY_CQRS_QUERY_CACHE', 'none'), + // 'store' => env('FIREFLY_CQRS_QUERY_CACHE_STORE', ''), + // 'prefix' => 'firefly:cqrs:', + // ], // ], ], From 21b8e0b67cb6bd9b58b448ced4fd0c2a10ff4e1c Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 18:55:49 -0700 Subject: [PATCH 11/47] feat(cqrs): evict cached queries on committed domain events and on EDA envelopes via #[InvalidatesQuery] --- .../cqrs/cache/firefly-cqrs-components.php | 21 +- packages/cqrs/cache/firefly-cqrs-context.php | 25 ++- .../Boot/QueryCacheInvalidationWiringPass.php | 79 +++++++ .../cqrs/src/Cache/QueryCacheInvalidator.php | 197 ++++++++++++++++++ packages/cqrs/src/CqrsAutoConfiguration.php | 33 +++ packages/cqrs/src/CqrsWiringProvider.php | 6 +- .../tests/Cache/QueryCacheInvalidatorTest.php | 94 +++++++++ .../CapstoneCqrsQueryCacheDisabledTest.php | 6 + ...CapstoneCqrsQueryCacheInvalidationTest.php | 75 +++++++ 9 files changed, 528 insertions(+), 8 deletions(-) create mode 100644 packages/cqrs/src/Boot/QueryCacheInvalidationWiringPass.php create mode 100644 packages/cqrs/src/Cache/QueryCacheInvalidator.php create mode 100644 packages/cqrs/tests/Cache/QueryCacheInvalidatorTest.php create mode 100644 packages/cqrs/tests/CapstoneCqrsQueryCacheInvalidationTest.php diff --git a/packages/cqrs/cache/firefly-cqrs-components.php b/packages/cqrs/cache/firefly-cqrs-components.php index 49df8aea..7d2cd957 100644 --- a/packages/cqrs/cache/firefly-cqrs-components.php +++ b/packages/cqrs/cache/firefly-cqrs-components.php @@ -108,6 +108,21 @@ ], ], 8 => [ + 'method' => 'queryCacheInvalidator', + 'returns' => 'Firefly\\Cqrs\\Cache\\QueryCacheInvalidator', + 'name' => null, + 'scope' => 'Singleton', + 'primary' => false, + 'order' => 0, + 'lazy' => false, + 'dependencies' => [ + 0 => 'Firefly\\Cqrs\\Cache\\QueryCache', + 1 => 'Firefly\\Cqrs\\Handler\\HandlerManifest', + 2 => 'Firefly\\Config\\Config', + 3 => 'Illuminate\\Container\\Container', + ], + ], + 9 => [ 'method' => 'commandEventPublisher', 'returns' => 'Firefly\\Cqrs\\Event\\CommandEventPublisher', 'name' => null, @@ -122,7 +137,7 @@ 3 => 'Firefly\\Cqrs\\Correlation\\CorrelationContext', ], ], - 9 => [ + 10 => [ 'method' => 'domainEventBridge', 'returns' => 'Firefly\\Cqrs\\Event\\DomainEventBridge', 'name' => null, @@ -136,7 +151,7 @@ 2 => 'Illuminate\\Container\\Container', ], ], - 10 => [ + 11 => [ 'method' => 'commandBus', 'returns' => 'Firefly\\Cqrs\\Command\\CommandBus', 'name' => null, @@ -153,7 +168,7 @@ 5 => 'Firefly\\Cqrs\\Tracing\\CqrsTracing', ], ], - 11 => [ + 12 => [ 'method' => 'queryBus', 'returns' => 'Firefly\\Cqrs\\Query\\QueryBus', 'name' => null, diff --git a/packages/cqrs/cache/firefly-cqrs-context.php b/packages/cqrs/cache/firefly-cqrs-context.php index b85e9c31..e21461ca 100644 --- a/packages/cqrs/cache/firefly-cqrs-context.php +++ b/packages/cqrs/cache/firefly-cqrs-context.php @@ -105,6 +105,25 @@ ], ], 8 => [ + 'method' => 'queryCacheInvalidator', + 'conditions' => [ + 0 => [ + 'type' => 'Firefly\\Context\\Condition\\Attributes\\ConditionalOnProperty', + 'args' => [ + 0 => 'firefly.cqrs.query.cache.provider', + 1 => 'cache', + 2 => false, + ], + ], + 1 => [ + 'type' => 'Firefly\\Context\\Condition\\Attributes\\ConditionalOnMissingBean', + 'args' => [ + 0 => 'Firefly\\Cqrs\\Cache\\QueryCacheInvalidator', + ], + ], + ], + ], + 9 => [ 'method' => 'commandEventPublisher', 'conditions' => [ 0 => [ @@ -115,7 +134,7 @@ ], ], ], - 9 => [ + 10 => [ 'method' => 'domainEventBridge', 'conditions' => [ 0 => [ @@ -126,7 +145,7 @@ ], ], ], - 10 => [ + 11 => [ 'method' => 'commandBus', 'conditions' => [ 0 => [ @@ -137,7 +156,7 @@ ], ], ], - 11 => [ + 12 => [ 'method' => 'queryBus', 'conditions' => [ 0 => [ diff --git a/packages/cqrs/src/Boot/QueryCacheInvalidationWiringPass.php b/packages/cqrs/src/Boot/QueryCacheInvalidationWiringPass.php new file mode 100644 index 00000000..06dfd00d --- /dev/null +++ b/packages/cqrs/src/Boot/QueryCacheInvalidationWiringPass.php @@ -0,0 +1,79 @@ +container; + if (! $container->bound(QueryCacheInvalidator::class)) { + return; + } + + /** @var QueryCacheInvalidator $invalidator */ + $invalidator = $container->make(QueryCacheInvalidator::class); + + if ($invalidator->hasClassRules()) { + /** @var Dispatcher $dispatcher */ + $dispatcher = $container->make('events'); + $dispatcher->listen('*', DispatcherEventPublisher::guardListener( + /** + * @param array $payload + */ + static function (string $eventName, array $payload) use ($invalidator): void { + $event = $payload[0] ?? null; + if (is_object($event) && ! $event instanceof EventEnvelope) { + $invalidator->onEvent($event); + } + }, + )); + } + + if (! $container->bound(EventPublisher::class)) { + return; + } + + /** @var EventPublisher $publisher */ + $publisher = $container->make(EventPublisher::class); + foreach ($invalidator->eventTypes() as $eventType) { + $publisher->subscribe($eventType, static function (EventEnvelope $envelope) use ($invalidator): void { + $invalidator->onEnvelope($envelope); + }); + } + } +} diff --git a/packages/cqrs/src/Cache/QueryCacheInvalidator.php b/packages/cqrs/src/Cache/QueryCacheInvalidator.php new file mode 100644 index 00000000..a3aae030 --- /dev/null +++ b/packages/cqrs/src/Cache/QueryCacheInvalidator.php @@ -0,0 +1,197 @@ + $rules + */ + public function __construct( + private readonly QueryCache $cache, + private readonly QueryCachePolicies $policies, + private readonly array $rules, + private readonly ?LoggerInterface $logger = null, + ) {} + + /** + * @return list the distinct event types the rules match, for one exact EDA subscription each + */ + public function eventTypes(): array + { + return array_values(array_unique(array_map(static fn (InvalidationRule $rule): string => $rule->eventType, $this->rules))); + } + + public function hasClassRules(): bool + { + foreach ($this->rules as $rule) { + if ($rule->eventClass !== '') { + return true; + } + } + + return false; + } + + public function onEvent(object $event): int + { + $values = null; + $evicted = 0; + + foreach ($this->rules as $rule) { + if ($rule->eventClass === '' || ! $event instanceof $rule->eventClass) { + continue; + } + + $values ??= self::publicProperties($event); + $evicted += $this->apply($rule, $values, $event::class); + } + + return $evicted; + } + + public function onEnvelope(EventEnvelope $envelope): int + { + $evicted = 0; + + foreach ($this->rules as $rule) { + if ($rule->eventType === $envelope->eventType) { + $evicted += $this->apply($rule, $envelope->payload, $envelope->eventType); + } + } + + return $evicted; + } + + /** + * @param array $values + */ + private function apply(InvalidationRule $rule, array $values, string $source): int + { + $group = $this->policies->groupFor($rule->queryClass); + + try { + $key = $rule->evictsGroup() ? null : $this->keyFor($rule, $values, $source); + + if ($key !== null) { + if ($this->cache instanceof GroupedQueryCache) { + $this->cache->evictKey($group, $key); + } else { + $this->cache->evict($key); + } + + return 1; + } + + if ($this->cache instanceof GroupedQueryCache) { + $this->cache->evictGroup($group); + + return 1; + } + + if (! $this->warnedGroupless) { + $this->warnedGroupless = true; + $this->logger?->warning('The bound QueryCache has no groups, so #[InvalidatesQuery] rules that evict a whole group are skipped; bind a GroupedQueryCache (firefly.cqrs.query.cache.provider=cache).'); + } + + return 0; + } catch (Throwable $e) { + $this->logger?->warning("Query cache invalidation for [{$rule->queryClass}] on [{$source}] failed: {$e->getMessage()}", ['exception' => $e]); + + return 0; + } + } + + /** + * @param array $values + */ + private function keyFor(InvalidationRule $rule, array $values, string $source): ?string + { + if ($rule->key !== null) { + $missing = null; + $key = preg_replace_callback('/\{([A-Za-z_][A-Za-z0-9_.]*)\}/', static function (array $match) use ($values, &$missing): string { + $value = self::path($values, $match[1]); + if (! is_scalar($value)) { + $missing ??= $match[1]; + + return $match[0]; + } + + return (string) $value; + }, $rule->key); + + if ($missing !== null || ! is_string($key)) { + $this->logger?->warning("[{$source}] has no scalar `{$missing}` for the key template '{$rule->key}' of [{$rule->queryClass}]; evicting the whole group instead."); + + return null; + } + + return $key; + } + + $properties = []; + foreach ($rule->with as $queryProperty => $eventProperty) { + if (! array_key_exists($eventProperty, $values)) { + $this->logger?->warning("[{$source}] has no `{$eventProperty}` for the derived key of [{$rule->queryClass}]; evicting the whole group instead."); + + return null; + } + $properties[$queryProperty] = $values[$eventProperty]; + } + + return QueryCacheKey::fromProperties($rule->queryClass, $properties); + } + + /** + * @param array $values + */ + private static function path(array $values, string $path): mixed + { + $current = $values; + foreach (explode('.', $path) as $segment) { + if (! is_array($current) || ! array_key_exists($segment, $current)) { + return null; + } + $current = $current[$segment]; + } + + return $current; + } + + /** + * @return array + */ + private static function publicProperties(object $event): array + { + $properties = []; + foreach (get_object_vars($event) as $name => $value) { + $properties[(string) $name] = $value; + } + + return $properties; + } +} diff --git a/packages/cqrs/src/CqrsAutoConfiguration.php b/packages/cqrs/src/CqrsAutoConfiguration.php index b63ec19a..e4510cca 100644 --- a/packages/cqrs/src/CqrsAutoConfiguration.php +++ b/packages/cqrs/src/CqrsAutoConfiguration.php @@ -9,9 +9,12 @@ use Firefly\Container\Attributes\Configuration; use Firefly\Container\Attributes\Order; use Firefly\Context\Condition\Attributes\ConditionalOnMissingBean; +use Firefly\Context\Condition\Attributes\ConditionalOnProperty; use Firefly\Cqrs\Cache\CacheQueryCache; +use Firefly\Cqrs\Cache\InvalidationRule; use Firefly\Cqrs\Cache\NoOpQueryCache; use Firefly\Cqrs\Cache\QueryCache; +use Firefly\Cqrs\Cache\QueryCacheInvalidator; use Firefly\Cqrs\Cache\QueryCachePolicies; use Firefly\Cqrs\Command\CommandBus; use Firefly\Cqrs\Command\DefaultCommandBus; @@ -144,6 +147,36 @@ public function queryCache(Config $config, Container $container): QueryCache ); } + /** + * The invalidator behind #[InvalidatesQuery], bound only with the real cache. Its rules are the compiled + * attribute rules plus firefly.cqrs.query.cache.invalidation — `event type => [query classes]` — for events this + * process has no class for (another service's envelope); those evict the queries' whole groups. A malformed + * entry refuses the boot. + */ + #[Bean] + #[ConditionalOnProperty(name: 'firefly.cqrs.query.cache.provider', havingValue: 'cache')] + #[ConditionalOnMissingBean(QueryCacheInvalidator::class)] + public function queryCacheInvalidator(QueryCache $cache, HandlerManifest $manifest, Config $config, Container $container): QueryCacheInvalidator + { + $rules = $manifest->invalidations(); + + foreach ($config->array('firefly.cqrs.query.cache.invalidation', []) as $eventType => $queries) { + if (! is_string($eventType) || ! is_array($queries)) { + throw new CqrsConfigurationException('firefly.cqrs.query.cache.invalidation must map an event type to a list of query classes.'); + } + + foreach ($queries as $query) { + if (! is_string($query) || ! class_exists($query)) { + throw new CqrsConfigurationException("firefly.cqrs.query.cache.invalidation.{$eventType} names ".get_debug_type($query).' that is not a query class.'); + } + + $rules[] = new InvalidationRule('', $eventType, $query); + } + } + + return new QueryCacheInvalidator($cache, QueryCachePolicies::fromManifest($manifest), $rules, $this->logger($container)); + } + #[Bean] #[ConditionalOnMissingBean(CommandEventPublisher::class)] public function commandEventPublisher(Container $container, Config $config, HandlerManifest $manifest, CorrelationContext $correlation): CommandEventPublisher diff --git a/packages/cqrs/src/CqrsWiringProvider.php b/packages/cqrs/src/CqrsWiringProvider.php index 481d45af..f3f73a2c 100644 --- a/packages/cqrs/src/CqrsWiringProvider.php +++ b/packages/cqrs/src/CqrsWiringProvider.php @@ -9,6 +9,7 @@ use Firefly\Context\Scan\AppScan; use Firefly\Cqrs\Boot\CqrsHandlerWiringPass; use Firefly\Cqrs\Boot\DomainEventBridgeWiringPass; +use Firefly\Cqrs\Boot\QueryCacheInvalidationWiringPass; use Firefly\Cqrs\Handler\HandlerManifest; use Firefly\Cqrs\Scanner\HandlerScanner; use Illuminate\Contracts\Container\Container; @@ -18,7 +19,8 @@ * final register() records candidacy ONLY and never consumes passes(). So — exactly like EdaWiringProvider / * SchedulingWiringProvider — this plain FireflyServiceProvider contributes the wiring pass(es) via passes() and * resolves the HandlerManifest behind a bound() guard. Both this and CqrsServiceProvider are listed in - * extra.laravel.providers. Task 12 adds DomainEventBridgeWiringPass to passes(). + * extra.laravel.providers. It contributes the handler wiring, the domain-event bridge and the query-cache + * invalidation passes. * * The binding resolves its own manifest (compiled artifact first, then an in-process scan of firefly.scan.paths, * then empty) rather than binding an unconditional empty default. Previously only firefly/cli's @@ -56,6 +58,6 @@ public function register(): void */ public function passes(): array { - return [new CqrsHandlerWiringPass, new DomainEventBridgeWiringPass]; + return [new CqrsHandlerWiringPass, new DomainEventBridgeWiringPass, new QueryCacheInvalidationWiringPass]; } } diff --git a/packages/cqrs/tests/Cache/QueryCacheInvalidatorTest.php b/packages/cqrs/tests/Cache/QueryCacheInvalidatorTest.php new file mode 100644 index 00000000..8dc975f8 --- /dev/null +++ b/packages/cqrs/tests/Cache/QueryCacheInvalidatorTest.php @@ -0,0 +1,94 @@ + $rules + * @return array{0: QueryCacheInvalidator, 1: CacheQueryCache, 2: RecordingLogger} + */ +function w4Invalidator(array $rules): array +{ + $cache = new CacheQueryCache(new Repository(new ArrayStore)); + $policies = new QueryCachePolicies([ + LedgerBalance::class => new CachedQueryPolicy(LedgerBalance::class, null, 'balances'), + LedgerTotals::class => new CachedQueryPolicy(LedgerTotals::class, null, 'totals'), + ]); + $logger = new RecordingLogger; + + return [new QueryCacheInvalidator($cache, $policies, $rules, $logger), $cache, $logger]; +} + +function w4BalanceKey(string $account): string +{ + return (string) QueryCacheKey::of(new LedgerBalance($account)); +} + +it('evicts the one entry a with-map rule names, from an in-process event', function () { + [$invalidator, $cache] = w4Invalidator([new InvalidationRule(EntryPosted::class, 'EntryPosted', LedgerBalance::class, null, ['account' => 'account'])]); + $cache->store('balances', w4BalanceKey('a'), 1, null); + $cache->store('balances', w4BalanceKey('b'), 2, null); + + expect($invalidator->onEvent(new EntryPosted('a', 5)))->toBe(1) + ->and($cache->lookup('balances', w4BalanceKey('a')))->toBeNull() + ->and($cache->lookup('balances', w4BalanceKey('b')))->toBe(2); +}); + +it('evicts a whole group for a rule with neither key nor with', function () { + [$invalidator, $cache] = w4Invalidator([new InvalidationRule(EntryPosted::class, 'EntryPosted', LedgerTotals::class)]); + $cache->store('totals', 'x', 1, null); + + $invalidator->onEvent(new EntryPosted('a', 5)); + + expect($cache->lookup('totals', 'x'))->toBeNull(); +}); + +it('resolves a key template from the event, or from a dotted path into an envelope payload', function () { + [$invalidator, $cache] = w4Invalidator([ + new InvalidationRule(EntryPosted::class, 'EntryPosted', LedgerBalance::class, 'statement:{account}'), + new InvalidationRule('', 'ledger.adjusted', LedgerBalance::class, 'statement:{entry.account}'), + ]); + $cache->store('balances', 'statement:a', 1, null); + $cache->store('balances', 'statement:b', 2, null); + + $invalidator->onEvent(new EntryPosted('a', 5)); + $invalidator->onEnvelope(new EventEnvelope('ledger.adjusted', 'ledger.events', ['entry' => ['account' => 'b']])); + + expect($cache->lookup('balances', 'statement:a'))->toBeNull() + ->and($cache->lookup('balances', 'statement:b'))->toBeNull(); +}); + +it('falls back to evicting the group when the event cannot fill a placeholder', function () { + [$invalidator, $cache, $logger] = w4Invalidator([new InvalidationRule('', 'ledger.adjusted', LedgerBalance::class, null, ['account' => 'account'])]); + $cache->store('balances', w4BalanceKey('a'), 1, null); + + $invalidator->onEnvelope(new EventEnvelope('ledger.adjusted', 'ledger.events', ['other' => 'x'])); + + expect($cache->lookup('balances', w4BalanceKey('a')))->toBeNull() + ->and($logger->records[0]['message'])->toContain('evicting the whole group'); +}); + +it('matches envelopes by exact event type and events by class, and ignores everything else', function () { + [$invalidator, $cache] = w4Invalidator([new InvalidationRule(EntryPosted::class, 'EntryPosted', LedgerTotals::class)]); + $cache->store('totals', 'x', 1, null); + + expect($invalidator->onEnvelope(new EventEnvelope('EntryPostedLater', 'd')))->toBe(0) + ->and($invalidator->onEvent(new stdClass))->toBe(0) + ->and($cache->lookup('totals', 'x'))->toBe(1) + ->and($invalidator->onEnvelope(new EventEnvelope('EntryPosted', 'd')))->toBe(1) + ->and($invalidator->eventTypes())->toBe(['EntryPosted']) + ->and($invalidator->hasClassRules())->toBeTrue(); +}); diff --git a/packages/cqrs/tests/CapstoneCqrsQueryCacheDisabledTest.php b/packages/cqrs/tests/CapstoneCqrsQueryCacheDisabledTest.php index 274e98df..d5950be8 100644 --- a/packages/cqrs/tests/CapstoneCqrsQueryCacheDisabledTest.php +++ b/packages/cqrs/tests/CapstoneCqrsQueryCacheDisabledTest.php @@ -4,6 +4,7 @@ use Firefly\Cqrs\Cache\NoOpQueryCache; use Firefly\Cqrs\Cache\QueryCache; +use Firefly\Cqrs\Cache\QueryCacheInvalidator; use Firefly\Cqrs\Tests\CapstoneFixtures\Ledger\HandlerCalls; use Firefly\Cqrs\Tests\CapstoneFixtures\Ledger\LedgerBalance; use Firefly\Cqrs\Tests\Support\CqrsNoCacheCapstoneTestCase; @@ -18,3 +19,8 @@ expect($this->app()->make(QueryCache::class))->toBeInstanceOf(NoOpQueryCache::class) ->and(HandlerCalls::of('balance'))->toBe(2); }); + +it('registers no invalidator with provider=none', function () { + /** @var CqrsNoCacheCapstoneTestCase $this */ + expect($this->app()->bound(QueryCacheInvalidator::class))->toBeFalse(); +}); diff --git a/packages/cqrs/tests/CapstoneCqrsQueryCacheInvalidationTest.php b/packages/cqrs/tests/CapstoneCqrsQueryCacheInvalidationTest.php new file mode 100644 index 00000000..0530497b --- /dev/null +++ b/packages/cqrs/tests/CapstoneCqrsQueryCacheInvalidationTest.php @@ -0,0 +1,75 @@ +queryBus(); + $queries->ask(new LedgerBalance('alice')); + $queries->ask(new LedgerBalance('bob')); + + $this->commandBus()->send(new PostEntry('alice', 7)); + + expect($queries->ask(new LedgerBalance('alice')))->toBe(7) + ->and($queries->ask(new LedgerBalance('bob')))->toBe(0) + ->and(HandlerCalls::of('balance'))->toBe(3); +}); + +it('evicts a whole group and a Cacheable key template from the same event', function () { + /** @var CqrsCacheCapstoneTestCase $this */ + $queries = $this->queryBus(); + $queries->ask(new LedgerTotals); + $queries->ask(new AccountStatement('alice')); + + $this->commandBus()->send(new PostEntry('alice', 3)); + + expect($queries->ask(new LedgerTotals))->toBe(3) + ->and($queries->ask(new AccountStatement('alice')))->toBe([3]) + ->and(HandlerCalls::of('totals'))->toBe(2) + ->and(HandlerCalls::of('statement'))->toBe(2); +}); + +it('evicts nothing when the transaction rolls back', function () { + /** @var CqrsCacheCapstoneTestCase $this */ + $queries = $this->queryBus(); + $queries->ask(new LedgerBalance('alice')); + + expect(fn () => $this->commandBus()->send(new RejectEntry('alice', 9)))->toThrow(RuntimeException::class); + + expect($queries->ask(new LedgerBalance('alice')))->toBe(0) + ->and(HandlerCalls::of('balance'))->toBe(1); +}); + +it('evicts on an EDA envelope another process published', function () { + /** @var CqrsCacheCapstoneTestCase $this */ + $queries = $this->queryBus(); + $queries->ask(new LedgerBalance('alice')); + + $this->events->publish('ledger.events', 'EntryPosted', ['account' => 'alice', 'amount' => 1]); + $queries->ask(new LedgerBalance('alice')); + + expect(HandlerCalls::of('balance'))->toBe(2); +}); + +it('applies the firefly.cqrs.query.cache.invalidation rules to events with no local class', function () { + /** @var CqrsCacheCapstoneTestCase $this */ + $queries = $this->queryBus(); + $queries->ask(new LedgerTotals); + + $this->events->publish('ledger.events', 'ledger.closed', ['period' => '2026-09']); + $queries->ask(new LedgerTotals); + + expect(HandlerCalls::of('totals'))->toBe(2) + ->and($this->app()->make(QueryCacheInvalidator::class)->eventTypes())->toContain('EntryPosted', 'ledger.closed'); +}); From 67642f8a52dac197d1eaeec3d62b85b5d740df54 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 18:55:49 -0700 Subject: [PATCH 12/47] docs(config): document firefly.cqrs.query.cache.invalidation in the skeleton --- skeleton/config/firefly.php | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/skeleton/config/firefly.php b/skeleton/config/firefly.php index 2163b237..e71b3b27 100644 --- a/skeleton/config/firefly.php +++ b/skeleton/config/firefly.php @@ -2324,7 +2324,13 @@ | is its lifetime, falling back to `cache_ttl` (UNSET means "no TTL" — not zero — so leave it | commented out unless you want one). | - | Defaults: provider 'none', store '', prefix 'firefly:cqrs:', cache_ttl unset. + | With provider 'cache', #[InvalidatesQuery] on an event class evicts cached queries when that event + | is dispatched in process (a DomainEvent, after its transaction commits; nothing on rollback) or + | arrives as an EDA envelope of the same event type. `invalidation` adds rules for events this + | process has no class for (another service's envelopes): `event type => [query classes]`, each + | evicting the whole group of the queries it lists. A malformed entry refuses the boot. + | + | Defaults: provider 'none', store '', prefix 'firefly:cqrs:', cache_ttl unset, invalidation []. */ // 'query' => [ // 'cache_ttl' => 60, @@ -2332,6 +2338,9 @@ // 'provider' => env('FIREFLY_CQRS_QUERY_CACHE', 'none'), // 'store' => env('FIREFLY_CQRS_QUERY_CACHE_STORE', ''), // 'prefix' => 'firefly:cqrs:', + // 'invalidation' => [ + // 'ledger.closed' => [App\Ledger\LedgerTotals::class], + // ], // ], // ], ], From 18b146f1bbd959287711752c75c32d939293f8c1 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 19:04:19 -0700 Subject: [PATCH 13/47] fix(cqrs): evict the group when an EDA envelope cannot rebuild a typed with-map query key --- .../cqrs/src/Attributes/InvalidatesQuery.php | 6 +- packages/cqrs/src/Cache/InvalidationRule.php | 15 +++- .../cqrs/src/Cache/QueryCacheInvalidator.php | 76 ++++++++++++++++-- packages/cqrs/src/Scanner/HandlerScanner.php | 32 ++++++-- .../tests/Cache/Fixtures/LimitedBalance.php | 13 +++ .../tests/Cache/QueryCacheInvalidatorTest.php | 79 +++++++++++++++++++ ...CapstoneCqrsQueryCacheInvalidationTest.php | 44 +++++++++++ .../CapstoneFixtures/Ledger/LedgerTier.php | 12 +++ .../CapstoneFixtures/Ledger/TierBalance.php | 15 ++++ .../Ledger/TierBalanceHandler.php | 18 +++++ .../CapstoneFixtures/Ledger/TierChanged.php | 14 ++++ .../tests/Scanner/CacheAttributeScanTest.php | 14 +++- skeleton/config/firefly.php | 9 ++- 13 files changed, 325 insertions(+), 22 deletions(-) create mode 100644 packages/cqrs/tests/Cache/Fixtures/LimitedBalance.php create mode 100644 packages/cqrs/tests/CapstoneFixtures/Ledger/LedgerTier.php create mode 100644 packages/cqrs/tests/CapstoneFixtures/Ledger/TierBalance.php create mode 100644 packages/cqrs/tests/CapstoneFixtures/Ledger/TierBalanceHandler.php create mode 100644 packages/cqrs/tests/CapstoneFixtures/Ledger/TierChanged.php diff --git a/packages/cqrs/src/Attributes/InvalidatesQuery.php b/packages/cqrs/src/Attributes/InvalidatesQuery.php index 5e9e7d71..451c96c4 100644 --- a/packages/cqrs/src/Attributes/InvalidatesQuery.php +++ b/packages/cqrs/src/Attributes/InvalidatesQuery.php @@ -21,7 +21,11 @@ * * An event published over EDA is matched by $eventType — default: the short class name, which is what * DomainEvent::eventType() returns — so another process's envelope evicts this process's cache too. A placeholder - * the event cannot fill evicts the whole group instead: serving stale data is the failure worth preventing. + * the event cannot fill evicts the whole group instead: serving stale data is the failure worth preventing. The + * same holds for a `with` map whose query properties are not plain scalars: an envelope crossing a broker arrives + * JSON-decoded (an enum as its backing value, a date or value object as an array), which can no longer rebuild the + * typed key, so it evicts the group. Only in process, or with scalar-typed query properties, does an envelope + * evict exactly one entry. * PyFly's @cache_evict, moved to the event because the event is what knows what it makes stale. */ #[Attribute(Attribute::TARGET_CLASS | Attribute::IS_REPEATABLE)] diff --git a/packages/cqrs/src/Cache/InvalidationRule.php b/packages/cqrs/src/Cache/InvalidationRule.php index a8470c50..8295e845 100644 --- a/packages/cqrs/src/Cache/InvalidationRule.php +++ b/packages/cqrs/src/Cache/InvalidationRule.php @@ -9,12 +9,19 @@ * an event that has no class in this process and is only ever seen as an EDA envelope). See the attribute for * what $key and $with mean; neither set means "evict the whole group". * - * @phpstan-type InvalidationRow array{eventClass: string, eventType: string, queryClass: string, key: string|null, with: array} + * $withTypes records, per mapped query property, the type HandlerScanner read off the query (`string`, `?int`, an + * enum or class name, '' when untyped or a union). An in-process event hands over real typed values, but an EDA + * envelope from another process carries JSON-decoded ones, so the invalidator rebuilds a with-map key from an + * envelope only when the payload value has the declared type — otherwise it evicts the group. A rule without types + * (built by hand, or loaded from a manifest compiled before they existed) always takes that safe path for envelopes. + * + * @phpstan-type InvalidationRow array{eventClass: string, eventType: string, queryClass: string, key: string|null, with: array, withTypes?: array} */ final readonly class InvalidationRule { /** - * @param array $with + * @param array $with query property => event property + * @param array $withTypes query property => declared type */ public function __construct( public string $eventClass, @@ -22,6 +29,7 @@ public function __construct( public string $queryClass, public ?string $key = null, public array $with = [], + public array $withTypes = [], ) {} public function evictsGroup(): bool @@ -40,6 +48,7 @@ public function toArray(): array 'queryClass' => $this->queryClass, 'key' => $this->key, 'with' => $this->with, + 'withTypes' => $this->withTypes, ]; } @@ -48,6 +57,6 @@ public function toArray(): array */ public static function fromArray(array $row): self { - return new self($row['eventClass'], $row['eventType'], $row['queryClass'], $row['key'], $row['with']); + return new self($row['eventClass'], $row['eventType'], $row['queryClass'], $row['key'], $row['with'], $row['withTypes'] ?? []); } } diff --git a/packages/cqrs/src/Cache/QueryCacheInvalidator.php b/packages/cqrs/src/Cache/QueryCacheInvalidator.php index a3aae030..a169cd03 100644 --- a/packages/cqrs/src/Cache/QueryCacheInvalidator.php +++ b/packages/cqrs/src/Cache/QueryCacheInvalidator.php @@ -17,6 +17,15 @@ * * onEnvelope() — an EDA envelope (another process's event, or this one's echoed by the domain bridge). Matched * by EXACT event type; placeholders read its payload, dotted paths reaching into nested arrays. * + * A with-map key is a QueryCacheKey digest, and that digest is TYPED: an enum is written by case, a date as an + * instant, a value object by its class. An envelope from another process arrives JSON-decoded — the enum is now its + * backing scalar, the date a {date, timezone_type, timezone} array, the value object a plain array — so a key rebuilt + * from it would name an entry that does not exist and evict nothing. onEnvelope() therefore rebuilds a with-map key + * only when every payload value has the query property's declared type (HandlerScanner records it: an int may stand + * for a float, which is how JSON writes 5.0; an object passes only when it is already an instance, as on an + * in-memory bus) and otherwise evicts the group, warning once per rule. Key templates are unaffected: they only + * ever interpolate scalars, which survive JSON as they are. + * * Evicting twice is harmless, so an event that arrives both ways costs one extra delete. A rule whose placeholder * cannot be filled evicts the whole GROUP instead and logs why: an unfillable key would otherwise evict nothing and * leave the stale entry serving. With a cache that has no groups (an application's own QueryCache), key evictions @@ -28,6 +37,9 @@ final class QueryCacheInvalidator { private bool $warnedGroupless = false; + /** @var array rules whose envelope key could not be rebuilt, already warned about */ + private array $warnedUntyped = []; + /** * @param list $rules */ @@ -68,7 +80,7 @@ public function onEvent(object $event): int } $values ??= self::publicProperties($event); - $evicted += $this->apply($rule, $values, $event::class); + $evicted += $this->apply($rule, $values, $event::class, false); } return $evicted; @@ -80,7 +92,7 @@ public function onEnvelope(EventEnvelope $envelope): int foreach ($this->rules as $rule) { if ($rule->eventType === $envelope->eventType) { - $evicted += $this->apply($rule, $envelope->payload, $envelope->eventType); + $evicted += $this->apply($rule, $envelope->payload, $envelope->eventType, true); } } @@ -90,12 +102,12 @@ public function onEnvelope(EventEnvelope $envelope): int /** * @param array $values */ - private function apply(InvalidationRule $rule, array $values, string $source): int + private function apply(InvalidationRule $rule, array $values, string $source, bool $fromEnvelope): int { $group = $this->policies->groupFor($rule->queryClass); try { - $key = $rule->evictsGroup() ? null : $this->keyFor($rule, $values, $source); + $key = $rule->evictsGroup() ? null : $this->keyFor($rule, $values, $source, $fromEnvelope); if ($key !== null) { if ($this->cache instanceof GroupedQueryCache) { @@ -129,7 +141,7 @@ private function apply(InvalidationRule $rule, array $values, string $source): i /** * @param array $values */ - private function keyFor(InvalidationRule $rule, array $values, string $source): ?string + private function keyFor(InvalidationRule $rule, array $values, string $source, bool $fromEnvelope): ?string { if ($rule->key !== null) { $missing = null; @@ -160,12 +172,64 @@ private function keyFor(InvalidationRule $rule, array $values, string $source): return null; } - $properties[$queryProperty] = $values[$eventProperty]; + $value = $values[$eventProperty]; + if ($fromEnvelope) { + $type = $rule->withTypes[$queryProperty] ?? ''; + if (! self::envelopeValueFits($type, $value)) { + $this->warnUntypedOnce($rule, $source, $queryProperty, $eventProperty, $type, $value); + + return null; + } + if (ltrim($type, '?') === 'float' && is_int($value)) { + $value = (float) $value; + } + } + $properties[$queryProperty] = $value; } return QueryCacheKey::fromProperties($rule->queryClass, $properties); } + /** + * Whether an envelope payload value is exactly what the query property holds, so the digest it builds is the + * digest the cached entry was stored under. Unknown types ('' — untyped, a union, a rule built without types), + * mixed, array and every class or enum type the value is not already an instance of answer false. + */ + private static function envelopeValueFits(string $type, mixed $value): bool + { + if ($type === '' || $type === 'mixed') { + return false; + } + if ($value === null) { + return str_starts_with($type, '?') || $type === 'null'; + } + + $base = ltrim($type, '?'); + + return match ($base) { + 'string' => is_string($value), + 'int' => is_int($value), + 'float' => is_float($value) || is_int($value), + 'bool' => is_bool($value), + 'true' => $value === true, + 'false' => $value === false, + 'array', 'iterable', 'object', 'null', 'callable', 'static', 'self' => false, + default => $value instanceof $base, + }; + } + + private function warnUntypedOnce(InvalidationRule $rule, string $source, string $queryProperty, string $eventProperty, string $type, mixed $value): void + { + $id = $source."\0".$rule->queryClass."\0".$queryProperty; + if (isset($this->warnedUntyped[$id])) { + return; + } + $this->warnedUntyped[$id] = true; + + $declared = $type === '' ? 'an unrecorded or non-scalar type' : "`{$type}`"; + $this->logger?->warning("[{$source}] carries `{$eventProperty}` as ".get_debug_type($value).", which cannot rebuild {$declared} `{$queryProperty}` of [{$rule->queryClass}] after crossing a broker; evicting the whole group instead."); + } + /** * @param array $values */ diff --git a/packages/cqrs/src/Scanner/HandlerScanner.php b/packages/cqrs/src/Scanner/HandlerScanner.php index d26e201f..106e5f1b 100644 --- a/packages/cqrs/src/Scanner/HandlerScanner.php +++ b/packages/cqrs/src/Scanner/HandlerScanner.php @@ -99,10 +99,8 @@ public function scan(array $psr4): array if ($rule->key !== null && ! is_subclass_of($rule->query, Cacheable::class)) { throw new CqrsConfigurationException("#[InvalidatesQuery({$rule->query}::class)] on [{$class}] declares a key template, but [{$rule->query}] is keyed by value (QueryCacheKey); use a with map or no key to evict its group."); } - if ($rule->with !== []) { - $this->assertWithMapsQuery($class, $rule->query, $rule->with); - } - $invalidations[] = new InvalidationRule($class, $rule->eventType ?? $reflection->getShortName(), $rule->query, $rule->key, $rule->with); + $withTypes = $rule->with === [] ? [] : $this->assertWithMapsQuery($class, $rule->query, $rule->with); + $invalidations[] = new InvalidationRule($class, $rule->eventType ?? $reflection->getShortName(), $rule->query, $rule->key, $rule->with, $withTypes); } } @@ -115,17 +113,22 @@ public function scan(array $psr4): array * eviction silently misses. A query holding private or protected state is never keyed by value (QueryCacheKey * answers null), and a Cacheable query is keyed by its own cacheKey(), so `with` can match neither. * + * Answers each mapped property's declared type (see publicPropertiesOrFail()), which the invalidator needs to tell + * whether an EDA payload value can still rebuild the key once it has crossed a JSON broker. + * * @param class-string $queryClass * @param array $with + * @return array query property => declared type */ - private function assertWithMapsQuery(string $eventClass, string $queryClass, array $with): void + private function assertWithMapsQuery(string $eventClass, string $queryClass, array $with): array { $where = "#[InvalidatesQuery({$queryClass}::class)] on [{$eventClass}]"; if (is_subclass_of($queryClass, Cacheable::class)) { throw new CqrsConfigurationException("{$where} declares a with map, but the query is Cacheable and keyed by its own cacheKey(); use a key template instead."); } - $public = $this->publicPropertiesOrFail($queryClass, $where, 'declares a with map'); + $types = $this->publicPropertiesOrFail($queryClass, $where, 'declares a with map'); + $public = array_keys($types); $unknown = array_diff(array_keys($with), $public); if ($unknown !== []) { @@ -136,6 +139,13 @@ private function assertWithMapsQuery(string $eventClass, string $queryClass, arr if ($unmapped !== []) { throw new CqrsConfigurationException("{$where} leaves [".implode(', ', $unmapped).'] unmapped; a with map names every public property of the query.'); } + + $mapped = []; + foreach (array_keys($with) as $property) { + $mapped[$property] = $types[$property]; + } + + return $mapped; } /** @@ -143,8 +153,11 @@ private function assertWithMapsQuery(string $eventClass, string $queryClass, arr * and answers null for an object holding more, so a value-keyed query with private or protected state would never * be cached (#[CachedQuery] silently doing nothing) and no `with` map could name its entries. * + * Each name maps to its declared type: a single named type as written (`string`, `?int`, an enum or class name), + * or '' for an untyped property and a union or intersection type, which no payload value can be checked against. + * * @param class-string $queryClass - * @return list + * @return array property name => declared type */ private function publicPropertiesOrFail(string $queryClass, string $where, string $what): array { @@ -156,7 +169,10 @@ private function publicPropertiesOrFail(string $queryClass, string $where, strin if (! $property->isPublic()) { throw new CqrsConfigurationException("{$where} {$what}, but [{$queryClass}::\${$property->getName()}] is not public; a query keyed by value holds only public state (or implements Cacheable to key itself)."); } - $public[] = $property->getName(); + $type = $property->getType(); + $public[$property->getName()] = $type instanceof ReflectionNamedType + ? ($type->allowsNull() && ! in_array($type->getName(), ['null', 'mixed'], true) ? '?' : '').$type->getName() + : ''; } return $public; diff --git a/packages/cqrs/tests/Cache/Fixtures/LimitedBalance.php b/packages/cqrs/tests/Cache/Fixtures/LimitedBalance.php new file mode 100644 index 00000000..1c637a10 --- /dev/null +++ b/packages/cqrs/tests/Cache/Fixtures/LimitedBalance.php @@ -0,0 +1,13 @@ + new CachedQueryPolicy(LedgerBalance::class, null, 'balances'), LedgerTotals::class => new CachedQueryPolicy(LedgerTotals::class, null, 'totals'), + TierBalance::class => new CachedQueryPolicy(TierBalance::class, null, 'tiers'), + LimitedBalance::class => new CachedQueryPolicy(LimitedBalance::class, null, 'limits'), ]); $logger = new RecordingLogger; @@ -92,3 +97,77 @@ function w4BalanceKey(string $account): string ->and($invalidator->eventTypes())->toBe(['EntryPosted']) ->and($invalidator->hasClassRules())->toBeTrue(); }); + +function w4TierRule(): InvalidationRule +{ + return new InvalidationRule('', 'TierChanged', TierBalance::class, null, ['account' => 'account', 'tier' => 'tier'], [ + 'account' => 'string', + 'tier' => LedgerTier::class, + ]); +} + +it('evicts the group, warning once, when an envelope carries an enum-typed query property as its backing value', function () { + [$invalidator, $cache, $logger] = w4Invalidator([w4TierRule()]); + $gold = (string) QueryCacheKey::of(new TierBalance('a', LedgerTier::Gold)); + $silver = (string) QueryCacheKey::of(new TierBalance('b', LedgerTier::Silver)); + $cache->store('tiers', $gold, 1, null); + $cache->store('tiers', $silver, 2, null); + + $invalidator->onEnvelope(new EventEnvelope('TierChanged', 'ledger.events', ['account' => 'a', 'tier' => 'gold'])); + $cache->store('tiers', $gold, 1, null); + $invalidator->onEnvelope(new EventEnvelope('TierChanged', 'ledger.events', ['account' => 'a', 'tier' => 'gold'])); + + expect($cache->lookup('tiers', $gold))->toBeNull() + ->and($cache->lookup('tiers', $silver))->toBeNull() + ->and($logger->records)->toHaveCount(1) + ->and($logger->records[0]['message'])->toContain('carries `tier` as string')->toContain('evicting the whole group'); +}); + +it('evicts the one enum-typed entry when the envelope payload still carries the case, as on an in-memory bus', function () { + [$invalidator, $cache, $logger] = w4Invalidator([w4TierRule()]); + $gold = (string) QueryCacheKey::of(new TierBalance('a', LedgerTier::Gold)); + $silver = (string) QueryCacheKey::of(new TierBalance('b', LedgerTier::Silver)); + $cache->store('tiers', $gold, 1, null); + $cache->store('tiers', $silver, 2, null); + + $invalidator->onEnvelope(new EventEnvelope('TierChanged', 'ledger.events', ['account' => 'a', 'tier' => LedgerTier::Gold])); + + expect($cache->lookup('tiers', $gold))->toBeNull() + ->and($cache->lookup('tiers', $silver))->toBe(2) + ->and($logger->records)->toBe([]); +}); + +it('rebuilds a scalar key from an envelope, reading a JSON integer as the float the query declares and null as a nullable int', function () { + [$invalidator, $cache, $logger] = w4Invalidator([new InvalidationRule('', 'LimitChanged', LimitedBalance::class, null, + ['account' => 'account', 'limit' => 'limit', 'page' => 'page'], + ['account' => 'string', 'limit' => 'float', 'page' => '?int'], + )]); + $target = (string) QueryCacheKey::of(new LimitedBalance('a', 5.0, null)); + $other = (string) QueryCacheKey::of(new LimitedBalance('b', 5.0, null)); + $cache->store('limits', $target, 1, null); + $cache->store('limits', $other, 2, null); + + $invalidator->onEnvelope(new EventEnvelope('LimitChanged', 'd', ['account' => 'a', 'limit' => 5, 'page' => null])); + + expect($cache->lookup('limits', $target))->toBeNull() + ->and($cache->lookup('limits', $other))->toBe(2) + ->and($logger->records)->toBe([]); +}); + +it('evicts the group for an envelope when a scalar payload value has the wrong type or the rule has no recorded types', function () { + [$invalidator, $cache] = w4Invalidator([ + new InvalidationRule('', 'LimitChanged', LimitedBalance::class, null, + ['account' => 'account', 'limit' => 'limit', 'page' => 'page'], + ['account' => 'string', 'limit' => 'float', 'page' => '?int'], + ), + new InvalidationRule(EntryPosted::class, 'EntryPosted', LedgerBalance::class, null, ['account' => 'account']), + ]); + $cache->store('limits', (string) QueryCacheKey::of(new LimitedBalance('b', 5.0, 2)), 2, null); + $cache->store('balances', w4BalanceKey('b'), 2, null); + + $invalidator->onEnvelope(new EventEnvelope('LimitChanged', 'd', ['account' => 'a', 'limit' => 5, 'page' => '2'])); + $invalidator->onEnvelope(new EventEnvelope('EntryPosted', 'd', ['account' => 'a'])); + + expect($cache->lookup('limits', (string) QueryCacheKey::of(new LimitedBalance('b', 5.0, 2))))->toBeNull() + ->and($cache->lookup('balances', w4BalanceKey('b')))->toBeNull(); +}); diff --git a/packages/cqrs/tests/CapstoneCqrsQueryCacheInvalidationTest.php b/packages/cqrs/tests/CapstoneCqrsQueryCacheInvalidationTest.php index 0530497b..a7a7543a 100644 --- a/packages/cqrs/tests/CapstoneCqrsQueryCacheInvalidationTest.php +++ b/packages/cqrs/tests/CapstoneCqrsQueryCacheInvalidationTest.php @@ -6,9 +6,12 @@ use Firefly\Cqrs\Tests\CapstoneFixtures\Ledger\AccountStatement; use Firefly\Cqrs\Tests\CapstoneFixtures\Ledger\HandlerCalls; use Firefly\Cqrs\Tests\CapstoneFixtures\Ledger\LedgerBalance; +use Firefly\Cqrs\Tests\CapstoneFixtures\Ledger\LedgerTier; use Firefly\Cqrs\Tests\CapstoneFixtures\Ledger\LedgerTotals; use Firefly\Cqrs\Tests\CapstoneFixtures\Ledger\PostEntry; use Firefly\Cqrs\Tests\CapstoneFixtures\Ledger\RejectEntry; +use Firefly\Cqrs\Tests\CapstoneFixtures\Ledger\TierBalance; +use Firefly\Cqrs\Tests\CapstoneFixtures\Ledger\TierChanged; use Firefly\Cqrs\Tests\Support\CqrsCacheCapstoneTestCase; uses(CqrsCacheCapstoneTestCase::class); @@ -73,3 +76,44 @@ expect(HandlerCalls::of('totals'))->toBe(2) ->and($this->app()->make(QueryCacheInvalidator::class)->eventTypes())->toContain('EntryPosted', 'ledger.closed'); }); + +it('evicts exactly the one entry a scalar with map names when another process publishes the envelope', function () { + /** @var CqrsCacheCapstoneTestCase $this */ + $queries = $this->queryBus(); + $queries->ask(new LedgerBalance('alice')); + $queries->ask(new LedgerBalance('bob')); + + $this->events->publish('ledger.events', 'EntryPosted', ['account' => 'alice', 'amount' => 1]); + $queries->ask(new LedgerBalance('alice')); + $queries->ask(new LedgerBalance('bob')); + + expect(HandlerCalls::of('balance'))->toBe(3); +}); + +// The envelope carries the enum as 'gold' (what JSON makes of LedgerTier::Gold), which cannot rebuild the typed +// digest the entry lives under: the group goes instead, so the stale alice entry stops serving. +it('evicts the group when an envelope payload cannot rebuild an enum-typed with-map key', function () { + /** @var CqrsCacheCapstoneTestCase $this */ + $queries = $this->queryBus(); + $queries->ask(new TierBalance('alice', LedgerTier::Gold)); + $queries->ask(new TierBalance('bob', LedgerTier::Silver)); + + $this->events->publish('ledger.events', 'TierChanged', ['account' => 'alice', 'tier' => 'gold']); + $queries->ask(new TierBalance('alice', LedgerTier::Gold)); + $queries->ask(new TierBalance('bob', LedgerTier::Silver)); + + expect(HandlerCalls::of('tier'))->toBe(4); +}); + +it('evicts the one enum-typed entry in process, where the event carries the real case', function () { + /** @var CqrsCacheCapstoneTestCase $this */ + $queries = $this->queryBus(); + $queries->ask(new TierBalance('alice', LedgerTier::Gold)); + $queries->ask(new TierBalance('bob', LedgerTier::Silver)); + + $this->app()->make('events')->dispatch(new TierChanged('alice', LedgerTier::Gold)); + $queries->ask(new TierBalance('alice', LedgerTier::Gold)); + $queries->ask(new TierBalance('bob', LedgerTier::Silver)); + + expect(HandlerCalls::of('tier'))->toBe(3); +}); diff --git a/packages/cqrs/tests/CapstoneFixtures/Ledger/LedgerTier.php b/packages/cqrs/tests/CapstoneFixtures/Ledger/LedgerTier.php new file mode 100644 index 00000000..b01e49d3 --- /dev/null +++ b/packages/cqrs/tests/CapstoneFixtures/Ledger/LedgerTier.php @@ -0,0 +1,12 @@ +where('account', $query->account)->sum('amount'); + } +} diff --git a/packages/cqrs/tests/CapstoneFixtures/Ledger/TierChanged.php b/packages/cqrs/tests/CapstoneFixtures/Ledger/TierChanged.php new file mode 100644 index 00000000..696c3982 --- /dev/null +++ b/packages/cqrs/tests/CapstoneFixtures/Ledger/TierChanged.php @@ -0,0 +1,14 @@ + 'account', 'tier' => 'tier'])] +final readonly class TierChanged +{ + public function __construct(public string $account, public LedgerTier $tier) {} +} diff --git a/packages/cqrs/tests/Scanner/CacheAttributeScanTest.php b/packages/cqrs/tests/Scanner/CacheAttributeScanTest.php index 17df2ca7..34487d17 100644 --- a/packages/cqrs/tests/Scanner/CacheAttributeScanTest.php +++ b/packages/cqrs/tests/Scanner/CacheAttributeScanTest.php @@ -14,6 +14,8 @@ use Firefly\Cqrs\Tests\CacheScannerFixtures\EntryPosted; use Firefly\Cqrs\Tests\CacheScannerFixtures\LedgerBalance; use Firefly\Cqrs\Tests\CacheScannerFixtures\LedgerTotals; +use Firefly\Cqrs\Tests\CapstoneFixtures\Ledger\LedgerTier; +use Firefly\Cqrs\Tests\CapstoneFixtures\Ledger\TierBalance; it('compiles #[CachedQuery] into per-query policies with the class as the default group', function () { $result = (new HandlerScanner)->scan(['Firefly\\Cqrs\\Tests\\CacheScannerFixtures\\' => dirname(__DIR__).'/CacheScannerFixtures']); @@ -28,13 +30,23 @@ $result = (new HandlerScanner)->scan(['Firefly\\Cqrs\\Tests\\CacheScannerFixtures\\' => dirname(__DIR__).'/CacheScannerFixtures']); expect($result['invalidations'])->toEqual([ - new InvalidationRule(EntryPosted::class, 'EntryPosted', LedgerBalance::class, null, ['account' => 'account']), + new InvalidationRule(EntryPosted::class, 'EntryPosted', LedgerBalance::class, null, ['account' => 'account'], ['account' => 'string']), new InvalidationRule(EntryPosted::class, 'ledger.entry-posted', LedgerTotals::class), ]) ->and($result['invalidations'][1]->evictsGroup())->toBeTrue() ->and($result['invalidations'][0]->evictsGroup())->toBeFalse(); }); +it('records the declared type of every query property a with map names, enums by class', function () { + $result = (new HandlerScanner)->scan(['Firefly\\Cqrs\\Tests\\CapstoneFixtures\\Ledger\\' => dirname(__DIR__).'/CapstoneFixtures/Ledger']); + $tier = array_values(array_filter($result['invalidations'], static fn (InvalidationRule $r): bool => $r->queryClass === TierBalance::class)); + + expect($tier)->toHaveCount(1) + ->and($tier[0]->withTypes)->toBe(['account' => 'string', 'tier' => LedgerTier::class]) + ->and(InvalidationRule::fromArray($tier[0]->toArray()))->toEqual($tier[0]) + ->and(InvalidationRule::fromArray(['eventClass' => '', 'eventType' => 'x', 'queryClass' => TierBalance::class, 'key' => null, 'with' => ['tier' => 'tier']])->withTypes)->toBe([]); +}); + it('refuses a rule that names both a key template and a with map, and a TTL below one second', function () { $scan = static fn (): array => (new HandlerScanner)->scan(['Firefly\\Cqrs\\Tests\\CacheScannerBadFixtures\\' => dirname(__DIR__).'/CacheScannerBadFixtures']); diff --git a/skeleton/config/firefly.php b/skeleton/config/firefly.php index e71b3b27..5d652510 100644 --- a/skeleton/config/firefly.php +++ b/skeleton/config/firefly.php @@ -2326,9 +2326,12 @@ | | With provider 'cache', #[InvalidatesQuery] on an event class evicts cached queries when that event | is dispatched in process (a DomainEvent, after its transaction commits; nothing on rollback) or - | arrives as an EDA envelope of the same event type. `invalidation` adds rules for events this - | process has no class for (another service's envelopes): `event type => [query classes]`, each - | evicting the whole group of the queries it lists. A malformed entry refuses the boot. + | arrives as an EDA envelope of the same event type. An envelope evicts the ONE entry a `with` map + | names only when its payload values carry the query properties' declared types (JSON delivers an + | enum as its backing value, a date as an array); otherwise it evicts the group, with a warning. + | `invalidation` adds rules for events this process has no class for (another service's + | envelopes): `event type => [query classes]`, each evicting the whole group of the queries it + | lists. A malformed entry refuses the boot. | | Defaults: provider 'none', store '', prefix 'firefly:cqrs:', cache_ttl unset, invalidation []. */ From 02c0d9af412e244862e910d733e987077b91d42f Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 19:12:56 -0700 Subject: [PATCH 14/47] fix(resilience): select the store with firefly.resilience.store.provider and refuse to boot on a cache store without atomic locks --- .../cache/firefly-resilience-components.php | 1 + .../src/ResilienceAutoConfiguration.php | 31 +++++- .../src/Store/CacheResilienceStore.php | 37 ++++++- .../tests/Fixtures/LocklessStore.php | 96 +++++++++++++++++++ .../tests/Store/ResilienceStoreBootTest.php | 37 +++++++ .../Store/ResilienceStoreSelectionTest.php | 69 +++++++++++++ .../tests/Support/RecordingLogger.php | 32 +++++++ 7 files changed, 299 insertions(+), 4 deletions(-) create mode 100644 packages/resilience/tests/Fixtures/LocklessStore.php create mode 100644 packages/resilience/tests/Store/ResilienceStoreBootTest.php create mode 100644 packages/resilience/tests/Store/ResilienceStoreSelectionTest.php create mode 100644 packages/resilience/tests/Support/RecordingLogger.php diff --git a/packages/resilience/cache/firefly-resilience-components.php b/packages/resilience/cache/firefly-resilience-components.php index f180733e..5269a7b8 100644 --- a/packages/resilience/cache/firefly-resilience-components.php +++ b/packages/resilience/cache/firefly-resilience-components.php @@ -44,6 +44,7 @@ 'dependencies' => [ 0 => 'Illuminate\\Contracts\\Cache\\Repository', 1 => 'Firefly\\Config\\Config', + 2 => 'Psr\\Log\\LoggerInterface', ], ], 1 => [ diff --git a/packages/resilience/src/ResilienceAutoConfiguration.php b/packages/resilience/src/ResilienceAutoConfiguration.php index 176664d8..180024ed 100644 --- a/packages/resilience/src/ResilienceAutoConfiguration.php +++ b/packages/resilience/src/ResilienceAutoConfiguration.php @@ -10,15 +10,21 @@ use Firefly\Container\Attributes\Order; use Firefly\Context\Condition\Attributes\ConditionalOnMissingBean; use Firefly\Context\Condition\Attributes\ConditionalOnProperty; +use Firefly\Kernel\Exception\Framework\ConfigurationException; use Firefly\Resilience\Method\ResilienceMethodInterceptor; use Firefly\Resilience\Store\CacheResilienceStore; +use Firefly\Resilience\Store\InMemoryResilienceStore; use Firefly\Resilience\Store\ResilienceStore; +use Illuminate\Contracts\Cache\LockProvider; use Illuminate\Contracts\Cache\Repository; +use Psr\Log\LoggerInterface; /** * Always-on resilience wiring (an empty registry is cheap). #[Order(1000)] places it after user definitions; * each bean backs off #[ConditionalOnMissingBean] so an app that supplies its own store/registry wins. The - * default store is Cache-backed (cross-FPM survival); the registry reads firefly.resilience.* and shares it. + * store is chosen by firefly.resilience.store.provider: `cache` (the default — cross-FPM survival on Laravel's + * cache, which must have atomic locks unless firefly.resilience.store.allow-lockless=true) or `memory` (one + * process: tests, CLI, a single long-running worker); the registry reads firefly.resilience.* and shares it. */ #[Configuration] #[Order(1000)] @@ -26,9 +32,28 @@ final class ResilienceAutoConfiguration { #[Bean] #[ConditionalOnMissingBean(ResilienceStore::class)] - public function resilienceStore(Repository $cache, Config $config): ResilienceStore + public function resilienceStore(Repository $cache, Config $config, ?LoggerInterface $logger = null): ResilienceStore { - return new CacheResilienceStore($cache, $this->lockBlockTimeout($config)); + $provider = $config->string('firefly.resilience.store.provider', 'cache'); + + if ($provider === 'memory') { + return new InMemoryResilienceStore; + } + + if ($provider !== 'cache') { + throw new ConfigurationException("firefly.resilience.store.provider must be 'cache' or 'memory', got '{$provider}'."); + } + + $allowLockless = $config->bool('firefly.resilience.store.allow-lockless', false); + $store = $cache->getStore(); + + // Refused HERE, at boot: the bean is an eager singleton, so a deployment on a lockless store learns it + // before the first request rather than in the middle of an outage the breaker was meant to contain. + if (! $store instanceof LockProvider && ! $allowLockless) { + throw CacheResilienceStore::lockless($store); + } + + return new CacheResilienceStore($cache, $this->lockBlockTimeout($config), $allowLockless, $logger); } #[Bean] diff --git a/packages/resilience/src/Store/CacheResilienceStore.php b/packages/resilience/src/Store/CacheResilienceStore.php index 846d2833..13b600b0 100644 --- a/packages/resilience/src/Store/CacheResilienceStore.php +++ b/packages/resilience/src/Store/CacheResilienceStore.php @@ -4,10 +4,12 @@ namespace Firefly\Resilience\Store; +use Firefly\Kernel\Exception\Framework\ConfigurationException; use Firefly\Kernel\Exception\Infrastructure\ServiceUnavailableException; use Illuminate\Contracts\Cache\Lock; use Illuminate\Contracts\Cache\LockProvider; use Illuminate\Contracts\Cache\Repository; +use Psr\Log\LoggerInterface; /** * The default ResilienceStore: Laravel's cache. Every mutator maps to an atomic cache primitive @@ -36,6 +38,11 @@ * retryable error code RESILIENCE_STORE_LOCK_TIMEOUT, which is the honest description of "the shared state * store is too contended to answer right now" and is a status a caller, a load balancer and an SLO dashboard * all already know how to read. + * + * A store WITHOUT atomic locks (Laravel's apc and session stores) is refused: withLock() used to run the section + * unlocked without a word, which is the over-admitting race the store exists to prevent. Now it throws + * ConfigurationException (RESILIENCE_STORE_LOCKLESS) unless the store was built with $allowLockless — what + * firefly.resilience.store.allow-lockless=true does — and then it warns once per store before running unlocked. */ final class CacheResilienceStore implements ResilienceStore { @@ -61,11 +68,31 @@ final class CacheResilienceStore implements ResilienceStore */ private const LOCK_POLL_MICROSECONDS = 5000; + /** Whether the lockless warning was already logged by this store, so a hot breaker does not flood the log. */ + private bool $warnedLockless = false; + public function __construct( private readonly Repository $cache, private readonly float $lockBlockTimeout = self::DEFAULT_LOCK_BLOCK_TIMEOUT, + private readonly bool $allowLockless = false, + private readonly ?LoggerInterface $logger = null, ) {} + /** + * The refusal, worded once for both places it is raised: at boot (ResilienceAutoConfiguration) and from a + * store someone constructed by hand. + */ + public static function lockless(object $store): ConfigurationException + { + return new ConfigurationException( + 'The resilience store\'s cache ['.$store::class.'] has no atomic locks, so a breaker or limiter transition could race ' + .'between workers. Use a store that implements Illuminate\Contracts\Cache\LockProvider (redis, database, file, ' + .'memcached, dynamodb, array), set firefly.resilience.store.provider=memory for a single process, or accept the ' + .'race with firefly.resilience.store.allow-lockless=true.', + 'RESILIENCE_STORE_LOCKLESS', + ); + } + public function get(string $key): mixed { return $this->cache->get($key); @@ -104,7 +131,15 @@ public function withLock(string $key, float $ttlSeconds, callable $callback): mi { $store = $this->cache->getStore(); if (! $store instanceof LockProvider) { - // The driver has no atomic lock (unusual): run without a critical section rather than fail hard. + if (! $this->allowLockless) { + throw self::lockless($store); + } + + if (! $this->warnedLockless) { + $this->warnedLockless = true; + $this->logger?->warning('The resilience store\'s cache ['.$store::class.'] has no atomic locks; circuit breaker, rate limiter and bulkhead transitions run without a critical section because firefly.resilience.store.allow-lockless is true.'); + } + return $callback(); } diff --git a/packages/resilience/tests/Fixtures/LocklessStore.php b/packages/resilience/tests/Fixtures/LocklessStore.php new file mode 100644 index 00000000..eb1af247 --- /dev/null +++ b/packages/resilience/tests/Fixtures/LocklessStore.php @@ -0,0 +1,96 @@ + */ + private array $values = []; + + public function get($key): mixed + { + return $this->values[$key] ?? null; + } + + /** + * @param array $keys + * @return array + */ + public function many(array $keys): array + { + $found = []; + foreach ($keys as $key) { + $found[$key] = $this->values[$key] ?? null; + } + + return $found; + } + + public function put($key, $value, $seconds): bool + { + $this->values[$key] = $value; + + return true; + } + + /** @param array $values */ + public function putMany(array $values, $seconds): bool + { + foreach ($values as $key => $value) { + $this->values[$key] = $value; + } + + return true; + } + + public function increment($key, $value = 1): int + { + $current = $this->values[$key] ?? 0; + $next = (is_int($current) ? $current : 0) + (is_numeric($value) ? (int) $value : 1); + $this->values[$key] = $next; + + return $next; + } + + public function decrement($key, $value = 1): int + { + return $this->increment($key, -(is_numeric($value) ? (int) $value : 1)); + } + + public function forever($key, $value): bool + { + return $this->put($key, $value, 0); + } + + public function touch($key, $seconds): bool + { + return array_key_exists($key, $this->values); + } + + public function forget($key): bool + { + unset($this->values[$key]); + + return true; + } + + public function flush(): bool + { + $this->values = []; + + return true; + } + + public function getPrefix(): string + { + return ''; + } +} diff --git a/packages/resilience/tests/Store/ResilienceStoreBootTest.php b/packages/resilience/tests/Store/ResilienceStoreBootTest.php new file mode 100644 index 00000000..8b50c752 --- /dev/null +++ b/packages/resilience/tests/Store/ResilienceStoreBootTest.php @@ -0,0 +1,37 @@ + $store + */ +function w4BootResilience(array $store): Application +{ + return fireflyApplication( + config: ['firefly' => ['resilience' => ['store' => $store, 'method' => ['enabled' => false]]]], + providers: [ResilienceServiceProvider::class], + bindings: [Repository::class => new CacheRepository(new LocklessStore)], + ); +} + +it('refuses to boot on a lockless default cache store', function () { + expect(fn () => w4BootResilience([]))->toThrow(ConfigurationException::class, 'has no atomic locks'); +}); + +it('boots on a lockless store when allow-lockless is set, and on memory regardless', function () { + expect(w4BootResilience(['allow-lockless' => true])->make(ResilienceRegistry::class))->toBeInstanceOf(ResilienceRegistry::class) + ->and(w4BootResilience(['provider' => 'memory'])->make(ResilienceStore::class))->toBeInstanceOf(InMemoryResilienceStore::class); +}); diff --git a/packages/resilience/tests/Store/ResilienceStoreSelectionTest.php b/packages/resilience/tests/Store/ResilienceStoreSelectionTest.php new file mode 100644 index 00000000..88e19037 --- /dev/null +++ b/packages/resilience/tests/Store/ResilienceStoreSelectionTest.php @@ -0,0 +1,69 @@ + $store + */ +function w4ResilienceConfig(array $store = []): Config +{ + return new Config(new Repository(['firefly' => ['resilience' => ['store' => $store]]])); +} + +it('keeps the cache store as the default', function () { + expect((new ResilienceAutoConfiguration)->resilienceStore(new CacheRepository(new ArrayStore), w4ResilienceConfig())) + ->toBeInstanceOf(CacheResilienceStore::class); +}); + +it('selects the in-memory store with provider=memory, whatever the cache is', function () { + expect((new ResilienceAutoConfiguration)->resilienceStore(new CacheRepository(new LocklessStore), w4ResilienceConfig(['provider' => 'memory']))) + ->toBeInstanceOf(InMemoryResilienceStore::class); +}); + +it('refuses an unknown provider, naming the key', function () { + expect(fn () => (new ResilienceAutoConfiguration)->resilienceStore(new CacheRepository(new ArrayStore), w4ResilienceConfig(['provider' => 'redis']))) + ->toThrow(ConfigurationException::class, 'firefly.resilience.store.provider'); +}); + +it('refuses a cache store without atomic locks unless allow-lockless is set', function () { + try { + (new ResilienceAutoConfiguration)->resilienceStore(new CacheRepository(new LocklessStore), w4ResilienceConfig()); + Assert::fail('a lockless store was accepted'); + } catch (ConfigurationException $e) { + expect($e->errorCode())->toBe('RESILIENCE_STORE_LOCKLESS') + ->and($e->getMessage())->toContain('firefly.resilience.store.allow-lockless'); + } +}); + +it('accepts a lockless store when allowed, runs the section unlocked and says so once', function () { + $logger = new RecordingLogger; + $store = (new ResilienceAutoConfiguration)->resilienceStore(new CacheRepository(new LocklessStore), w4ResilienceConfig(['allow-lockless' => true]), $logger); + + $first = $store->withLock('k', 1.0, static fn (): int => 1); + $second = $store->withLock('k', 1.0, static fn (): int => 2); + $warnings = $logger->warnings(); + + expect($first)->toBe(1) + ->and($second)->toBe(2) + ->and($warnings)->toHaveCount(1) + ->and($warnings[0] ?? '')->toContain('without a critical section'); +}); + +it('no longer degrades silently when constructed directly over a lockless store', function () { + $store = new CacheResilienceStore(new CacheRepository(new LocklessStore)); + + expect(fn () => $store->withLock('k', 1.0, static fn (): int => 1)) + ->toThrow(ConfigurationException::class, 'has no atomic locks'); +}); diff --git a/packages/resilience/tests/Support/RecordingLogger.php b/packages/resilience/tests/Support/RecordingLogger.php new file mode 100644 index 00000000..41712212 --- /dev/null +++ b/packages/resilience/tests/Support/RecordingLogger.php @@ -0,0 +1,32 @@ + */ + public array $records = []; + + /** + * @param array $context + */ + public function log($level, string|Stringable $message, array $context = []): void + { + $this->records[] = ['level' => is_string($level) ? $level : 'unknown', 'message' => (string) $message]; + } + + /** @return list */ + public function warnings(): array + { + return array_values(array_map( + static fn (array $record): string => $record['message'], + array_filter($this->records, static fn (array $record): bool => $record['level'] === 'warning'), + )); + } +} From 695c9f19dfcc41b449d701be5c4ad7f6d600c5f0 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 19:12:56 -0700 Subject: [PATCH 15/47] docs(config): document firefly.resilience.store provider and allow-lockless in the skeleton and the resilience module --- docs/modules/resilience.md | 23 ++++++++++++++++++++--- skeleton/config/firefly.php | 14 ++++++++++++++ 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/docs/modules/resilience.md b/docs/modules/resilience.md index c394eff9..9034cdaa 100644 --- a/docs/modules/resilience.md +++ b/docs/modules/resilience.md @@ -57,6 +57,8 @@ several of those example values are deliberately not the framework default (the // … 'store' => [ 'lock-block-timeout' => '500ms', + 'provider' => env('FIREFLY_RESILIENCE_STORE_PROVIDER', 'cache'), + 'allow-lockless' => env('FIREFLY_RESILIENCE_STORE_ALLOW_LOCKLESS', false), ], // … // 'retry' => [ @@ -371,6 +373,21 @@ injected `Illuminate\Contracts\Cache\Repository`. `ResilienceRegistry` itself is `resilience` config section still boots a working, empty registry — install the package and it's live with no required configuration. +### Choosing the store + +| Key | Type | Default | Meaning | +|---|---|---|---| +| `firefly.resilience.store.provider` | `cache` \| `memory` | `cache` | `cache` keeps the state on Laravel's default cache store, shared by every worker; `memory` keeps it in the process (`InMemoryResilienceStore`: tests, CLI, a single long-running worker). Anything else refuses to boot, naming the key. | +| `firefly.resilience.store.allow-lockless` | bool | `false` | Lets the `cache` provider run on a store without atomic locks, accepting the race between workers. | + +The `cache` provider needs a store that implements `Illuminate\Contracts\Cache\LockProvider` (redis, database, +file, memcached, dynamodb, array), because every breaker, limiter and bulkhead transition is a read-then-write +that must be a real critical section. The store bean is an eager singleton, so on a store without locks (apc, +session) the application refuses to **boot** with `ConfigurationException` `RESILIENCE_STORE_LOCKLESS` rather +than over-admitting under the first burst of load. `allow-lockless=true` accepts the race instead: the store +runs each section unlocked and logs one warning per store. A `CacheResilienceStore` constructed by hand over a +lockless cache throws the same exception from `withLock()` unless it is built with `allowLockless: true`. + ### The mutex wait budget `withLock()` separates two numbers that are easy to conflate: @@ -416,9 +433,9 @@ These are carried-forward, documented limitations of the M7 shipment — not bug store to actually survive across FPM requests and stay race-free: `file`, `database`, and `redis` qualify. The `array` driver is **per-request** (each FPM request gets a fresh in-memory array), so breaker/limiter/ bulkhead state resets every request — fine for tests and single-shot CLI runs, not for production - cross-request behaviour. The `null` cache driver's locks are no-ops (`withLock()` degrades to running the - callback without a critical section whenever the driver isn't a `LockProvider`), so it must not be used in - production for these patterns either. + cross-request behaviour. The `null` cache driver persists nothing and its locks are no-ops, so it must not + be used in production for these patterns either. A driver that is not a `LockProvider` at all is refused at + boot (see [Choosing the store](#choosing-the-store)). - **An idle record is reclaimed on the cache's schedule, not on a sweep the framework runs.** Every cache-backed pattern now writes its record with an expiry refreshed on every write — `idle-ttl` for [`CircuitBreaker`](#circuitbreaker) and [`RateLimiter`](#ratelimiter), `permit-ttl` for diff --git a/skeleton/config/firefly.php b/skeleton/config/firefly.php index 5d652510..c94eab70 100644 --- a/skeleton/config/firefly.php +++ b/skeleton/config/firefly.php @@ -2081,9 +2081,23 @@ | availability zone can legitimately need tens of milliseconds. | | Default: 0.5 (500ms). + | + | provider — where breaker, limiter and bulkhead state lives. `cache` keeps it on Laravel's default + | cache store, so every FPM worker and queue process shares one view; `memory` keeps it in the + | process (tests, CLI, a single long-running worker). An application that binds its own + | ResilienceStore wins over both. + | + | allow-lockless — the `cache` provider needs a store with atomic locks (redis, database, file, + | memcached, dynamodb, array) so a transition is a real critical section. On a store without them + | (apc, session) the application refuses to boot with RESILIENCE_STORE_LOCKLESS; true accepts the + | race between workers instead, logging one warning per store. + | + | Default: provider 'cache', allow-lockless false. */ 'store' => [ 'lock-block-timeout' => '500ms', + 'provider' => env('FIREFLY_RESILIENCE_STORE_PROVIDER', 'cache'), + 'allow-lockless' => env('FIREFLY_RESILIENCE_STORE_ALLOW_LOCKLESS', false), ], /* From b0dbf1577e9b20934acd4f342114ac26809a394c Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 19:20:00 -0700 Subject: [PATCH 16/47] feat(actuator): opt-in health indicators for cache stores, queue connections and Redis --- composer.json | 1 + .../cache/firefly-actuator-components.php | 90 ++++++++++-- .../cache/firefly-actuator-context.php | 67 ++++++++- .../src/Health/CacheHealthIndicator.php | 89 +++++++++++ .../src/Health/QueueHealthIndicator.php | 76 ++++++++++ .../src/Health/RedisHealthIndicator.php | 74 ++++++++++ .../tests/CapstoneBrokenCacheHealthTest.php | 16 ++ ...apstoneInfrastructureHealthDefaultTest.php | 16 ++ .../CapstoneInfrastructureHealthTest.php | 18 +++ .../Health/InfrastructureIndicatorsTest.php | 138 ++++++++++++++++++ .../RedisHealthIntegrationTest.php | 16 ++ .../UnreachableRedisHealthIntegrationTest.php | 17 +++ .../BrokenCacheHealthCapstoneTestCase.php | 19 +++ .../InfrastructureHealthCapstoneTestCase.php | 23 +++ .../Support/RedisHealthCapstoneTestCase.php | 30 ++++ ...UnreachableRedisHealthCapstoneTestCase.php | 15 ++ 16 files changed, 688 insertions(+), 17 deletions(-) create mode 100644 packages/actuator/src/Health/CacheHealthIndicator.php create mode 100644 packages/actuator/src/Health/QueueHealthIndicator.php create mode 100644 packages/actuator/src/Health/RedisHealthIndicator.php create mode 100644 packages/actuator/tests/CapstoneBrokenCacheHealthTest.php create mode 100644 packages/actuator/tests/CapstoneInfrastructureHealthDefaultTest.php create mode 100644 packages/actuator/tests/CapstoneInfrastructureHealthTest.php create mode 100644 packages/actuator/tests/Health/InfrastructureIndicatorsTest.php create mode 100644 packages/actuator/tests/Integration/RedisHealthIntegrationTest.php create mode 100644 packages/actuator/tests/Integration/UnreachableRedisHealthIntegrationTest.php create mode 100644 packages/actuator/tests/Support/BrokenCacheHealthCapstoneTestCase.php create mode 100644 packages/actuator/tests/Support/InfrastructureHealthCapstoneTestCase.php create mode 100644 packages/actuator/tests/Support/RedisHealthCapstoneTestCase.php create mode 100644 packages/actuator/tests/Support/UnreachableRedisHealthCapstoneTestCase.php diff --git a/composer.json b/composer.json index b0e106a3..1d1f918d 100644 --- a/composer.json +++ b/composer.json @@ -50,6 +50,7 @@ "pestphp/pest": "^4.5", "pestphp/pest-plugin-browser": "^4.3", "phpstan/phpstan": "^2.1", + "predis/predis": "^2.3 || ^3.0", "symplify/monorepo-builder": "^11.2" }, "autoload-dev": { diff --git a/packages/actuator/cache/firefly-actuator-components.php b/packages/actuator/cache/firefly-actuator-components.php index 896f160c..8744309d 100644 --- a/packages/actuator/cache/firefly-actuator-components.php +++ b/packages/actuator/cache/firefly-actuator-components.php @@ -69,6 +69,26 @@ ], ], 1 => [ + 'class' => 'Firefly\\Actuator\\Health\\CacheHealthIndicator', + 'stereotype' => 'component', + 'name' => null, + 'scope' => 'Singleton', + 'primary' => false, + 'order' => 0, + 'qualifier' => null, + 'interfaces' => [ + 0 => 'Firefly\\Actuator\\Health\\ConditionalHealthIndicator', + 1 => 'Firefly\\Actuator\\Health\\HealthIndicator', + ], + 'beans' => [ + ], + 'lazy' => false, + 'dependencies' => [ + 0 => 'Illuminate\\Contracts\\Cache\\Factory', + 1 => 'Firefly\\Config\\Config', + ], + ], + 2 => [ 'class' => 'Firefly\\Actuator\\Health\\DbHealthIndicator', 'stereotype' => 'component', 'name' => null, @@ -88,7 +108,7 @@ 1 => 'Firefly\\Config\\Config', ], ], - 2 => [ + 3 => [ 'class' => 'Firefly\\Actuator\\Health\\DiskSpaceHealthIndicator', 'stereotype' => 'component', 'name' => null, @@ -106,7 +126,7 @@ 0 => 'Firefly\\Config\\Config', ], ], - 3 => [ + 4 => [ 'class' => 'Firefly\\Actuator\\Health\\HealthEndpoint', 'stereotype' => 'component', 'name' => null, @@ -127,7 +147,7 @@ 3 => 'Firefly\\Actuator\\Health\\HealthDetailsAuthorizer', ], ], - 4 => [ + 5 => [ 'class' => 'Firefly\\Actuator\\Health\\PingHealthIndicator', 'stereotype' => 'component', 'name' => null, @@ -144,7 +164,47 @@ 'dependencies' => [ ], ], - 5 => [ + 6 => [ + 'class' => 'Firefly\\Actuator\\Health\\QueueHealthIndicator', + 'stereotype' => 'component', + 'name' => null, + 'scope' => 'Singleton', + 'primary' => false, + 'order' => 0, + 'qualifier' => null, + 'interfaces' => [ + 0 => 'Firefly\\Actuator\\Health\\ConditionalHealthIndicator', + 1 => 'Firefly\\Actuator\\Health\\HealthIndicator', + ], + 'beans' => [ + ], + 'lazy' => false, + 'dependencies' => [ + 0 => 'Illuminate\\Contracts\\Queue\\Factory', + 1 => 'Firefly\\Config\\Config', + ], + ], + 7 => [ + 'class' => 'Firefly\\Actuator\\Health\\RedisHealthIndicator', + 'stereotype' => 'component', + 'name' => null, + 'scope' => 'Singleton', + 'primary' => false, + 'order' => 0, + 'qualifier' => null, + 'interfaces' => [ + 0 => 'Firefly\\Actuator\\Health\\ConditionalHealthIndicator', + 1 => 'Firefly\\Actuator\\Health\\HealthIndicator', + ], + 'beans' => [ + ], + 'lazy' => false, + 'dependencies' => [ + 0 => 'Illuminate\\Contracts\\Container\\Container', + 1 => 'Firefly\\Config\\Config', + ], + ], + 8 => [ 'class' => 'Firefly\\Actuator\\Info\\AppInfoContributor', 'stereotype' => 'component', 'name' => null, @@ -162,7 +222,7 @@ 0 => 'Firefly\\Config\\Config', ], ], - 6 => [ + 9 => [ 'class' => 'Firefly\\Actuator\\Info\\BuildInfoContributor', 'stereotype' => 'component', 'name' => null, @@ -180,7 +240,7 @@ 0 => 'Firefly\\Config\\Config', ], ], - 7 => [ + 10 => [ 'class' => 'Firefly\\Actuator\\Info\\InfoEndpoint', 'stereotype' => 'component', 'name' => null, @@ -198,7 +258,7 @@ 0 => 'Firefly\\Actuator\\Info\\InfoContributorRegistry', ], ], - 8 => [ + 11 => [ 'class' => 'Firefly\\Actuator\\Info\\RuntimeInfoContributor', 'stereotype' => 'component', 'name' => null, @@ -215,7 +275,7 @@ 'dependencies' => [ ], ], - 9 => [ + 12 => [ 'class' => 'Firefly\\Actuator\\Introspection\\BeansEndpoint', 'stereotype' => 'component', 'name' => null, @@ -233,7 +293,7 @@ 0 => 'Firefly\\Actuator\\Introspection\\BeansCatalog', ], ], - 10 => [ + 13 => [ 'class' => 'Firefly\\Actuator\\Introspection\\CachesEndpoint', 'stereotype' => 'component', 'name' => null, @@ -251,7 +311,7 @@ 0 => 'Illuminate\\Contracts\\Config\\Repository', ], ], - 11 => [ + 14 => [ 'class' => 'Firefly\\Actuator\\Introspection\\ConditionsEndpoint', 'stereotype' => 'component', 'name' => null, @@ -269,7 +329,7 @@ 0 => 'Firefly\\Context\\Condition\\ConditionEvaluationReport', ], ], - 12 => [ + 15 => [ 'class' => 'Firefly\\Actuator\\Introspection\\ConfigPropsEndpoint', 'stereotype' => 'component', 'name' => null, @@ -287,7 +347,7 @@ 0 => 'Illuminate\\Contracts\\Container\\Container', ], ], - 13 => [ + 16 => [ 'class' => 'Firefly\\Actuator\\Introspection\\EnvEndpoint', 'stereotype' => 'component', 'name' => null, @@ -305,7 +365,7 @@ 0 => 'Illuminate\\Contracts\\Config\\Repository', ], ], - 14 => [ + 17 => [ 'class' => 'Firefly\\Actuator\\Introspection\\LoggersEndpoint', 'stereotype' => 'component', 'name' => null, @@ -324,7 +384,7 @@ 1 => 'Illuminate\\Contracts\\Config\\Repository', ], ], - 15 => [ + 18 => [ 'class' => 'Firefly\\Actuator\\Introspection\\MappingsEndpoint', 'stereotype' => 'component', 'name' => null, @@ -342,7 +402,7 @@ 0 => 'Firefly\\Web\\Route\\RouteManifest', ], ], - 16 => [ + 19 => [ 'class' => 'Firefly\\Actuator\\Introspection\\ScheduledTasksEndpoint', 'stereotype' => 'component', 'name' => null, diff --git a/packages/actuator/cache/firefly-actuator-context.php b/packages/actuator/cache/firefly-actuator-context.php index 42b2cbe1..de9ca797 100644 --- a/packages/actuator/cache/firefly-actuator-context.php +++ b/packages/actuator/cache/firefly-actuator-context.php @@ -63,6 +63,27 @@ ], ], 1 => [ + 'class' => 'Firefly\\Actuator\\Health\\CacheHealthIndicator', + 'postConstruct' => [ + ], + 'preDestroy' => [ + ], + 'listeners' => [ + ], + 'conditions' => [ + 0 => [ + 'type' => 'Firefly\\Context\\Condition\\Attributes\\ConditionalOnProperty', + 'args' => [ + 0 => 'firefly.management.endpoint.health.cache.enabled', + 1 => 'true', + 2 => false, + ], + ], + ], + 'beanConditions' => [ + ], + ], + 2 => [ 'class' => 'Firefly\\Actuator\\Health\\DbHealthIndicator', 'postConstruct' => [ ], @@ -83,7 +104,49 @@ 'beanConditions' => [ ], ], - 2 => [ + 3 => [ + 'class' => 'Firefly\\Actuator\\Health\\QueueHealthIndicator', + 'postConstruct' => [ + ], + 'preDestroy' => [ + ], + 'listeners' => [ + ], + 'conditions' => [ + 0 => [ + 'type' => 'Firefly\\Context\\Condition\\Attributes\\ConditionalOnProperty', + 'args' => [ + 0 => 'firefly.management.endpoint.health.queue.enabled', + 1 => 'true', + 2 => false, + ], + ], + ], + 'beanConditions' => [ + ], + ], + 4 => [ + 'class' => 'Firefly\\Actuator\\Health\\RedisHealthIndicator', + 'postConstruct' => [ + ], + 'preDestroy' => [ + ], + 'listeners' => [ + ], + 'conditions' => [ + 0 => [ + 'type' => 'Firefly\\Context\\Condition\\Attributes\\ConditionalOnProperty', + 'args' => [ + 0 => 'firefly.management.endpoint.health.redis.enabled', + 1 => 'true', + 2 => false, + ], + ], + ], + 'beanConditions' => [ + ], + ], + 5 => [ 'class' => 'Firefly\\Actuator\\Info\\RuntimeInfoContributor', 'postConstruct' => [ ], @@ -104,7 +167,7 @@ 'beanConditions' => [ ], ], - 3 => [ + 6 => [ 'class' => 'Firefly\\Actuator\\Introspection\\ScheduledTasksEndpoint', 'postConstruct' => [ ], diff --git a/packages/actuator/src/Health/CacheHealthIndicator.php b/packages/actuator/src/Health/CacheHealthIndicator.php new file mode 100644 index 00000000..46f3b987 --- /dev/null +++ b/packages/actuator/src/Health/CacheHealthIndicator.php @@ -0,0 +1,89 @@ +config->get('cache.default'); + + return is_string($default) && $default !== ''; + } + + public function health(): Health + { + $stores = []; + $down = false; + + foreach ($this->stores() as $name) { + $driver = $this->config->get("cache.stores.{$name}.driver"); + $entry = ['status' => 'UP', 'driver' => is_string($driver) ? $driver : 'unknown']; + + try { + $cache = $this->caches->store($name); + $key = 'firefly:health:'.bin2hex(random_bytes(8)); + $value = bin2hex(random_bytes(4)); + $cache->put($key, $value, 60); + $read = $cache->get($key); + $cache->forget($key); + + if ($read !== $value) { + $entry = ['status' => 'DOWN', 'driver' => $entry['driver'], 'error' => 'the store did not return the value just written']; + } + } catch (Throwable $e) { + $entry = ['status' => 'DOWN', 'driver' => $entry['driver'], 'error' => $e::class.': '.$e->getMessage()]; + } + + $down = $down || $entry['status'] === 'DOWN'; + $stores[$name] = $entry; + } + + return $down ? Health::down(['stores' => $stores]) : Health::up(['stores' => $stores]); + } + + /** + * @return list + */ + private function stores(): array + { + $configured = array_values(array_filter( + $this->config->array('firefly.management.endpoint.health.cache.stores', []), + static fn (mixed $name): bool => is_string($name) && $name !== '', + )); + + if ($configured !== []) { + /** @var list $configured */ + return $configured; + } + + $default = $this->config->get('cache.default'); + + return is_string($default) && $default !== '' ? [$default] : []; + } +} diff --git a/packages/actuator/src/Health/QueueHealthIndicator.php b/packages/actuator/src/Health/QueueHealthIndicator.php new file mode 100644 index 00000000..e06f2993 --- /dev/null +++ b/packages/actuator/src/Health/QueueHealthIndicator.php @@ -0,0 +1,76 @@ +config->get('queue.default'); + + return is_string($default) && $default !== ''; + } + + public function health(): Health + { + $connections = []; + $down = false; + + foreach ($this->connections() as $name) { + $driver = $this->config->get("queue.connections.{$name}.driver"); + $driver = is_string($driver) ? $driver : 'unknown'; + + try { + $connections[$name] = ['status' => 'UP', 'driver' => $driver, 'size' => $this->queues->connection($name)->size()]; + } catch (Throwable $e) { + $down = true; + $connections[$name] = ['status' => 'DOWN', 'driver' => $driver, 'error' => $e::class.': '.$e->getMessage()]; + } + } + + return $down ? Health::down(['connections' => $connections]) : Health::up(['connections' => $connections]); + } + + /** + * @return list + */ + private function connections(): array + { + $configured = array_values(array_filter( + $this->config->array('firefly.management.endpoint.health.queue.connections', []), + static fn (mixed $name): bool => is_string($name) && $name !== '', + )); + + if ($configured !== []) { + /** @var list $configured */ + return $configured; + } + + $default = $this->config->get('queue.default'); + + return is_string($default) && $default !== '' ? [$default] : []; + } +} diff --git a/packages/actuator/src/Health/RedisHealthIndicator.php b/packages/actuator/src/Health/RedisHealthIndicator.php new file mode 100644 index 00000000..745713c0 --- /dev/null +++ b/packages/actuator/src/Health/RedisHealthIndicator.php @@ -0,0 +1,74 @@ +container->bound('redis') && is_array($this->config->get('database.redis')); + } + + public function health(): Health + { + $connections = []; + $down = false; + + try { + $redis = $this->container->make(Factory::class); + } catch (Throwable $e) { + return Health::down(['error' => $e::class.': '.$e->getMessage()]); + } + + foreach ($this->connections() as $name) { + try { + $redis->connection($name)->command('ping'); + $connections[$name] = ['status' => 'UP']; + } catch (Throwable $e) { + $down = true; + $connections[$name] = ['status' => 'DOWN', 'error' => $e::class.': '.$e->getMessage()]; + } + } + + return $down ? Health::down(['connections' => $connections]) : Health::up(['connections' => $connections]); + } + + /** + * @return list + */ + private function connections(): array + { + $configured = array_values(array_filter( + $this->config->array('firefly.management.endpoint.health.redis.connections', ['default']), + static fn (mixed $name): bool => is_string($name) && $name !== '', + )); + + /** @var list $configured */ + return $configured === [] ? ['default'] : $configured; + } +} diff --git a/packages/actuator/tests/CapstoneBrokenCacheHealthTest.php b/packages/actuator/tests/CapstoneBrokenCacheHealthTest.php new file mode 100644 index 00000000..e10da127 --- /dev/null +++ b/packages/actuator/tests/CapstoneBrokenCacheHealthTest.php @@ -0,0 +1,16 @@ +getJson('/actuator/health') + ->assertStatus(503) + ->assertJsonPath('components.cache.status', 'DOWN') + ->assertJsonPath('components.cache.details.stores.array.status', 'UP') + ->assertJsonPath('components.cache.details.stores.broken.status', 'DOWN'); +}); diff --git a/packages/actuator/tests/CapstoneInfrastructureHealthDefaultTest.php b/packages/actuator/tests/CapstoneInfrastructureHealthDefaultTest.php new file mode 100644 index 00000000..45091075 --- /dev/null +++ b/packages/actuator/tests/CapstoneInfrastructureHealthDefaultTest.php @@ -0,0 +1,16 @@ +getJson('/actuator/health') + ->assertStatus(200) + ->assertJsonMissingPath('components.cache') + ->assertJsonMissingPath('components.queue') + ->assertJsonMissingPath('components.redis'); +}); diff --git a/packages/actuator/tests/CapstoneInfrastructureHealthTest.php b/packages/actuator/tests/CapstoneInfrastructureHealthTest.php new file mode 100644 index 00000000..92ad4af3 --- /dev/null +++ b/packages/actuator/tests/CapstoneInfrastructureHealthTest.php @@ -0,0 +1,18 @@ +getJson('/actuator/health') + ->assertStatus(200) + ->assertJsonPath('components.cache.status', 'UP') + ->assertJsonPath('components.cache.details.stores.array.status', 'UP') + ->assertJsonPath('components.queue.status', 'UP') + ->assertJsonPath('components.queue.details.connections.sync.size', 0) + ->assertJsonMissingPath('components.redis'); +}); diff --git a/packages/actuator/tests/Health/InfrastructureIndicatorsTest.php b/packages/actuator/tests/Health/InfrastructureIndicatorsTest.php new file mode 100644 index 00000000..32ce7eb4 --- /dev/null +++ b/packages/actuator/tests/Health/InfrastructureIndicatorsTest.php @@ -0,0 +1,138 @@ + $stores + */ +function w4Caches(array $stores): CacheFactory +{ + return new class($stores) implements CacheFactory + { + /** @param array $stores */ + public function __construct(private readonly array $stores) {} + + public function store($name = null): CacheContract + { + $key = is_string($name) ? $name : 'array'; + + return $this->stores[$key] ?? throw new InvalidArgumentException("Cache store [{$key}] is not defined."); + } + }; +} + +/** + * @param array $queues + */ +function w4Queues(array $queues): QueueFactory +{ + return new class($queues) implements QueueFactory + { + /** @param array $queues */ + public function __construct(private readonly array $queues) {} + + public function connection($name = null): Queue + { + $key = is_string($name) ? $name : 'sync'; + + return $this->queues[$key] ?? throw new InvalidArgumentException("Queue connection [{$key}] is not configured."); + } + }; +} + +/** + * Walks a health reading's nested details (`stores..status`), so an assertion names the path it reads. + */ +function w4HealthDetail(Health $health, string ...$path): mixed +{ + $value = $health->details; + + foreach ($path as $segment) { + if (! is_array($value) || ! array_key_exists($segment, $value)) { + return null; + } + + $value = $value[$segment]; + } + + return $value; +} + +/** + * @param array $values + */ +function w4HealthConfig(array $values): Config +{ + return new Config(new Repository($values)); +} + +it('gates every indicator off by default behind its own key', function () { + foreach ([CacheHealthIndicator::class => 'cache', QueueHealthIndicator::class => 'queue', RedisHealthIndicator::class => 'redis'] as $class => $name) { + $condition = (new ReflectionClass($class))->getAttributes(ConditionalOnProperty::class)[0]->newInstance(); + + expect($condition->name)->toBe("firefly.management.endpoint.health.{$name}.enabled") + ->and($condition->havingValue)->toBe('true') + ->and($condition->matchIfMissing)->toBeFalse() + ->and(class_implements($class))->toHaveKey(ConditionalHealthIndicator::class); + } +}); + +it('reports every configured cache store UP after a write-read-forget round trip', function () { + $config = w4HealthConfig(['cache' => ['default' => 'array', 'stores' => ['array' => ['driver' => 'array'], 'second' => ['driver' => 'array']]], 'firefly' => ['management' => ['endpoint' => ['health' => ['cache' => ['stores' => ['array', 'second']]]]]]]); + $indicator = new CacheHealthIndicator(w4Caches(['array' => new CacheRepository(new ArrayStore), 'second' => new CacheRepository(new ArrayStore)]), $config); + $health = $indicator->health(); + + expect($indicator->available())->toBeTrue() + ->and($health->status)->toBe(Status::Up) + ->and(w4HealthDetail($health, 'stores', 'array', 'status'))->toBe('UP') + ->and(w4HealthDetail($health, 'stores', 'second', 'driver'))->toBe('array'); +}); + +it('reports DOWN, naming the store and the error, when one store fails', function () { + $config = w4HealthConfig(['cache' => ['default' => 'array', 'stores' => ['array' => ['driver' => 'array']]], 'firefly' => ['management' => ['endpoint' => ['health' => ['cache' => ['stores' => ['array', 'missing']]]]]]]); + $health = (new CacheHealthIndicator(w4Caches(['array' => new CacheRepository(new ArrayStore)]), $config))->health(); + + expect($health->status)->toBe(Status::Down) + ->and(w4HealthDetail($health, 'stores', 'array', 'status'))->toBe('UP') + ->and(w4HealthDetail($health, 'stores', 'missing', 'status'))->toBe('DOWN') + ->and(w4HealthDetail($health, 'stores', 'missing', 'error'))->toContain('not defined'); +}); + +it('is unavailable when no default cache store is configured', function () { + expect((new CacheHealthIndicator(w4Caches([]), w4HealthConfig(['cache' => []])))->available())->toBeFalse(); +}); + +it('reports queue connections with their size, DOWN when one cannot be reached', function () { + $config = w4HealthConfig(['queue' => ['default' => 'sync', 'connections' => ['sync' => ['driver' => 'sync']]], 'firefly' => ['management' => ['endpoint' => ['health' => ['queue' => ['connections' => ['sync', 'redis']]]]]]]); + $indicator = new QueueHealthIndicator(w4Queues(['sync' => new SyncQueue]), $config); + $health = $indicator->health(); + + expect($indicator->available())->toBeTrue() + ->and($health->status)->toBe(Status::Down) + ->and(w4HealthDetail($health, 'connections', 'sync'))->toBe(['status' => 'UP', 'driver' => 'sync', 'size' => 0]) + ->and(w4HealthDetail($health, 'connections', 'redis', 'status'))->toBe('DOWN'); +}); + +it('declines when no redis binding exists', function () { + $indicator = new RedisHealthIndicator(new Container, w4HealthConfig(['database' => ['redis' => ['default' => ['host' => '127.0.0.1']]]])); + + expect($indicator->available())->toBeFalse(); +}); diff --git a/packages/actuator/tests/Integration/RedisHealthIntegrationTest.php b/packages/actuator/tests/Integration/RedisHealthIntegrationTest.php new file mode 100644 index 00000000..2d6d2b23 --- /dev/null +++ b/packages/actuator/tests/Integration/RedisHealthIntegrationTest.php @@ -0,0 +1,16 @@ +getJson('/actuator/health') + ->assertStatus(200) + ->assertJsonPath('components.redis.status', 'UP') + ->assertJsonPath('components.redis.details.connections.default.status', 'UP'); +})->group('integration')->skip(getenv('FIREFLY_REDIS_URL') === false || ! class_exists(Client::class), 'Set FIREFLY_REDIS_URL (and install predis/predis) to run the Redis health integration test.'); diff --git a/packages/actuator/tests/Integration/UnreachableRedisHealthIntegrationTest.php b/packages/actuator/tests/Integration/UnreachableRedisHealthIntegrationTest.php new file mode 100644 index 00000000..a31f9d82 --- /dev/null +++ b/packages/actuator/tests/Integration/UnreachableRedisHealthIntegrationTest.php @@ -0,0 +1,17 @@ +getJson('/actuator/health') + ->assertStatus(503) + ->assertJsonPath('components.redis.status', 'DOWN') + ->assertJsonPath('components.redis.details.connections.default.status', 'UP') + ->assertJsonPath('components.redis.details.connections.unreachable.status', 'DOWN'); +})->group('integration')->skip(getenv('FIREFLY_REDIS_URL') === false || ! class_exists(Client::class), 'Set FIREFLY_REDIS_URL (and install predis/predis) to run the Redis health integration test.'); diff --git a/packages/actuator/tests/Support/BrokenCacheHealthCapstoneTestCase.php b/packages/actuator/tests/Support/BrokenCacheHealthCapstoneTestCase.php new file mode 100644 index 00000000..4a57195f --- /dev/null +++ b/packages/actuator/tests/Support/BrokenCacheHealthCapstoneTestCase.php @@ -0,0 +1,19 @@ + */ + protected function configOverrides(): array + { + return [ + ...parent::configOverrides(), + 'cache.stores.broken' => ['driver' => 'file', 'path' => '/dev/null/firefly-health'], + 'firefly.management.endpoint.health.cache.stores' => ['array', 'broken'], + ]; + } +} diff --git a/packages/actuator/tests/Support/InfrastructureHealthCapstoneTestCase.php b/packages/actuator/tests/Support/InfrastructureHealthCapstoneTestCase.php new file mode 100644 index 00000000..331140f1 --- /dev/null +++ b/packages/actuator/tests/Support/InfrastructureHealthCapstoneTestCase.php @@ -0,0 +1,23 @@ + */ + protected function configOverrides(): array + { + return [ + 'firefly.management.enabled' => true, + 'firefly.management.endpoint.health.show-details' => 'always', + 'firefly.management.endpoints.web.exposure.include' => 'health,info', + 'firefly.management.endpoint.health.cache.enabled' => true, + 'firefly.management.endpoint.health.queue.enabled' => true, + 'cache.default' => 'array', + 'queue.default' => 'sync', + ]; + } +} diff --git a/packages/actuator/tests/Support/RedisHealthCapstoneTestCase.php b/packages/actuator/tests/Support/RedisHealthCapstoneTestCase.php new file mode 100644 index 00000000..ce599673 --- /dev/null +++ b/packages/actuator/tests/Support/RedisHealthCapstoneTestCase.php @@ -0,0 +1,30 @@ + */ + protected function configOverrides(): array + { + return [ + 'firefly.management.enabled' => true, + 'firefly.management.endpoint.health.show-details' => 'always', + 'firefly.management.endpoints.web.exposure.include' => 'health,info', + 'firefly.management.endpoint.health.redis.enabled' => true, + 'firefly.management.endpoint.health.redis.connections' => $this->redisConnections(), + 'database.redis.client' => 'predis', + 'database.redis.default' => ['url' => (string) getenv('FIREFLY_REDIS_URL')], + 'database.redis.unreachable' => ['host' => '127.0.0.1', 'port' => 1, 'timeout' => 0.5, 'read_write_timeout' => 0.5], + ]; + } + + /** @return list */ + protected function redisConnections(): array + { + return ['default']; + } +} diff --git a/packages/actuator/tests/Support/UnreachableRedisHealthCapstoneTestCase.php b/packages/actuator/tests/Support/UnreachableRedisHealthCapstoneTestCase.php new file mode 100644 index 00000000..9dbc8002 --- /dev/null +++ b/packages/actuator/tests/Support/UnreachableRedisHealthCapstoneTestCase.php @@ -0,0 +1,15 @@ + */ + protected function redisConnections(): array + { + return ['default', 'unreachable']; + } +} From d99a0344b92bc4fc8ad45089c3297c6e0f547ac0 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 19:20:00 -0700 Subject: [PATCH 17/47] docs(config): document the cache, queue and redis health indicator keys in the skeleton and the actuator module --- docs/modules/actuator.md | 1 + skeleton/config/firefly.php | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/docs/modules/actuator.md b/docs/modules/actuator.md index 1921de98..eccf3234 100644 --- a/docs/modules/actuator.md +++ b/docs/modules/actuator.md @@ -132,6 +132,7 @@ data browser's sensitive columns, and `page_header` is page furniture, not a cre - `firefly.management.endpoint.health.roles` (default `[]`) — who `when-authorized` admits, Spring's `management.endpoint.health.roles`. **Empty means any AUTHENTICATED principal**; a non-empty list means one holding at least one of these roles, a bare name read as `ROLE_` (the spelling `hasRole:` uses) and the configured role hierarchy applied. A **CSV string is the same restriction as the list** — `'ADMIN,ACTUATOR'` grants what `['ADMIN', 'ACTUATOR']` grants, Spring's own spelling of the property and the one both neighbouring list keys above use. A list whose entries are ALL unusable — a blank string, a null left by a dangling key — **refuses every caller** rather than being read as the empty list, because a typo must not widen the surface it was written to narrow; a value that is neither a list nor a string counts as one unusable entry and refuses the same way, never failing the read (a probe must not get a 500 out of a config typo). The refusal is logged once per boot, naming this key. Read only by `firefly/security`'s authorizer: without that package installed and its master flag on, the key does nothing - `firefly.management.endpoint.health.group.{name}.include` - `firefly.management.endpoint.health.db.enabled` (default **`true`**) — the `db` health indicator, registered whenever `database.default` names a connection with a driver (Spring Boot's `DataSourceHealthIndicator` auto-configuration); a failing or missing database reports DOWN and `/actuator/health` answers 503; an application with no default database gets no `db` component at all. `false` removes the indicator. An indicator can decline registration itself by implementing `ConditionalHealthIndicator::available()`. +- `firefly.management.endpoint.health.cache.enabled` / `.queue.enabled` / `.redis.enabled` (each default **`false`**) — the opt-in infrastructure indicators, off so an upgrade never changes an existing application's `/health` status. `cache` writes, reads back and forgets a random sentinel key on each store in `firefly.management.endpoint.health.cache.stores` (default `[]`, meaning `cache.default`); `queue` publishes `size()` of each connection in `firefly.management.endpoint.health.queue.connections` (default `[]`, meaning `queue.default`); `redis` sends PING on each connection in `firefly.management.endpoint.health.redis.connections` (default `['default']`) through Laravel's Redis manager, phpredis or predis, and declines registration when the application has no `redis` binding. One store or connection DOWN makes its component DOWN, named in `details.stores` / `details.connections` with the error, and `/actuator/health` answers 503. - `firefly.management.info.app.*`, `firefly.management.info.build.path` - `firefly.management.info.runtime.enabled` (default `true`, `#[ConditionalOnProperty(matchIfMissing: true)]`) — the `runtime` fragment of `/actuator/info` (PHP version/SAPI/OPcache, Laravel version, LaraFly version, current+peak memory). Setting it `false` removes the contributor bean entirely. diff --git a/skeleton/config/firefly.php b/skeleton/config/firefly.php index c94eab70..68e046a2 100644 --- a/skeleton/config/firefly.php +++ b/skeleton/config/firefly.php @@ -849,6 +849,27 @@ // 'threshold' => 10485760, // ], + /* + | Infrastructure indicators, all OFF BY DEFAULT: turning one on adds a component that can + | be DOWN, and an existing application's /health must not change status because the + | framework was upgraded. + | + | `cache` writes, reads back and forgets a random sentinel key (one-minute TTL) on each + | Laravel cache store in `stores`; an empty list probes `cache.default`. `queue` asks each + | Laravel queue connection in `connections` for size() of its default queue and publishes + | the backlog as a detail; an empty list probes `queue.default`. `redis` sends PING on each + | connection of Laravel's Redis manager in `connections` (phpredis or predis, whichever + | `database.redis.client` names); an application with no `redis` binding gets no `redis` + | component even with the key on. Any one store or connection DOWN makes its component + | DOWN, naming it and the error. + | + | Defaults: cache.enabled false, cache.stores [] (cache.default); queue.enabled false, + | queue.connections [] (queue.default); redis.enabled false, redis.connections ['default']. + */ + // 'cache' => ['enabled' => true, 'stores' => ['redis', 'file']], + // 'queue' => ['enabled' => true, 'connections' => ['database']], + // 'redis' => ['enabled' => true, 'connections' => ['default', 'cache']], + /* | Probe groups served at /actuator/health/{name} — CSV of indicator names. An UNSET | group is a 404, so these must stay commented out until you mean them. From 1e2ee76dfac6141d89291c779bcf4e9fa420a957 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 19:24:27 -0700 Subject: [PATCH 18/47] fix(config): wire the cache, queue and redis health gates to their FIREFLY_HEALTH_*_ENABLED variables in the skeleton --- docs/modules/actuator.md | 2 +- ...keletonInfrastructureHealthDefaultTest.php | 21 ++++++ ...pstoneSkeletonInfrastructureHealthTest.php | 23 ++++++ ...onInfrastructureHealthCapstoneTestCase.php | 70 +++++++++++++++++++ ...structureHealthDefaultCapstoneTestCase.php | 14 ++++ ...structureHealthEnabledCapstoneTestCase.php | 17 +++++ skeleton/config/firefly.php | 24 ++++++- 7 files changed, 167 insertions(+), 4 deletions(-) create mode 100644 packages/actuator/tests/CapstoneSkeletonInfrastructureHealthDefaultTest.php create mode 100644 packages/actuator/tests/CapstoneSkeletonInfrastructureHealthTest.php create mode 100644 packages/actuator/tests/Support/SkeletonInfrastructureHealthCapstoneTestCase.php create mode 100644 packages/actuator/tests/Support/SkeletonInfrastructureHealthDefaultCapstoneTestCase.php create mode 100644 packages/actuator/tests/Support/SkeletonInfrastructureHealthEnabledCapstoneTestCase.php diff --git a/docs/modules/actuator.md b/docs/modules/actuator.md index eccf3234..6a2ad281 100644 --- a/docs/modules/actuator.md +++ b/docs/modules/actuator.md @@ -132,7 +132,7 @@ data browser's sensitive columns, and `page_header` is page furniture, not a cre - `firefly.management.endpoint.health.roles` (default `[]`) — who `when-authorized` admits, Spring's `management.endpoint.health.roles`. **Empty means any AUTHENTICATED principal**; a non-empty list means one holding at least one of these roles, a bare name read as `ROLE_` (the spelling `hasRole:` uses) and the configured role hierarchy applied. A **CSV string is the same restriction as the list** — `'ADMIN,ACTUATOR'` grants what `['ADMIN', 'ACTUATOR']` grants, Spring's own spelling of the property and the one both neighbouring list keys above use. A list whose entries are ALL unusable — a blank string, a null left by a dangling key — **refuses every caller** rather than being read as the empty list, because a typo must not widen the surface it was written to narrow; a value that is neither a list nor a string counts as one unusable entry and refuses the same way, never failing the read (a probe must not get a 500 out of a config typo). The refusal is logged once per boot, naming this key. Read only by `firefly/security`'s authorizer: without that package installed and its master flag on, the key does nothing - `firefly.management.endpoint.health.group.{name}.include` - `firefly.management.endpoint.health.db.enabled` (default **`true`**) — the `db` health indicator, registered whenever `database.default` names a connection with a driver (Spring Boot's `DataSourceHealthIndicator` auto-configuration); a failing or missing database reports DOWN and `/actuator/health` answers 503; an application with no default database gets no `db` component at all. `false` removes the indicator. An indicator can decline registration itself by implementing `ConditionalHealthIndicator::available()`. -- `firefly.management.endpoint.health.cache.enabled` / `.queue.enabled` / `.redis.enabled` (each default **`false`**) — the opt-in infrastructure indicators, off so an upgrade never changes an existing application's `/health` status. `cache` writes, reads back and forgets a random sentinel key on each store in `firefly.management.endpoint.health.cache.stores` (default `[]`, meaning `cache.default`); `queue` publishes `size()` of each connection in `firefly.management.endpoint.health.queue.connections` (default `[]`, meaning `queue.default`); `redis` sends PING on each connection in `firefly.management.endpoint.health.redis.connections` (default `['default']`) through Laravel's Redis manager, phpredis or predis, and declines registration when the application has no `redis` binding. One store or connection DOWN makes its component DOWN, named in `details.stores` / `details.connections` with the error, and `/actuator/health` answers 503. +- `firefly.management.endpoint.health.cache.enabled` / `.queue.enabled` / `.redis.enabled` (each default **`false`**; environment variables `FIREFLY_HEALTH_CACHE_ENABLED`, `FIREFLY_HEALTH_QUEUE_ENABLED`, `FIREFLY_HEALTH_REDIS_ENABLED` in the skeleton) — the opt-in infrastructure indicators, off so an upgrade never changes an existing application's `/health` status. `cache` writes, reads back and forgets a random sentinel key on each store in `firefly.management.endpoint.health.cache.stores` (default `[]`, meaning `cache.default`); `queue` publishes `size()` of each connection in `firefly.management.endpoint.health.queue.connections` (default `[]`, meaning `queue.default`); `redis` sends PING on each connection in `firefly.management.endpoint.health.redis.connections` (default `['default']`) through Laravel's Redis manager, phpredis or predis, and declines registration when the application has no `redis` binding. One store or connection DOWN makes its component DOWN, named in `details.stores` / `details.connections` with the error, and `/actuator/health` answers 503. - `firefly.management.info.app.*`, `firefly.management.info.build.path` - `firefly.management.info.runtime.enabled` (default `true`, `#[ConditionalOnProperty(matchIfMissing: true)]`) — the `runtime` fragment of `/actuator/info` (PHP version/SAPI/OPcache, Laravel version, LaraFly version, current+peak memory). Setting it `false` removes the contributor bean entirely. diff --git a/packages/actuator/tests/CapstoneSkeletonInfrastructureHealthDefaultTest.php b/packages/actuator/tests/CapstoneSkeletonInfrastructureHealthDefaultTest.php new file mode 100644 index 00000000..5a05987b --- /dev/null +++ b/packages/actuator/tests/CapstoneSkeletonInfrastructureHealthDefaultTest.php @@ -0,0 +1,21 @@ +getJson('/actuator/health') + ->assertStatus(200) + ->assertJsonPath('components.db.status', 'UP') + ->assertJsonMissingPath('components.cache') + ->assertJsonMissingPath('components.queue') + ->assertJsonMissingPath('components.redis'); +}); diff --git a/packages/actuator/tests/CapstoneSkeletonInfrastructureHealthTest.php b/packages/actuator/tests/CapstoneSkeletonInfrastructureHealthTest.php new file mode 100644 index 00000000..02584889 --- /dev/null +++ b/packages/actuator/tests/CapstoneSkeletonInfrastructureHealthTest.php @@ -0,0 +1,23 @@ +getJson('/actuator/health') + ->assertStatus(200) + ->assertJsonPath('components.cache.status', 'UP') + ->assertJsonPath('components.cache.details.stores.array.status', 'UP') + ->assertJsonPath('components.queue.status', 'UP') + ->assertJsonPath('components.queue.details.connections.sync.size', 0) + ->assertJsonMissingPath('components.redis'); +}); diff --git a/packages/actuator/tests/Support/SkeletonInfrastructureHealthCapstoneTestCase.php b/packages/actuator/tests/Support/SkeletonInfrastructureHealthCapstoneTestCase.php new file mode 100644 index 00000000..340afd24 --- /dev/null +++ b/packages/actuator/tests/Support/SkeletonInfrastructureHealthCapstoneTestCase.php @@ -0,0 +1,70 @@ + the environment variables set while the skeleton is evaluated */ + abstract protected function healthEnvironment(): array; + + /** @return array */ + protected function configOverrides(): array + { + $health = $this->skeletonHealth(); + $health['show-details'] = 'always'; + + return [ + ...parent::configOverrides(), + 'firefly.management.endpoint.health' => $health, + 'cache.default' => 'array', + 'queue.default' => 'sync', + ]; + } + + /** @return array */ + private function skeletonHealth(): array + { + $saved = []; + + foreach ($this->healthEnvironment() as $name => $value) { + $saved[$name] = [$_SERVER[$name] ?? null, $_ENV[$name] ?? null, getenv($name)]; + $_SERVER[$name] = $value; + $_ENV[$name] = $value; + putenv($name.'='.$value); + } + + try { + /** @var array{management: array{endpoint: array{health: array}}} $reference */ + $reference = require dirname(__DIR__, 4).'/skeleton/config/firefly.php'; + + return $reference['management']['endpoint']['health']; + } finally { + foreach ($saved as $name => [$server, $env, $process]) { + if ($server === null) { + unset($_SERVER[$name]); + } else { + $_SERVER[$name] = $server; + } + + if ($env === null) { + unset($_ENV[$name]); + } else { + $_ENV[$name] = $env; + } + + putenv($process === false ? $name : $name.'='.$process); + } + } + } +} diff --git a/packages/actuator/tests/Support/SkeletonInfrastructureHealthDefaultCapstoneTestCase.php b/packages/actuator/tests/Support/SkeletonInfrastructureHealthDefaultCapstoneTestCase.php new file mode 100644 index 00000000..cf5c00dc --- /dev/null +++ b/packages/actuator/tests/Support/SkeletonInfrastructureHealthDefaultCapstoneTestCase.php @@ -0,0 +1,14 @@ + 'true', + 'FIREFLY_HEALTH_QUEUE_ENABLED' => 'true', + ]; + } +} diff --git a/skeleton/config/firefly.php b/skeleton/config/firefly.php index 68e046a2..685349eb 100644 --- a/skeleton/config/firefly.php +++ b/skeleton/config/firefly.php @@ -863,12 +863,30 @@ | component even with the key on. Any one store or connection DOWN makes its component | DOWN, naming it and the error. | + | Each gate reads its own environment variable (FIREFLY_HEALTH_CACHE_ENABLED, + | FIREFLY_HEALTH_QUEUE_ENABLED, FIREFLY_HEALTH_REDIS_ENABLED), so an operator can turn + | one on without editing this file; the commented lines below are samples of the lists. + | | Defaults: cache.enabled false, cache.stores [] (cache.default); queue.enabled false, | queue.connections [] (queue.default); redis.enabled false, redis.connections ['default']. */ - // 'cache' => ['enabled' => true, 'stores' => ['redis', 'file']], - // 'queue' => ['enabled' => true, 'connections' => ['database']], - // 'redis' => ['enabled' => true, 'connections' => ['default', 'cache']], + 'cache' => [ + 'enabled' => env('FIREFLY_HEALTH_CACHE_ENABLED', false), + 'stores' => [ + // 'redis', 'file', + ], + ], + 'queue' => [ + 'enabled' => env('FIREFLY_HEALTH_QUEUE_ENABLED', false), + 'connections' => [ + // 'database', + ], + ], + 'redis' => [ + 'enabled' => env('FIREFLY_HEALTH_REDIS_ENABLED', false), + 'connections' => ['default'], + // 'connections' => ['default', 'cache'], + ], /* | Probe groups served at /actuator/health/{name} — CSV of indicator names. An UNSET From d76e4c9f48d9d8f6f3aba570a428563a471890f8 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 19:35:37 -0700 Subject: [PATCH 19/47] feat(messaging): a listener container contract with retry and back-off policies, and firefly:messaging:consume --- .../src/Boot/MessageListenerWiringPass.php | 11 ++ .../src/Console/ConsumeMessagesCommand.php | 119 ++++++++++++++++++ packages/messaging/src/Container/BackOff.php | 11 ++ .../src/Container/DeadLetterHeaders.php | 51 ++++++++ .../src/Container/DeliveryOutcome.php | 17 +++ .../src/Container/ExponentialBackOff.php | 20 +++ .../messaging/src/Container/LinearBackOff.php | 16 +++ .../src/Container/ListenerContainerBroker.php | 24 ++++ .../src/Container/ListenerInvoker.php | 58 +++++++++ .../src/Container/ListenerOptions.php | 32 +++++ .../messaging/src/Container/RetryPolicy.php | 53 ++++++++ packages/messaging/src/Container/Seconds.php | 40 ++++++ .../src/MessagingAutoConfiguration.php | 14 ++- .../messaging/src/MessagingWiringProvider.php | 10 +- ...stoneMessagingConsumeCommandMemoryTest.php | 13 ++ .../tests/CapstoneMessagingContainerTest.php | 50 ++++++++ .../tests/Container/ListenerInvokerTest.php | 49 ++++++++ .../tests/Container/RetryPolicyTest.php | 61 +++++++++ .../Fixtures/RecordingContainerBroker.php | 75 +++++++++++ .../tests/MessagingProviderGuardTest.php | 23 ++++ .../tests/Support/MessagingArtisan.php | 28 +++++ .../MessagingContainerCapstoneTestCase.php | 52 ++++++++ 22 files changed, 824 insertions(+), 3 deletions(-) create mode 100644 packages/messaging/src/Console/ConsumeMessagesCommand.php create mode 100644 packages/messaging/src/Container/BackOff.php create mode 100644 packages/messaging/src/Container/DeadLetterHeaders.php create mode 100644 packages/messaging/src/Container/DeliveryOutcome.php create mode 100644 packages/messaging/src/Container/ExponentialBackOff.php create mode 100644 packages/messaging/src/Container/LinearBackOff.php create mode 100644 packages/messaging/src/Container/ListenerContainerBroker.php create mode 100644 packages/messaging/src/Container/ListenerInvoker.php create mode 100644 packages/messaging/src/Container/ListenerOptions.php create mode 100644 packages/messaging/src/Container/RetryPolicy.php create mode 100644 packages/messaging/src/Container/Seconds.php create mode 100644 packages/messaging/tests/CapstoneMessagingConsumeCommandMemoryTest.php create mode 100644 packages/messaging/tests/CapstoneMessagingContainerTest.php create mode 100644 packages/messaging/tests/Container/ListenerInvokerTest.php create mode 100644 packages/messaging/tests/Container/RetryPolicyTest.php create mode 100644 packages/messaging/tests/Fixtures/RecordingContainerBroker.php create mode 100644 packages/messaging/tests/MessagingProviderGuardTest.php create mode 100644 packages/messaging/tests/Support/MessagingArtisan.php create mode 100644 packages/messaging/tests/Support/MessagingContainerCapstoneTestCase.php diff --git a/packages/messaging/src/Boot/MessageListenerWiringPass.php b/packages/messaging/src/Boot/MessageListenerWiringPass.php index f4e64da3..9b3d722c 100644 --- a/packages/messaging/src/Boot/MessageListenerWiringPass.php +++ b/packages/messaging/src/Boot/MessageListenerWiringPass.php @@ -8,6 +8,8 @@ use Firefly\Context\Boot\BootContext; use Firefly\Context\Boot\BootPass; use Firefly\Context\Boot\BootPhase; +use Firefly\Messaging\Container\ListenerContainerBroker; +use Firefly\Messaging\Container\ListenerOptions; use Firefly\Messaging\DeadLetter\DeadLetterStore; use Firefly\Messaging\DeadLetter\RetryingMessageHandler; use Firefly\Messaging\Listener\MessageListenerDescriptor; @@ -23,6 +25,9 @@ * DESCRIPTOR's OWN retries/retryDelay/deadLetterTopic (the per-listener policy, unlike eda's config-driven one) + * the bound DeadLetterStore, and resolves its target bean FRESH on every dispatch (never caching it). start() is * called last so the in-memory broker is publish-ready after boot (a no-op for the queue adapter). + * + * A ListenerContainerBroker (Kafka, RabbitMQ) is handed the BARE invoker plus the listener's ListenerOptions instead: + * its container owns retries and dead letters, so wrapping the listener here would retry every failure twice over. */ final class MessageListenerWiringPass implements BootPass { @@ -48,6 +53,12 @@ public function run(BootContext $context): void $dlq = $container->make(DeadLetterStore::class); foreach ($manifest->all() as $descriptor) { + if ($broker instanceof ListenerContainerBroker) { + $broker->subscribeListener($descriptor->topic, $this->invoker($container, $descriptor), $descriptor->group, ListenerOptions::fromDescriptor($descriptor)); + + continue; + } + $handler = RetryingMessageHandler::wrap( $this->invoker($container, $descriptor), $descriptor->retries, diff --git a/packages/messaging/src/Console/ConsumeMessagesCommand.php b/packages/messaging/src/Console/ConsumeMessagesCommand.php new file mode 100644 index 00000000..dd7c81a3 --- /dev/null +++ b/packages/messaging/src/Console/ConsumeMessagesCommand.php @@ -0,0 +1,119 @@ +string('firefly.messaging.provider', 'memory'); + $this->error("firefly.messaging.provider={$provider} has no consumer loop: memory delivers in process, queue runs in `php artisan queue:work`."); + + return self::FAILURE; + } + + $maxMessages = $this->positive('max-messages'); + $timeLimit = $this->positive('time-limit'); + $pollTimeout = max(1, (int) $this->option('poll-timeout')); + $sleep = max(0, (int) $this->option('sleep')); + $deadline = $timeLimit === null ? null : microtime(true) + $timeLimit; + $processed = 0; + + $this->info('firefly:messaging:consume — consuming.'); + + $this->stopping = false; + $previous = $this->installSignals(); + + try { + while (! $this->stopping && ($maxMessages === null || $processed < $maxMessages) && ($deadline === null || microtime(true) < $deadline)) { + if ($broker->poll($pollTimeout)) { + $processed++; + } elseif ($sleep > 0) { + usleep($sleep * 1000); + } + + $this->dispatchSignals(); + } + } finally { + $broker->stop(); + $this->restoreSignals($previous); + } + + $this->info("firefly:messaging:consume — processed {$processed} message(s)."); + + return self::SUCCESS; + } + + /** + * @return array the handlers that were installed before, by signal number + */ + private function installSignals(): array + { + if (! function_exists('pcntl_signal') || ! function_exists('pcntl_signal_get_handler')) { + return []; + } + + $previous = []; + $stop = function (): void { + $this->stopping = true; + }; + + foreach ([SIGINT, SIGTERM] as $signal) { + $previous[$signal] = pcntl_signal_get_handler($signal); + pcntl_signal($signal, $stop); + } + + return $previous; + } + + private function dispatchSignals(): void + { + if (function_exists('pcntl_signal_dispatch')) { + pcntl_signal_dispatch(); + } + } + + /** + * @param array $previous + */ + private function restoreSignals(array $previous): void + { + foreach ($previous as $signal => $handler) { + if (is_callable($handler) || is_int($handler)) { + pcntl_signal($signal, $handler); + } + } + } + + private function positive(string $option): ?int + { + $value = $this->option($option); + + return is_numeric($value) && (int) $value > 0 ? (int) $value : null; + } +} diff --git a/packages/messaging/src/Container/BackOff.php b/packages/messaging/src/Container/BackOff.php new file mode 100644 index 00000000..0305b017 --- /dev/null +++ b/packages/messaging/src/Container/BackOff.php @@ -0,0 +1,11 @@ + + */ + public static function for(Message $message, DeliveryOutcome $outcome): array + { + return [ + self::ORIGINAL_TOPIC => $message->topic, + self::EXCEPTION => self::reason($outcome), + self::EXCEPTION_MESSAGE => mb_substr($outcome->failure?->getMessage() ?? '', 0, 1000), + self::ATTEMPTS => (string) $outcome->attempts, + ]; + } + + public static function reason(DeliveryOutcome $outcome): string + { + if ($outcome->failure === null) { + return self::REASON_RETRIES_EXHAUSTED; + } + + $class = $outcome->failure::class; + $separator = strrpos($class, '\\'); + + return $separator === false ? $class : substr($class, $separator + 1); + } +} diff --git a/packages/messaging/src/Container/DeliveryOutcome.php b/packages/messaging/src/Container/DeliveryOutcome.php new file mode 100644 index 00000000..f2704691 --- /dev/null +++ b/packages/messaging/src/Container/DeliveryOutcome.php @@ -0,0 +1,17 @@ +maxDelay, $this->initial * ($this->multiplier ** max(0, $failures - 1)))); + } +} diff --git a/packages/messaging/src/Container/LinearBackOff.php b/packages/messaging/src/Container/LinearBackOff.php new file mode 100644 index 00000000..2114feb3 --- /dev/null +++ b/packages/messaging/src/Container/LinearBackOff.php @@ -0,0 +1,16 @@ +step * $failures); + } +} diff --git a/packages/messaging/src/Container/ListenerContainerBroker.php b/packages/messaging/src/Container/ListenerContainerBroker.php new file mode 100644 index 00000000..96fb63e1 --- /dev/null +++ b/packages/messaging/src/Container/ListenerContainerBroker.php @@ -0,0 +1,24 @@ +sleep = $sleep ?? static function (float $seconds): void { + if ($seconds > 0.0) { + usleep((int) round($seconds * 1_000_000)); + } + }; + } + + public function invoke(callable $handler, Message $message, RetryPolicy $policy): DeliveryOutcome + { + $attempt = 0; + + while (true) { + $attempt++; + + try { + $handler($message); + + return new DeliveryOutcome(true, $attempt); + } catch (Throwable $e) { + if ($attempt >= $policy->maxAttempts) { + return new DeliveryOutcome(false, $attempt, $e); + } + + $delay = $policy->backOff->delayAfter($attempt); + $this->logger?->warning("Listener on [{$message->topic}] failed attempt {$attempt}/{$policy->maxAttempts}; retrying in {$delay}s: {$e->getMessage()}"); + ($this->sleep)($delay); + } + } + } +} diff --git a/packages/messaging/src/Container/ListenerOptions.php b/packages/messaging/src/Container/ListenerOptions.php new file mode 100644 index 00000000..b6675531 --- /dev/null +++ b/packages/messaging/src/Container/ListenerOptions.php @@ -0,0 +1,32 @@ + 0` means N + 1 attempts with retryDelay as a LINEAR + * back-off, exactly as on the in-memory and queue brokers. A deadLetterTopic replaces the broker's default + * dead-letter route for this listener. + */ +final readonly class ListenerOptions +{ + public function __construct( + public ?int $maxAttempts = null, + public ?BackOff $backOff = null, + public ?string $deadLetterTopic = null, + ) {} + + public static function fromDescriptor(MessageListenerDescriptor $descriptor): self + { + if ($descriptor->retries > 0) { + return new self($descriptor->retries + 1, new LinearBackOff($descriptor->retryDelay), $descriptor->deadLetterTopic); + } + + return new self(null, null, $descriptor->deadLetterTopic); + } +} diff --git a/packages/messaging/src/Container/RetryPolicy.php b/packages/messaging/src/Container/RetryPolicy.php new file mode 100644 index 00000000..7537006c --- /dev/null +++ b/packages/messaging/src/Container/RetryPolicy.php @@ -0,0 +1,53 @@ +get('firefly.messaging.listener.retry.multiplier'); + if ($multiplier !== null && (! is_numeric($multiplier) || (float) $multiplier < 1.0)) { + throw new ConfigurationException('firefly.messaging.listener.retry.multiplier must be a number of at least 1 (a back-off never shrinks); got '.var_export($multiplier, true).'.'); + } + + return new self( + $config->int('firefly.messaging.listener.retry.max-attempts', 5), + new ExponentialBackOff( + Seconds::parse($config->get('firefly.messaging.listener.retry.initial-delay'), 1.0, 'firefly.messaging.listener.retry.initial-delay'), + $multiplier === null ? 2.0 : (float) $multiplier, + Seconds::parse($config->get('firefly.messaging.listener.retry.max-delay'), 30.0, 'firefly.messaging.listener.retry.max-delay'), + ), + ); + } + + public function withOptions(ListenerOptions $options): self + { + if ($options->maxAttempts === null && $options->backOff === null) { + return $this; + } + + return new self($options->maxAttempts ?? $this->maxAttempts, $options->backOff ?? $this->backOff); + } +} diff --git a/packages/messaging/src/Container/Seconds.php b/packages/messaging/src/Container/Seconds.php new file mode 100644 index 00000000..da7822e2 --- /dev/null +++ b/packages/messaging/src/Container/Seconds.php @@ -0,0 +1,40 @@ += 0) { + return (float) $value; + } + + if (is_string($value) && preg_match('/^(\d+(?:\.\d+)?)(ms|s|m|h)?$/', trim($value), $match) === 1) { + $number = (float) $match[1]; + + return match ($match[2] ?? 's') { + 'ms' => $number / 1000, + 'm' => $number * 60, + 'h' => $number * 3600, + default => $number, + }; + } + + throw new ConfigurationException("{$key} must be a non-negative duration (seconds, or 250ms / 30s / 5m / 1h); got ".var_export($value, true).'.'); + } +} diff --git a/packages/messaging/src/MessagingAutoConfiguration.php b/packages/messaging/src/MessagingAutoConfiguration.php index 9436e2a6..5364b383 100644 --- a/packages/messaging/src/MessagingAutoConfiguration.php +++ b/packages/messaging/src/MessagingAutoConfiguration.php @@ -9,6 +9,7 @@ use Firefly\Container\Attributes\Configuration; use Firefly\Container\Attributes\Order; use Firefly\Context\Condition\Attributes\ConditionalOnMissingBean; +use Firefly\Kernel\Exception\Framework\ConfigurationException; use Firefly\Messaging\Broker\InMemoryMessageBroker; use Firefly\Messaging\Broker\QueueMessageBroker; use Firefly\Messaging\DeadLetter\DeadLetterStore; @@ -17,7 +18,7 @@ /** * Always-on messaging transport wiring. Selects the MessageBrokerPort adapter by firefly.messaging.provider - * (`memory` default | `queue`) and the DeadLetterStore default. #[Order(1000)] places it after user definitions; + * (`memory` default | `queue`; `kafka` and `rabbitmq` are bound by the eda-kafka / eda-rabbitmq adapters) and the DeadLetterStore default. #[Order(1000)] places it after user definitions; * each #[ConditionalOnMissingBean] lets an app bind its own and win. Both adapters are singletons (the #[Bean] * default), so the wiring pass and the queue worker share ONE subscriber set. Mirrors EdaAutoConfiguration. */ @@ -29,7 +30,16 @@ final class MessagingAutoConfiguration #[ConditionalOnMissingBean(MessageBrokerPort::class)] public function messageBroker(Config $config, Container $container): MessageBrokerPort { - if ($config->string('firefly.messaging.provider', 'memory') === 'queue') { + $provider = $config->string('firefly.messaging.provider', 'memory'); + + // kafka and rabbitmq are bound at #[Order(900)] by eda-kafka / eda-rabbitmq, before this configuration is + // asked. Reaching here with either means that adapter did not bind — falling back to the in-memory broker + // would publish into the void. + if ($provider === 'kafka' || $provider === 'rabbitmq') { + throw new ConfigurationException("firefly.messaging.provider={$provider} but no {$provider} MessageBrokerPort was bound; is the firefly/eda-{$provider} adapter installed and, for kafka, ext-rdkafka loaded?"); + } + + if ($provider === 'queue') { $connection = $config->has('firefly.messaging.queue.connection') ? $config->string('firefly.messaging.queue.connection') : null; $queue = $config->has('firefly.messaging.queue.name') ? $config->string('firefly.messaging.queue.name') : null; diff --git a/packages/messaging/src/MessagingWiringProvider.php b/packages/messaging/src/MessagingWiringProvider.php index f386af82..baa86d71 100644 --- a/packages/messaging/src/MessagingWiringProvider.php +++ b/packages/messaging/src/MessagingWiringProvider.php @@ -8,6 +8,7 @@ use Firefly\Context\Boot\FireflyServiceProvider; use Firefly\Context\Scan\AppScan; use Firefly\Messaging\Boot\MessageListenerWiringPass; +use Firefly\Messaging\Console\ConsumeMessagesCommand; use Firefly\Messaging\Listener\MessageListenerManifest; use Firefly\Messaging\Scanner\MessageListenerScanner; use Illuminate\Contracts\Container\Container; @@ -17,7 +18,7 @@ * passes()). Contributes MessageListenerWiringPass via passes() and resolves the MessageListenerManifest behind a * bound() guard — compiled artifact first, then an in-process scan of firefly.scan.paths, then empty. Both this * and MessagingServiceProvider are listed in extra.laravel.providers. Mirrors EdaWiringProvider / - * SchedulingWiringProvider. + * SchedulingWiringProvider. It also registers `firefly:messaging:consume`. */ final class MessagingWiringProvider extends FireflyServiceProvider { @@ -38,6 +39,13 @@ public function register(): void parent::register(); } + public function boot(): void + { + if ($this->app->runningInConsole()) { + $this->commands([ConsumeMessagesCommand::class]); + } + } + /** * @return list */ diff --git a/packages/messaging/tests/CapstoneMessagingConsumeCommandMemoryTest.php b/packages/messaging/tests/CapstoneMessagingConsumeCommandMemoryTest.php new file mode 100644 index 00000000..431a0a9e --- /dev/null +++ b/packages/messaging/tests/CapstoneMessagingConsumeCommandMemoryTest.php @@ -0,0 +1,13 @@ +artisan('firefly:messaging:consume', ['--max-messages' => 1]), 1, 'has no consumer loop'); +}); diff --git a/packages/messaging/tests/CapstoneMessagingContainerTest.php b/packages/messaging/tests/CapstoneMessagingContainerTest.php new file mode 100644 index 00000000..3a91eeda --- /dev/null +++ b/packages/messaging/tests/CapstoneMessagingContainerTest.php @@ -0,0 +1,50 @@ +broker->subscriptions as $subscription) { + $byTopic[$subscription['topic']] = $subscription['options']; + } + + expect($byTopic['orders'])->toEqual(new ListenerOptions) + ->and($byTopic['failing'])->toEqual(new ListenerOptions(3, new LinearBackOff(0.0), 'failing.DLT')); +}); + +it('consumes through php artisan firefly:messaging:consume and stops the broker', function () { + /** @var MessagingContainerCapstoneTestCase $this */ + $this->broker->publish('orders', 'bytes-1'); + + MessagingArtisan::outputContains($this->artisan('firefly:messaging:consume', ['--max-messages' => 1, '--poll-timeout' => 10]), 0, 'processed 1 message'); + + expect($this->app()->make(ListenerSpy::class)->seen)->toBe(['orders']) + ->and($this->broker->stopped)->toBeTrue() + ->and($this->deadLetters->all())->toBe([]); +}); + +it('stops between deliveries on SIGTERM and still stops the broker', function () { + /** @var MessagingContainerCapstoneTestCase $this */ + $broker = $this->broker; + foreach (['bytes-1', 'bytes-2', 'bytes-3'] as $value) { + $broker->publish('orders', $value); + } + $broker->onPoll = static function (): void { + posix_kill(posix_getpid(), SIGTERM); + }; + + MessagingArtisan::outputContains($this->artisan('firefly:messaging:consume', ['--max-messages' => 3, '--poll-timeout' => 10]), 0, 'processed 1 message'); + + expect($this->app()->make(ListenerSpy::class)->seen)->toBe(['orders']) + ->and($this->broker->published)->toHaveCount(2) + ->and($this->broker->stopped)->toBeTrue(); +})->skip(! function_exists('pcntl_signal') || ! function_exists('posix_kill'), 'needs ext-pcntl and ext-posix'); diff --git a/packages/messaging/tests/Container/ListenerInvokerTest.php b/packages/messaging/tests/Container/ListenerInvokerTest.php new file mode 100644 index 00000000..63cac159 --- /dev/null +++ b/packages/messaging/tests/Container/ListenerInvokerTest.php @@ -0,0 +1,49 @@ +invoke(static function (Message $m): void {}, new Message('t', 'v'), new RetryPolicy(3)); + + expect($outcome->handled)->toBeTrue()->and($outcome->attempts)->toBe(1)->and($slept)->toBe([]); +}); + +it('retries with the back-off between attempts and succeeds on a later one', function () { + $slept = []; + $calls = 0; + $outcome = (new ListenerInvoker(function (float $s) use (&$slept): void { + $slept[] = $s; + }))->invoke(function () use (&$calls): void { + if (++$calls < 3) { + throw new RuntimeException('not yet'); + } + }, new Message('t', 'v'), new RetryPolicy(5, new ExponentialBackOff(1.0, 2.0, 30.0))); + + expect($outcome->handled)->toBeTrue()->and($outcome->attempts)->toBe(3)->and($slept)->toBe([1.0, 2.0]); +}); + +it('gives up after the last attempt and reports why, for the dead-letter headers', function () { + $outcome = (new ListenerInvoker(static function (float $s): void {})) + ->invoke(static function (): void { + throw new DomainException('broken payload'); + }, new Message('orders', 'v'), new RetryPolicy(2)); + + expect($outcome->handled)->toBeFalse() + ->and($outcome->attempts)->toBe(2) + ->and($outcome->failure)->toBeInstanceOf(DomainException::class) + ->and(DeadLetterHeaders::for(new Message('orders', 'v'), $outcome))->toBe([ + 'x-original-topic' => 'orders', + 'x-exception' => 'DomainException', + 'x-exception-message' => 'broken payload', + 'x-delivery-attempts' => '2', + ]); +}); diff --git a/packages/messaging/tests/Container/RetryPolicyTest.php b/packages/messaging/tests/Container/RetryPolicyTest.php new file mode 100644 index 00000000..ec528036 --- /dev/null +++ b/packages/messaging/tests/Container/RetryPolicyTest.php @@ -0,0 +1,61 @@ +delayAfter(...), [1, 2, 3, 4]))->toBe([1.0, 2.0, 4.0, 5.0]) + ->and(array_map($linear->delayAfter(...), [1, 2, 3]))->toBe([0.5, 1.0, 1.5]); +}); + +it("defaults to PyFly's container policy: five attempts, 1s doubling, capped at 30s", function () { + $policy = RetryPolicy::fromConfig(new Config(new Repository(['firefly' => []]))); + + expect($policy->maxAttempts)->toBe(5) + ->and($policy->backOff)->toEqual(new ExponentialBackOff(1.0, 2.0, 30.0)); +}); + +it('reads the listener retry keys, durations included', function () { + $policy = RetryPolicy::fromConfig(new Config(new Repository(['firefly' => ['messaging' => ['listener' => ['retry' => [ + 'max-attempts' => 3, 'initial-delay' => '250ms', 'multiplier' => 3, 'max-delay' => '2s', + ]]]]]))); + + expect($policy->maxAttempts)->toBe(3) + ->and($policy->backOff)->toEqual(new ExponentialBackOff(0.25, 3.0, 2.0)); +}); + +it("lets a listener's own retries override the container default", function () { + $options = ListenerOptions::fromDescriptor(new MessageListenerDescriptor('C', 'm', 'orders', null, 2, 0.1, 'orders.dead')); + $policy = (new RetryPolicy)->withOptions($options); + + expect($options->deadLetterTopic)->toBe('orders.dead') + ->and($policy->maxAttempts)->toBe(3) + ->and($policy->backOff)->toEqual(new LinearBackOff(0.1)) + ->and((new RetryPolicy)->withOptions(ListenerOptions::fromDescriptor(new MessageListenerDescriptor('C', 'm', 'orders'))))->toEqual(new RetryPolicy); +}); + +it('refuses a policy that would never deliver, and a malformed duration', function () { + expect(fn () => new RetryPolicy(0))->toThrow(ConfigurationException::class) + ->and(fn () => Seconds::parse('soon', 1.0, 'firefly.messaging.listener.retry.initial-delay')) + ->toThrow(ConfigurationException::class, 'firefly.messaging.listener.retry.initial-delay') + ->and(Seconds::parse('1m', 0.0, 'k'))->toBe(60.0) + ->and(Seconds::parse(null, 1.5, 'k'))->toBe(1.5); +}); + +it('refuses a multiplier that is not a number of at least 1', function (mixed $multiplier) { + $config = new Config(new Repository(['firefly' => ['messaging' => ['listener' => ['retry' => ['multiplier' => $multiplier]]]]])); + + expect(fn () => RetryPolicy::fromConfig($config))->toThrow(ConfigurationException::class, 'firefly.messaging.listener.retry.multiplier'); +})->with(['twice', 0.5, -2]); diff --git a/packages/messaging/tests/Fixtures/RecordingContainerBroker.php b/packages/messaging/tests/Fixtures/RecordingContainerBroker.php new file mode 100644 index 00000000..aba038db --- /dev/null +++ b/packages/messaging/tests/Fixtures/RecordingContainerBroker.php @@ -0,0 +1,75 @@ + */ + public array $subscriptions = []; + + /** @var list */ + public array $published = []; + + public bool $stopped = false; + + /** @var (Closure(): void)|null runs at the start of every poll(), e.g. to deliver a signal mid-run */ + public ?Closure $onPoll = null; + + /** @var array> */ + private array $listeners = []; + + public function publish(string $topic, string $value, ?string $key = null, array $headers = []): void + { + $this->published[] = new Message($topic, $value, $key, $headers); + } + + public function subscribe(string $topic, callable $handler, ?string $group = null): void + { + $this->subscribeListener($topic, $handler, $group, new ListenerOptions); + } + + public function subscribeListener(string $topic, callable $handler, ?string $group, ListenerOptions $options): void + { + $this->subscriptions[] = ['topic' => $topic, 'group' => $group, 'options' => $options]; + $this->listeners[$topic][] = ['handler' => $handler, 'options' => $options]; + } + + public function poll(int $timeoutMs): bool + { + if ($this->onPoll !== null) { + ($this->onPoll)(); + } + + $message = array_shift($this->published); + if ($message === null) { + return false; + } + + $invoker = new ListenerInvoker(static function (float $seconds): void {}); + foreach ($this->listeners[$message->topic] ?? [] as $listener) { + $invoker->invoke($listener['handler'], $message, (new RetryPolicy(1))->withOptions($listener['options'])); + } + + return true; + } + + public function start(): void {} + + public function stop(): void + { + $this->stopped = true; + } +} diff --git a/packages/messaging/tests/MessagingProviderGuardTest.php b/packages/messaging/tests/MessagingProviderGuardTest.php new file mode 100644 index 00000000..540facad --- /dev/null +++ b/packages/messaging/tests/MessagingProviderGuardTest.php @@ -0,0 +1,23 @@ + ['messaging' => ['provider' => $provider]]])); + + expect(fn () => (new MessagingAutoConfiguration)->messageBroker($config, new Container)) + ->toThrow(ConfigurationException::class, "firefly.messaging.provider={$provider}"); +})->with(['kafka', 'rabbitmq']); + +it('keeps any other unknown provider on the in-memory broker, as before', function () { + $config = new Config(new Repository(['firefly' => ['messaging' => ['provider' => 'carrier-pigeon']]])); + + expect((new MessagingAutoConfiguration)->messageBroker($config, new Container))->toBeInstanceOf(InMemoryMessageBroker::class); +}); diff --git a/packages/messaging/tests/Support/MessagingArtisan.php b/packages/messaging/tests/Support/MessagingArtisan.php new file mode 100644 index 00000000..88b65feb --- /dev/null +++ b/packages/messaging/tests/Support/MessagingArtisan.php @@ -0,0 +1,28 @@ +expectsOutputToContain($needle)->assertExitCode($exitCode)->run(); + + return; + } + + expect($result)->toBe($exitCode); + } +} diff --git a/packages/messaging/tests/Support/MessagingContainerCapstoneTestCase.php b/packages/messaging/tests/Support/MessagingContainerCapstoneTestCase.php new file mode 100644 index 00000000..ae7949cd --- /dev/null +++ b/packages/messaging/tests/Support/MessagingContainerCapstoneTestCase.php @@ -0,0 +1,52 @@ + */ + protected function fireflyProviders(): array + { + return [MessagingWiringProvider::class]; + } + + /** @return array */ + protected function configOverrides(): array + { + return ['firefly.messaging.provider' => 'kafka']; + } + + protected function defineFireflyEnvironment(Application $app): void + { + $this->broker = new RecordingContainerBroker; + $this->deadLetters = new InMemoryDeadLetterStore; + $app->instance(MessageBrokerPort::class, $this->broker); + $app->instance(DeadLetterStore::class, $this->deadLetters); + $app->instance(MessageListenerManifest::class, new MessageListenerManifest( + (new MessageListenerScanner)->scan(['Firefly\\Messaging\\Tests\\Fixtures\\' => dirname(__DIR__).'/Fixtures']), + )); + $app->singleton(ListenerSpy::class); + } +} From c34acae278c8d764296e9d7216b898703fc71657 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 19:35:37 -0700 Subject: [PATCH 20/47] docs(config): document firefly.messaging.listener.retry and firefly:messaging:consume in the skeleton and the messaging module --- docs/modules/messaging.md | 35 ++++++++++++++++++++++++++++++++--- skeleton/config/firefly.php | 26 ++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 3 deletions(-) diff --git a/docs/modules/messaging.md b/docs/modules/messaging.md index 7f72bd28..c5711352 100644 --- a/docs/modules/messaging.md +++ b/docs/modules/messaging.md @@ -170,9 +170,38 @@ rather than an `EventEnvelope`, faithful to the two ports' different payload sha | `firefly.messaging.provider` | `memory`\|`queue` | `memory` | Selects `InMemoryMessageBroker` or `QueueMessageBroker` (`MessagingAutoConfiguration`). | | `firefly.messaging.queue.connection` | string\|null | `null` (default connection) | Queue connection `QueueMessageBroker`/`DispatchMessageJob` dispatch onto, when `provider=queue`. | | `firefly.messaging.queue.name` | string\|null | `null` (default queue) | Queue name, when `provider=queue`. | - -There is no `firefly.messaging.retries`/`retry_delay` config key — retry/DLQ policy lives entirely on each -`#[MessageListener]` attribute (see above), not in configuration. +| `firefly.messaging.listener.retry.max-attempts` | int | `5` | Attempts per delivery (the first included) on a listener container broker. | +| `firefly.messaging.listener.retry.initial-delay` | duration | `1s` | Exponential back-off after the first failure (seconds, or `250ms`/`30s`/`5m`/`1h`). | +| `firefly.messaging.listener.retry.multiplier` | float >= 1 | `2.0` | Back-off multiplier after each further failure. | +| `firefly.messaging.listener.retry.max-delay` | duration | `30s` | Back-off cap. | + +There is no `firefly.messaging.retries`/`retry_delay` config key — on the `memory` and `queue` adapters +retry/DLQ policy lives entirely on each `#[MessageListener]` attribute (see above), not in configuration. The +`listener.retry.*` keys apply only to a listener container broker, described next. + +## Listener container brokers and `firefly:messaging:consume` + +A broker that consumes from a real server (the Kafka and RabbitMQ adapters) implements +`Firefly\Messaging\Container\ListenerContainerBroker`, a `MessageBrokerPort` with two more methods: +`subscribeListener(string $topic, callable $handler, ?string $group, ListenerOptions $options)` and +`poll(int $timeoutMs): bool`. Its container owns retries and dead letters and acknowledges a delivery only once +its listeners have handled it or it has been dead-lettered, so `MessageListenerWiringPass` hands it the BARE +listener plus the listener's `ListenerOptions` instead of wrapping it in `RetryingMessageHandler` — wrapping it +too would retry every failure twice over. + +Each delivery runs under a `RetryPolicy` through the shared `ListenerInvoker`: `firefly.messaging.listener.retry.*` +(five attempts, 1s doubling, capped at 30s by default — PyFly's listener container), unless the listener sets +`retries: N > 0`, which means N + 1 attempts with `retryDelay` as a linear back-off, exactly as on the in-memory +and queue adapters. A listener's `deadLetterTopic` replaces the broker's default dead-letter route. A +dead-lettered message carries `x-original-topic`, `x-exception` (the short exception class name), +`x-exception-message` and `x-delivery-attempts` (`DeadLetterHeaders`). + +`php artisan firefly:messaging:consume {--max-messages=} {--time-limit=} {--poll-timeout=1000} {--sleep=0}` is +the worker: it polls one delivery at a time until a bound trips or a SIGINT/SIGTERM arrives (honored between +deliveries when ext-pcntl is loaded), then stops the broker. On `memory` (delivers in process) and `queue` (runs +in `php artisan queue:work`) it refuses with exit code 1 and the reason. Setting +`firefly.messaging.provider=kafka|rabbitmq` without the matching adapter bound refuses the boot instead of +falling back to the in-memory broker. `MessagingAutoConfiguration` (`#[Configuration] #[Order(1000)]`) binds the `MessageBrokerPort` and `DeadLetterStore` beans, each behind `#[ConditionalOnMissingBean]` so an application binding its own wins. It diff --git a/skeleton/config/firefly.php b/skeleton/config/firefly.php index 685349eb..47cfec88 100644 --- a/skeleton/config/firefly.php +++ b/skeleton/config/firefly.php @@ -2567,6 +2567,32 @@ // 'connection' => 'redis', // 'name' => 'messages', // ], + + /* + | listener.retry — how a LISTENER CONTAINER broker (a MessageBrokerPort that implements + | ListenerContainerBroker and is driven by `php artisan firefly:messaging:consume`) retries one + | delivery before dead-lettering it. memory and queue keep the per-listener policy of + | #[MessageListener(retries:, retryDelay:)] and never read these keys. + | + | max-attempts — attempts per delivery, the first included; at least 1. + | initial-delay / multiplier / max-delay — exponential back-off: initial-delay after the first + | failure, times multiplier after each further one, capped at max-delay. Durations are seconds or + | '250ms' / '30s' / '5m' / '1h'; the multiplier is at least 1. A malformed value refuses the boot, + | naming the key. + | + | A listener that sets its own `retries: N > 0` overrides these with N + 1 attempts and a linear + | retryDelay back-off; its deadLetterTopic replaces the broker's default dead-letter route. + | + | Default: 5 attempts, 1s doubling, capped at 30s (PyFly's listener container). + */ + 'listener' => [ + 'retry' => [ + 'max-attempts' => env('FIREFLY_MESSAGING_LISTENER_RETRY_MAX_ATTEMPTS', 5), + 'initial-delay' => env('FIREFLY_MESSAGING_LISTENER_RETRY_INITIAL_DELAY', '1s'), + 'multiplier' => env('FIREFLY_MESSAGING_LISTENER_RETRY_MULTIPLIER', 2.0), + 'max-delay' => env('FIREFLY_MESSAGING_LISTENER_RETRY_MAX_DELAY', '30s'), + ], + ], ], /* From 683d8af5693553c3110bb694447f4844597f3b40 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 19:40:06 -0700 Subject: [PATCH 21/47] feat(eda-kafka): a Kafka listener container for firefly/messaging with retries, back-off and dead-letter topics --- deptrac.yaml | 6 +- .../cache/firefly-eda-kafka-components.php | 29 +++ .../cache/firefly-eda-kafka-context.php | 26 +++ packages/eda-kafka/composer.json | 6 +- .../src/Messaging/KafkaMessageBroker.php | 165 +++++++++++++++++ .../KafkaMessagingAutoConfiguration.php | 59 ++++++ .../eda-kafka/src/Messaging/KafkaRecord.php | 26 +++ .../src/Messaging/KafkaRecordClient.php | 28 +++ .../src/Messaging/RdKafkaRecordClient.php | 110 +++++++++++ .../tests/Fixtures/FakeKafkaRecordClient.php | 73 ++++++++ .../Messaging/KafkaMessageBrokerTest.php | 172 ++++++++++++++++++ .../KafkaMessagingAutoConfigurationTest.php | 37 ++++ .../Messaging/RdKafkaRecordClientTest.php | 21 +++ 13 files changed, 755 insertions(+), 3 deletions(-) create mode 100644 packages/eda-kafka/src/Messaging/KafkaMessageBroker.php create mode 100644 packages/eda-kafka/src/Messaging/KafkaMessagingAutoConfiguration.php create mode 100644 packages/eda-kafka/src/Messaging/KafkaRecord.php create mode 100644 packages/eda-kafka/src/Messaging/KafkaRecordClient.php create mode 100644 packages/eda-kafka/src/Messaging/RdKafkaRecordClient.php create mode 100644 packages/eda-kafka/tests/Fixtures/FakeKafkaRecordClient.php create mode 100644 packages/eda-kafka/tests/Messaging/KafkaMessageBrokerTest.php create mode 100644 packages/eda-kafka/tests/Messaging/KafkaMessagingAutoConfigurationTest.php create mode 100644 packages/eda-kafka/tests/Messaging/RdKafkaRecordClientTest.php diff --git a/deptrac.yaml b/deptrac.yaml index e1eab59b..479e1162 100644 --- a/deptrac.yaml +++ b/deptrac.yaml @@ -276,7 +276,10 @@ deptrac: # SchedulingPostgres/EdaRabbitmq/EdaPostgres precedent) so the core eda package stays free of the rdkafka # dependency. It reads Config indirectly via #[ConditionalOnProperty], carries Container/Context attributes, # extends AutoConfigure's base, implements Eda's EventPublisher, and supplies an Actuator HealthIndicator. It - # is depended on by NONE. + # is depended on by NONE. The => Messaging edge is the raw-bytes messaging adapter (src/Messaging): + # KafkaMessageBroker implements Messaging's ListenerContainerBroker over this package's own + # KafkaConsumerFactory/KafkaProducerFactory, so the two transports share one librdkafka wiring. Core Messaging + # still has no edge to or from Eda. EdaKafka: - Kernel - Container @@ -285,6 +288,7 @@ deptrac: - AutoConfigure - Eda - Actuator + - Messaging # Messaging is a lower-level raw-bytes transport leaf capability: a MessageBrokerPort + in-memory/queue adapters, # a #[MessageListener] scanner/manifest, and a bytes-aware retry/DLQ. It reads Config, reuses Kernel infra diff --git a/packages/eda-kafka/cache/firefly-eda-kafka-components.php b/packages/eda-kafka/cache/firefly-eda-kafka-components.php index e1490bc4..a4ceda87 100644 --- a/packages/eda-kafka/cache/firefly-eda-kafka-components.php +++ b/packages/eda-kafka/cache/firefly-eda-kafka-components.php @@ -75,4 +75,33 @@ 'dependencies' => [ ], ], + 2 => [ + 'class' => 'Firefly\\Eda\\Kafka\\Messaging\\KafkaMessagingAutoConfiguration', + 'stereotype' => 'configuration', + 'name' => null, + 'scope' => 'Singleton', + 'primary' => false, + 'order' => 900, + 'qualifier' => null, + 'interfaces' => [ + ], + 'beans' => [ + 0 => [ + 'method' => 'messageBroker', + 'returns' => 'Firefly\\Messaging\\MessageBrokerPort', + 'name' => null, + 'scope' => 'Singleton', + 'primary' => false, + 'order' => 0, + 'lazy' => false, + 'dependencies' => [ + 0 => 'Firefly\\Config\\Config', + 1 => 'Illuminate\\Container\\Container', + ], + ], + ], + 'lazy' => false, + 'dependencies' => [ + ], + ], ]; diff --git a/packages/eda-kafka/cache/firefly-eda-kafka-context.php b/packages/eda-kafka/cache/firefly-eda-kafka-context.php index 5799a4ed..4eae3b98 100644 --- a/packages/eda-kafka/cache/firefly-eda-kafka-context.php +++ b/packages/eda-kafka/cache/firefly-eda-kafka-context.php @@ -78,4 +78,30 @@ 'beanConditions' => [ ], ], + 2 => [ + 'class' => 'Firefly\\Eda\\Kafka\\Messaging\\KafkaMessagingAutoConfiguration', + 'postConstruct' => [ + ], + 'preDestroy' => [ + ], + 'listeners' => [ + ], + 'conditions' => [ + ], + 'beanConditions' => [ + 0 => [ + 'method' => 'messageBroker', + 'conditions' => [ + 0 => [ + 'type' => 'Firefly\\Context\\Condition\\Attributes\\ConditionalOnProperty', + 'args' => [ + 0 => 'firefly.messaging.provider', + 1 => 'kafka', + 2 => false, + ], + ], + ], + ], + ], + ], ]; diff --git a/packages/eda-kafka/composer.json b/packages/eda-kafka/composer.json index fae9a280..3dc80157 100644 --- a/packages/eda-kafka/composer.json +++ b/packages/eda-kafka/composer.json @@ -1,6 +1,6 @@ { "name": "firefly/eda-kafka", - "description": "LaraFly eda Kafka adapter: a KafkaEventPublisher + consumer over the OPTIONAL ext-rdkafka (suggest + extension_loaded guard — never a hard require), auto-configured behind firefly.eda.provider=kafka, with a DLT topic for exhausted retries and a KafkaHealthIndicator.", + "description": "LaraFly eda Kafka adapter: a KafkaEventPublisher + consumer over the OPTIONAL ext-rdkafka (suggest + extension_loaded guard — never a hard require), auto-configured behind firefly.eda.provider=kafka, with a DLT topic for exhausted retries and a KafkaHealthIndicator; also the Kafka adapter for firefly/messaging (firefly.messaging.provider=kafka).", "type": "library", "license": "Apache-2.0", "homepage": "https://github.com/fireflyframework/fireflyframework-php", @@ -28,9 +28,11 @@ "firefly/context": "*@dev", "firefly/eda": "*@dev", "firefly/kernel": "*@dev", + "firefly/messaging": "*@dev", "illuminate/console": "^13.0", "illuminate/contracts": "^13.0", - "illuminate/support": "^13.0" + "illuminate/support": "^13.0", + "psr/log": "^3.0" }, "suggest": { "ext-rdkafka": "Required ONLY to activate the Kafka adapter (firefly.eda.provider=kafka). Needs librdkafka >= 1.5.3. Absent = the adapter is inert; the default gate + non-Kafka consumers never need it." diff --git a/packages/eda-kafka/src/Messaging/KafkaMessageBroker.php b/packages/eda-kafka/src/Messaging/KafkaMessageBroker.php new file mode 100644 index 00000000..2b48a767 --- /dev/null +++ b/packages/eda-kafka/src/Messaging/KafkaMessageBroker.php @@ -0,0 +1,165 @@ +` (`.DLT`), with the + * diagnostic headers and the eda adapter's x-dlt-* provenance headers PyFly also writes. With dead letters off + * (firefly.messaging.listener.dead-letter.enabled=false) the record is logged and skipped — an explicit choice. + * * A dead letter that cannot be produced throws OUT of poll() with the offset NOT committed: the worker exits, + * its supervisor restarts it, and the record is consumed again — never silently lost. + * * publish() produces and flushes before returning, so a message published from a PHP-FPM request is on the + * broker when the request ends. + */ +final class KafkaMessageBroker implements ListenerContainerBroker +{ + /** @var array>> group => topic => listeners */ + private array $listeners = []; + + /** @var array */ + private array $consumers = []; + + /** + * @param Closure(string): KafkaRecordClient $consumerFor builds the consumer of one group + */ + public function __construct( + private readonly Closure $consumerFor, + private readonly KafkaRecordClient $producer, + private readonly RetryPolicy $policy, + private readonly ListenerInvoker $invoker, + private readonly string $defaultGroup = 'firefly', + private readonly string $deadLetterSuffix = '.DLT', + private readonly bool $deadLetterEnabled = true, + private readonly int $flushTimeoutMs = 10000, + private readonly ?LoggerInterface $logger = null, + ) {} + + public function publish(string $topic, string $value, ?string $key = null, array $headers = []): void + { + $this->producer->produce($topic, $value, $key, $headers); + $this->producer->flush($this->flushTimeoutMs); + } + + public function subscribe(string $topic, callable $handler, ?string $group = null): void + { + $this->subscribeListener($topic, $handler, $group, new ListenerOptions); + } + + public function subscribeListener(string $topic, callable $handler, ?string $group, ListenerOptions $options): void + { + $group ??= $this->defaultGroup; + if (isset($this->consumers[$group])) { + throw new MessagingException("The Kafka consumer of group [{$group}] is already polling; subscribe every listener before the first poll()."); + } + + $this->listeners[$group][$topic][] = ['handler' => $handler, 'options' => $options]; + } + + public function poll(int $timeoutMs): bool + { + if ($this->listeners === []) { + return false; + } + + $slice = max(1, intdiv($timeoutMs, count($this->listeners))); + + foreach (array_keys($this->listeners) as $group) { + $group = (string) $group; + $consumer = $this->consumer($group); + $record = $consumer->consume($slice); + if ($record === null) { + continue; + } + + $this->dispatch($group, $record); + $consumer->commit($record); + + return true; + } + + return false; + } + + public function start(): void {} + + public function stop(): void + { + foreach ($this->consumers as $consumer) { + $consumer->close(); + } + $this->consumers = []; + $this->producer->flush($this->flushTimeoutMs); + } + + private function consumer(string $group): KafkaRecordClient + { + if (! isset($this->consumers[$group])) { + $client = ($this->consumerFor)($group); + $client->subscribe(array_map('strval', array_keys($this->listeners[$group]))); + $this->consumers[$group] = $client; + } + + return $this->consumers[$group]; + } + + private function dispatch(string $group, KafkaRecord $record): void + { + $message = new Message($record->topic, $record->value, $record->key, $record->headers); + + foreach ($this->listeners[$group][$record->topic] ?? [] as $listener) { + $outcome = $this->invoker->invoke($listener['handler'], $message, $this->policy->withOptions($listener['options'])); + if (! $outcome->handled) { + $this->deadLetter($record, $message, $outcome, $listener['options']); + } + } + } + + private function deadLetter(KafkaRecord $record, Message $message, DeliveryOutcome $outcome, ListenerOptions $options): void + { + $target = $options->deadLetterTopic ?? ($this->deadLetterEnabled ? $record->topic.$this->deadLetterSuffix : null); + $cause = $outcome->failure?->getMessage() ?? ''; + + if ($target === null) { + $this->logger?->error("Kafka record {$record->topic}@{$record->offset} failed {$outcome->attempts} attempt(s) and dead letters are off; skipping it: {$cause}"); + + return; + } + + $headers = array_merge($record->headers, DeadLetterHeaders::for($message, $outcome), [ + RdKafkaConsumerClient::REASON_HEADER => DeadLetterHeaders::reason($outcome), + RdKafkaConsumerClient::SOURCE_TOPIC_HEADER => $record->topic, + RdKafkaConsumerClient::SOURCE_OFFSET_HEADER => (string) $record->offset, + ]); + + try { + $this->producer->produce($target, $record->value, $record->key, $headers); + $this->producer->flush($this->flushTimeoutMs); + } catch (Throwable $e) { + throw new MessagingException("Kafka record {$record->topic}@{$record->offset} could not be dead-lettered to [{$target}]; its offset is left uncommitted so it is consumed again.", 'MESSAGING_DEAD_LETTER_FAILED', $e); + } + + $this->logger?->error("Kafka record {$record->topic}@{$record->offset} failed {$outcome->attempts} attempt(s); dead-lettered to [{$target}]: {$cause}"); + } +} diff --git a/packages/eda-kafka/src/Messaging/KafkaMessagingAutoConfiguration.php b/packages/eda-kafka/src/Messaging/KafkaMessagingAutoConfiguration.php new file mode 100644 index 00000000..930eca76 --- /dev/null +++ b/packages/eda-kafka/src/Messaging/KafkaMessagingAutoConfiguration.php @@ -0,0 +1,59 @@ +has('firefly.messaging.kafka.brokers') + ? $config->string('firefly.messaging.kafka.brokers') + : $config->string('firefly.eda.kafka.brokers', '127.0.0.1:9092'); + $logger = $container->bound(LoggerInterface::class) ? $container->make(LoggerInterface::class) : null; + $logger = $logger instanceof LoggerInterface ? $logger : null; + + return new KafkaMessageBroker( + static fn (string $group): KafkaRecordClient => new RdKafkaRecordClient(new KafkaConsumerFactory($brokers, $group), new KafkaProducerFactory($brokers)), + new RdKafkaRecordClient(new KafkaConsumerFactory($brokers), new KafkaProducerFactory($brokers)), + RetryPolicy::fromConfig($config), + new ListenerInvoker(logger: $logger), + $config->string('firefly.messaging.kafka.group-id', 'firefly'), + $config->string('firefly.messaging.kafka.dlt-suffix', '.DLT'), + $config->bool('firefly.messaging.listener.dead-letter.enabled', true), + logger: $logger, + ); + } +} diff --git a/packages/eda-kafka/src/Messaging/KafkaRecord.php b/packages/eda-kafka/src/Messaging/KafkaRecord.php new file mode 100644 index 00000000..4484a5e3 --- /dev/null +++ b/packages/eda-kafka/src/Messaging/KafkaRecord.php @@ -0,0 +1,26 @@ + $headers + */ + public function __construct( + public string $topic, + public string $value, + public ?string $key, + public array $headers, + public int $partition, + public int $offset, + public mixed $handle, + ) {} +} diff --git a/packages/eda-kafka/src/Messaging/KafkaRecordClient.php b/packages/eda-kafka/src/Messaging/KafkaRecordClient.php new file mode 100644 index 00000000..c69eb292 --- /dev/null +++ b/packages/eda-kafka/src/Messaging/KafkaRecordClient.php @@ -0,0 +1,28 @@ + $topics */ + public function subscribe(array $topics): void; + + /** Block up to $timeoutMs for one record; null on timeout or any non-NO_ERROR code. */ + public function consume(int $timeoutMs): ?KafkaRecord; + + public function commit(KafkaRecord $record): void; + + /** @param array $headers */ + public function produce(string $topic, string $value, ?string $key, array $headers): void; + + public function flush(int $timeoutMs): void; + + public function close(): void; +} diff --git a/packages/eda-kafka/src/Messaging/RdKafkaRecordClient.php b/packages/eda-kafka/src/Messaging/RdKafkaRecordClient.php new file mode 100644 index 00000000..ca959b32 --- /dev/null +++ b/packages/eda-kafka/src/Messaging/RdKafkaRecordClient.php @@ -0,0 +1,110 @@ +consumer()->subscribe($topics); + } + + public function consume(int $timeoutMs): ?KafkaRecord + { + $message = $this->consumer()->consume($timeoutMs); + if ($message->err !== RD_KAFKA_RESP_ERR_NO_ERROR) { + return null; + } + + return self::record($message->topic_name, $message->payload, $message->key, $message->headers, $message->partition, $message->offset, $message); + } + + /** + * The pure half of consume(), public and static so it is testable without the extension's Message class. + * + * @param array|null $headers + */ + public static function record(string $topic, ?string $payload, ?string $key, ?array $headers, int $partition, int $offset, mixed $handle): KafkaRecord + { + $strings = []; + foreach ($headers ?? [] as $name => $value) { + if (is_scalar($value)) { + $strings[(string) $name] = (string) $value; + } + } + + return new KafkaRecord($topic, $payload ?? '', $key, $strings, $partition, $offset, $handle); + } + + public function commit(KafkaRecord $record): void + { + if (! $record->handle instanceof Message) { + throw new RuntimeException('A Kafka commit replays the rdkafka Message the record was consumed as.'); + } + + $this->consumer()->commit($record->handle); + } + + public function produce(string $topic, string $value, ?string $key, array $headers): void + { + // RD_KAFKA_PARTITION_UA (-1): librdkafka picks the partition from the key hash. + $this->producer()->newTopic($topic)->producev(RD_KAFKA_PARTITION_UA, 0, $value, $key, $headers === [] ? null : $headers); + } + + public function flush(int $timeoutMs): void + { + if ($this->producer === null) { + return; + } + + $result = $this->producer->flush($timeoutMs); + if ($result !== RD_KAFKA_RESP_ERR_NO_ERROR) { + throw new MessagingException("Kafka did not acknowledge the produced records within {$timeoutMs}ms (librdkafka error {$result}).", 'MESSAGING_KAFKA_FLUSH_TIMEOUT'); + } + } + + public function close(): void + { + $this->consumer?->unsubscribe(); + $this->consumer = null; + } + + private function consumer(): KafkaConsumer + { + return $this->consumer ??= $this->consumers->consumer(); + } + + private function producer(): Producer + { + return $this->producer ??= $this->producers->producer(); + } +} diff --git a/packages/eda-kafka/tests/Fixtures/FakeKafkaRecordClient.php b/packages/eda-kafka/tests/Fixtures/FakeKafkaRecordClient.php new file mode 100644 index 00000000..638d7a4f --- /dev/null +++ b/packages/eda-kafka/tests/Fixtures/FakeKafkaRecordClient.php @@ -0,0 +1,73 @@ + */ + public array $incoming = []; + + /** @var list> */ + public array $subscriptions = []; + + /** @var list */ + public array $committed = []; + + /** @var list}> */ + public array $produced = []; + + public int $flushes = 0; + + public bool $closed = false; + + public ?Throwable $produceFailure = null; + + /** + * @param array $headers + */ + public static function record(string $topic, string $value, ?string $key = null, array $headers = [], int $offset = 0): KafkaRecord + { + return new KafkaRecord($topic, $value, $key, $headers, 0, $offset, null); + } + + public function subscribe(array $topics): void + { + $this->subscriptions[] = $topics; + } + + public function consume(int $timeoutMs): ?KafkaRecord + { + return array_shift($this->incoming); + } + + public function commit(KafkaRecord $record): void + { + $this->committed[] = $record; + } + + public function produce(string $topic, string $value, ?string $key, array $headers): void + { + if ($this->produceFailure !== null) { + throw $this->produceFailure; + } + + $this->produced[] = ['topic' => $topic, 'value' => $value, 'key' => $key, 'headers' => $headers]; + } + + public function flush(int $timeoutMs): void + { + $this->flushes++; + } + + public function close(): void + { + $this->closed = true; + } +} diff --git a/packages/eda-kafka/tests/Messaging/KafkaMessageBrokerTest.php b/packages/eda-kafka/tests/Messaging/KafkaMessageBrokerTest.php new file mode 100644 index 00000000..1419d458 --- /dev/null +++ b/packages/eda-kafka/tests/Messaging/KafkaMessageBrokerTest.php @@ -0,0 +1,172 @@ +, 2: FakeKafkaRecordClient} + */ +function w4KafkaBroker(int $maxAttempts = 1, bool $deadLetters = true): array +{ + /** @var ArrayObject $consumers */ + $consumers = new ArrayObject; + $producer = new FakeKafkaRecordClient; + $broker = new KafkaMessageBroker( + static function (string $group) use ($consumers): KafkaRecordClient { + return $consumers[$group] ??= new FakeKafkaRecordClient; + }, + $producer, + new RetryPolicy($maxAttempts, new LinearBackOff(0.0)), + new ListenerInvoker(static function (float $seconds): void {}), + deadLetterEnabled: $deadLetters, + ); + + return [$broker, $consumers, $producer]; +} + +/** + * The consumer the broker built for $group; failing loudly, rather than answering null, when it built none. + * + * @param ArrayObject $consumers + */ +function w4KafkaConsumer(ArrayObject $consumers, string $group): FakeKafkaRecordClient +{ + $consumer = $consumers[$group] ?? null; + if ($consumer === null) { + throw new LogicException("The broker built no consumer for group [{$group}]."); + } + + return $consumer; +} + +it('is a listener-container broker that publishes value, key and headers, and flushes', function () { + [$broker, , $producer] = w4KafkaBroker(); + + $broker->publish('orders', 'bytes', 'k-1', ['x-trace' => 't']); + + expect($broker)->toBeInstanceOf(ListenerContainerBroker::class) + ->and($producer->produced)->toBe([['topic' => 'orders', 'value' => 'bytes', 'key' => 'k-1', 'headers' => ['x-trace' => 't']]]) + ->and($producer->flushes)->toBe(1); +}); + +it('builds one consumer per group on the first poll, subscribed to that group\'s topics', function () { + [$broker, $consumers] = w4KafkaBroker(); + $noop = static function (Message $message): void {}; + $broker->subscribeListener('a', $noop, 'g1', new ListenerOptions); + $broker->subscribeListener('b', $noop, 'g2', new ListenerOptions); + $broker->subscribeListener('c', $noop, 'g1', new ListenerOptions); + $broker->subscribe('d', $noop); + + expect($broker->poll(10))->toBeFalse() + ->and(w4KafkaConsumer($consumers, 'g1')->subscriptions)->toBe([['a', 'c']]) + ->and(w4KafkaConsumer($consumers, 'g2')->subscriptions)->toBe([['b']]) + ->and(w4KafkaConsumer($consumers, 'firefly')->subscriptions)->toBe([['d']]) + ->and(fn () => $broker->subscribeListener('e', $noop, 'g1', new ListenerOptions))->toThrow(MessagingException::class, 'already polling'); +}); + +it('delivers a record to the listeners of its topic and commits it afterwards', function () { + [$broker, $consumers] = w4KafkaBroker(); + $seen = []; + $broker->subscribeListener('orders', function (Message $message) use (&$seen): void { + $seen[] = [$message->topic, $message->value, $message->key, $message->headers]; + }, null, new ListenerOptions); + $broker->poll(1); + w4KafkaConsumer($consumers, 'firefly')->incoming[] = FakeKafkaRecordClient::record('orders', 'v', 'k', ['h' => '1'], 7); + + expect($broker->poll(10))->toBeTrue() + ->and($seen)->toBe([['orders', 'v', 'k', ['h' => '1']]]) + ->and(w4KafkaConsumer($consumers, 'firefly')->committed)->toHaveCount(1); +}); + +it('retries under the policy, then dead-letters to .DLT with provenance headers, then commits', function () { + [$broker, $consumers, $producer] = w4KafkaBroker(maxAttempts: 3); + $calls = 0; + $broker->subscribeListener('orders', function () use (&$calls): void { + $calls++; + + throw new RuntimeException('handler down'); + }, null, new ListenerOptions); + $broker->poll(1); + w4KafkaConsumer($consumers, 'firefly')->incoming[] = FakeKafkaRecordClient::record('orders', 'v', 'k', ['h' => '1'], 42); + + $broker->poll(10); + + expect($calls)->toBe(3) + ->and($producer->produced)->toHaveCount(1) + ->and($producer->produced[0]['topic'])->toBe('orders.DLT') + ->and($producer->produced[0]['value'])->toBe('v') + ->and($producer->produced[0]['key'])->toBe('k') + ->and($producer->produced[0]['headers'])->toMatchArray([ + 'h' => '1', + 'x-original-topic' => 'orders', + 'x-exception' => 'RuntimeException', + 'x-exception-message' => 'handler down', + 'x-delivery-attempts' => '3', + 'x-dlt-reason' => 'RuntimeException', + 'x-dlt-source-topic' => 'orders', + 'x-dlt-source-offset' => '42', + ]) + ->and(w4KafkaConsumer($consumers, 'firefly')->committed)->toHaveCount(1); +}); + +it("routes to the listener's own dead-letter topic and honours its retries", function () { + [$broker, $consumers, $producer] = w4KafkaBroker(maxAttempts: 5); + $calls = 0; + $broker->subscribeListener('orders', function () use (&$calls): void { + $calls++; + + throw new RuntimeException('no'); + }, null, new ListenerOptions(2, new LinearBackOff(0.0), 'orders.parked')); + $broker->poll(1); + w4KafkaConsumer($consumers, 'firefly')->incoming[] = FakeKafkaRecordClient::record('orders', 'v'); + + $broker->poll(10); + + expect($calls)->toBe(2)->and($producer->produced[0]['topic'])->toBe('orders.parked'); +}); + +it('logs and skips after the last attempt when dead letters are off', function () { + [$broker, $consumers, $producer] = w4KafkaBroker(deadLetters: false); + $broker->subscribeListener('orders', static function (): void { + throw new RuntimeException('no'); + }, null, new ListenerOptions); + $broker->poll(1); + w4KafkaConsumer($consumers, 'firefly')->incoming[] = FakeKafkaRecordClient::record('orders', 'v'); + + $broker->poll(10); + + expect($producer->produced)->toBe([])->and(w4KafkaConsumer($consumers, 'firefly')->committed)->toHaveCount(1); +}); + +it('leaves the offset uncommitted when the dead letter cannot be produced', function () { + [$broker, $consumers, $producer] = w4KafkaBroker(); + $broker->subscribeListener('orders', static function (): void { + throw new RuntimeException('no'); + }, null, new ListenerOptions); + $broker->poll(1); + w4KafkaConsumer($consumers, 'firefly')->incoming[] = FakeKafkaRecordClient::record('orders', 'v'); + $producer->produceFailure = new RuntimeException('broker down'); + + expect(fn () => $broker->poll(10))->toThrow(MessagingException::class, 'left uncommitted') + ->and(w4KafkaConsumer($consumers, 'firefly')->committed)->toBe([]); +}); + +it('closes every consumer and flushes the producer on stop', function () { + [$broker, $consumers, $producer] = w4KafkaBroker(); + $broker->subscribeListener('orders', static function (): void {}, null, new ListenerOptions); + $broker->poll(1); + + $broker->stop(); + + expect(w4KafkaConsumer($consumers, 'firefly')->closed)->toBeTrue()->and($producer->flushes)->toBe(1); +}); diff --git a/packages/eda-kafka/tests/Messaging/KafkaMessagingAutoConfigurationTest.php b/packages/eda-kafka/tests/Messaging/KafkaMessagingAutoConfigurationTest.php new file mode 100644 index 00000000..e1975ad7 --- /dev/null +++ b/packages/eda-kafka/tests/Messaging/KafkaMessagingAutoConfigurationTest.php @@ -0,0 +1,37 @@ +getAttributes(ConditionalOnProperty::class)[0]->newInstance(); + + expect($condition->name)->toBe('firefly.messaging.provider')->and($condition->havingValue)->toBe('kafka'); +}); + +it('builds the Kafka broker without opening a connection', function () { + $config = new Config(new Repository(['firefly' => ['messaging' => ['provider' => 'kafka', 'kafka' => ['brokers' => '127.0.0.1:1']]]])); + + expect((new KafkaMessagingAutoConfiguration)->messageBroker($config, new Container))->toBeInstanceOf(KafkaMessageBroker::class); +})->skip(! extension_loaded('rdkafka'), 'needs ext-rdkafka'); + +it('wins over the core in-memory broker through the real boot', function () { + $app = fireflyApplication( + config: ['firefly' => ['messaging' => ['provider' => 'kafka', 'kafka' => ['brokers' => '127.0.0.1:1']]]], + providers: [MessagingServiceProvider::class, MessagingWiringProvider::class, EdaKafkaServiceProvider::class], + bindings: [MessageListenerManifest::class => new MessageListenerManifest([])], + ); + + expect($app->make(MessageBrokerPort::class))->toBeInstanceOf(KafkaMessageBroker::class); +})->skip(! extension_loaded('rdkafka'), 'needs ext-rdkafka'); diff --git a/packages/eda-kafka/tests/Messaging/RdKafkaRecordClientTest.php b/packages/eda-kafka/tests/Messaging/RdKafkaRecordClientTest.php new file mode 100644 index 00000000..0077362c --- /dev/null +++ b/packages/eda-kafka/tests/Messaging/RdKafkaRecordClientTest.php @@ -0,0 +1,21 @@ + '1', 'b' => 2, 'c' => null], 3, 41, 'handle'); + + expect($record->topic)->toBe('orders') + ->and($record->value)->toBe('v') + ->and($record->key)->toBe('k') + ->and($record->headers)->toBe(['a' => '1', 'b' => '2']) + ->and($record->partition)->toBe(3) + ->and($record->offset)->toBe(41) + ->and($record->handle)->toBe('handle'); +}); + +it('maps a tombstone (null payload) to an empty value', function () { + expect(RdKafkaRecordClient::record('orders', null, null, null, 0, 0, null)->value)->toBe(''); +}); From 74acc43a711de980b966fe6c7b3adbc1ba5485d4 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 19:41:10 -0700 Subject: [PATCH 22/47] docs(config): document firefly.messaging.kafka and firefly.messaging.listener.dead-letter.enabled in the skeleton --- skeleton/config/firefly.php | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/skeleton/config/firefly.php b/skeleton/config/firefly.php index 47cfec88..3593b5fc 100644 --- a/skeleton/config/firefly.php +++ b/skeleton/config/firefly.php @@ -2557,6 +2557,8 @@ | The point-to-point #[MessageListener] transport, independent of the event bus above. | 'memory' — in-process. The default. | 'queue' — Laravel queue; both keys UNSET mean the application's own defaults. + | 'kafka' — Kafka through firefly/eda-kafka (needs ext-rdkafka); a listener container driven by + | `php artisan firefly:messaging:consume`, with the retry and dead-letter keys below. | */ @@ -2592,6 +2594,39 @@ 'multiplier' => env('FIREFLY_MESSAGING_LISTENER_RETRY_MULTIPLIER', 2.0), 'max-delay' => env('FIREFLY_MESSAGING_LISTENER_RETRY_MAX_DELAY', '30s'), ], + + /* + | listener.dead-letter.enabled — what a listener container broker does with a delivery whose + | attempts ran out. true: it is published to the listener's own deadLetterTopic, or to the + | broker's default route (Kafka: ``), with x-original-topic, x-exception, + | x-exception-message and x-delivery-attempts headers, and only then acknowledged. false: it is + | logged and skipped — an explicit choice to lose it. A listener that names its own + | deadLetterTopic is always dead-lettered there. + | + | Default: true. + */ + 'dead-letter' => [ + 'enabled' => env('FIREFLY_MESSAGING_LISTENER_DEAD_LETTER_ENABLED', true), + ], + ], + + /* + | kafka — read only when provider is 'kafka'. Nothing connects at boot: one consumer per consumer + | group is built on the first poll, the producer on the first publish, and publish() flushes before + | it returns. Offsets are committed by hand, after every listener of a record handled it or had it + | dead-lettered; a dead letter that cannot be produced stops the worker with the offset uncommitted. + | + | brokers — UNSET means the event bus's firefly.eda.kafka.brokers (else 127.0.0.1:9092), so one + | cluster serves both transports unless you split them. + | group-id — the consumer group of a #[MessageListener] that names none. + | dlt-suffix — the default dead-letter topic is the record's topic plus this suffix. + | + | Default: group-id 'firefly', dlt-suffix '.DLT'. + */ + 'kafka' => [ + // 'brokers' => env('FIREFLY_MESSAGING_KAFKA_BROKERS', '127.0.0.1:9092'), + 'group-id' => env('FIREFLY_MESSAGING_KAFKA_GROUP_ID', 'firefly'), + 'dlt-suffix' => env('FIREFLY_MESSAGING_KAFKA_DLT_SUFFIX', '.DLT'), ], ], From 13addbb5df32b526e5170db10ec5895d19fe777e Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 19:46:07 -0700 Subject: [PATCH 23/47] fix(eda-kafka): fail a Kafka flush on a failed delivery report and round-robin poll() across consumer groups --- .../eda-kafka/src/KafkaProducerFactory.php | 17 ++++- .../src/Messaging/KafkaMessageBroker.php | 26 +++++-- .../src/Messaging/RdKafkaRecordClient.php | 42 +++++++++++- .../tests/Fixtures/FakeKafkaRecordClient.php | 11 +++ .../Messaging/KafkaMessageBrokerTest.php | 67 +++++++++++++++++++ .../Messaging/RdKafkaRecordClientTest.php | 39 +++++++++++ 6 files changed, 193 insertions(+), 9 deletions(-) diff --git a/packages/eda-kafka/src/KafkaProducerFactory.php b/packages/eda-kafka/src/KafkaProducerFactory.php index e6898668..5e406422 100644 --- a/packages/eda-kafka/src/KafkaProducerFactory.php +++ b/packages/eda-kafka/src/KafkaProducerFactory.php @@ -4,7 +4,9 @@ namespace Firefly\Eda\Kafka; +use Closure; use RdKafka\Conf; +use RdKafka\Message; use RdKafka\Producer; use RuntimeException; @@ -23,6 +25,13 @@ * BEFORE that point when the extension is missing — so this declaration is never evaluated against a real value * on a no-ext machine. The genuinely load-bearing guard is extension_loaded('rdkafka') as the FIRST statement of * producer(), not the absence of a `use` statement. + * + * $deliveryReport is librdkafka's delivery-report callback (dr_msg_cb), served from flush()/poll() once per + * produced message with its final outcome. It is the ONLY place a permanent delivery failure surfaces — + * TOPIC_AUTHORIZATION_FAILED, a record over the topic's max.message.bytes, a message that timed out — because + * Producer::flush() answers NO_ERROR as soon as the out-queue is empty, failed messages included. The eda + * publisher passes none (its defaults are unchanged); the firefly/messaging RdKafkaRecordClient passes one so a + * dead letter it could not deliver is never acknowledged. */ final class KafkaProducerFactory { @@ -33,7 +42,10 @@ public static function available(): bool return extension_loaded('rdkafka'); } - public function producer(): Producer + /** + * @param (Closure(Producer, Message): void)|null $deliveryReport + */ + public function producer(?Closure $deliveryReport = null): Producer { if (! self::available()) { throw new RuntimeException('The Kafka adapter requires ext-rdkafka (firefly.eda.provider=kafka). Install librdkafka + the rdkafka extension, or choose another provider.'); @@ -41,6 +53,9 @@ public function producer(): Producer $conf = new Conf; $conf->set('metadata.broker.list', $this->brokers); + if ($deliveryReport !== null) { + $conf->setDrMsgCb($deliveryReport); + } return new Producer($conf); } diff --git a/packages/eda-kafka/src/Messaging/KafkaMessageBroker.php b/packages/eda-kafka/src/Messaging/KafkaMessageBroker.php index 2b48a767..3c9a163a 100644 --- a/packages/eda-kafka/src/Messaging/KafkaMessageBroker.php +++ b/packages/eda-kafka/src/Messaging/KafkaMessageBroker.php @@ -28,10 +28,15 @@ * first), then its record is produced to its own deadLetterTopic, or to `` (`.DLT`), with the * diagnostic headers and the eda adapter's x-dlt-* provenance headers PyFly also writes. With dead letters off * (firefly.messaging.listener.dead-letter.enabled=false) the record is logged and skipped — an explicit choice. - * * A dead letter that cannot be produced throws OUT of poll() with the offset NOT committed: the worker exits, - * its supervisor restarts it, and the record is consumed again — never silently lost. - * * publish() produces and flushes before returning, so a message published from a PHP-FPM request is on the - * broker when the request ends. + * * A dead letter that cannot be produced — refused at produce(), not acknowledged within the flush timeout, or + * reported as failed by librdkafka's delivery report (RdKafkaRecordClient::flush() throws on each) — throws + * OUT of poll() with the offset NOT committed: the worker exits, its supervisor restarts it, and the record is + * consumed again — never silently lost. + * * publish() produces and flushes before returning, and throws when the broker did not take the message, so a + * message published from a PHP-FPM request that returns normally is on the broker when the request ends. + * * FAIR ACROSS GROUPS: each poll() starts at the group after the one that delivered last, so a backlog on one + * group never starves another group's listeners, nor keeps its consumer from calling consume() for longer than + * max.poll.interval.ms (which would make librdkafka drop it from its group and rebalance). */ final class KafkaMessageBroker implements ListenerContainerBroker { @@ -41,6 +46,9 @@ final class KafkaMessageBroker implements ListenerContainerBroker /** @var array */ private array $consumers = []; + /** The index, in subscription order, of the group the next poll() asks first. */ + private int $cursor = 0; + /** * @param Closure(string): KafkaRecordClient $consumerFor builds the consumer of one group */ @@ -83,16 +91,20 @@ public function poll(int $timeoutMs): bool return false; } - $slice = max(1, intdiv($timeoutMs, count($this->listeners))); + $groups = array_map('strval', array_keys($this->listeners)); + $count = count($groups); + $slice = max(1, intdiv($timeoutMs, $count)); - foreach (array_keys($this->listeners) as $group) { - $group = (string) $group; + for ($i = 0; $i < $count; $i++) { + $index = ($this->cursor + $i) % $count; + $group = $groups[$index]; $consumer = $this->consumer($group); $record = $consumer->consume($slice); if ($record === null) { continue; } + $this->cursor = ($index + 1) % $count; $this->dispatch($group, $record); $consumer->commit($record); diff --git a/packages/eda-kafka/src/Messaging/RdKafkaRecordClient.php b/packages/eda-kafka/src/Messaging/RdKafkaRecordClient.php index ca959b32..7a99156c 100644 --- a/packages/eda-kafka/src/Messaging/RdKafkaRecordClient.php +++ b/packages/eda-kafka/src/Messaging/RdKafkaRecordClient.php @@ -19,6 +19,14 @@ * after the factories' extension_loaded() guard; every `RD_KAFKA_*` constant sits in a body that runs only after it, * so this file autoloads on a machine without the extension (ReflectionFreeEdaKafkaTest requires it to). * + * DELIVERY IS CONFIRMED PER MESSAGE, not per flush. Producer::flush() answers NO_ERROR once librdkafka's out-queue + * is empty, and a message that failed delivery permanently (TOPIC_AUTHORIZATION_FAILED, over the topic's + * max.message.bytes, timed out) leaves that queue too — its failure is reported only to the delivery-report + * callback. So the producer is built with one (KafkaProducerFactory::producer($deliveryReport)), every failed + * report is collected by delivered(), and flush() throws MessagingException when any arrived since the last flush: + * KafkaMessageBroker then leaves the source offset uncommitted, and publish() fails instead of answering as if the + * message were on the broker. + * * close() unsubscribes and drops the consumer instead of calling KafkaConsumer::close(), for the thread leak * RdKafkaConsumerClient::close() documents (ext-rdkafka 6.0.5 never destroys a closed handle). */ @@ -28,6 +36,9 @@ final class RdKafkaRecordClient implements KafkaRecordClient private ?Producer $producer = null; + /** @var list the delivery failures reported since the last flush() */ + private array $undelivered = []; + public function __construct( private readonly KafkaConsumerFactory $consumers, private readonly KafkaProducerFactory $producers, @@ -83,6 +94,8 @@ public function produce(string $topic, string $value, ?string $key, array $heade public function flush(int $timeoutMs): void { if ($this->producer === null) { + $this->throwIfUndelivered(); + return; } @@ -90,6 +103,31 @@ public function flush(int $timeoutMs): void if ($result !== RD_KAFKA_RESP_ERR_NO_ERROR) { throw new MessagingException("Kafka did not acknowledge the produced records within {$timeoutMs}ms (librdkafka error {$result}).", 'MESSAGING_KAFKA_FLUSH_TIMEOUT'); } + + $this->throwIfUndelivered(); + } + + /** + * The pure half of the delivery-report callback, public so it is testable without a broker: an error code other + * than 0 (RD_KAFKA_RESP_ERR_NO_ERROR) is a message librdkafka gave up on, held until the next flush() throws it. + */ + public function delivered(int $error, string $reason, ?string $topic): void + { + if ($error !== 0) { + $this->undelivered[] = '['.($topic ?? '?')."] {$reason} (librdkafka error {$error})"; + } + } + + private function throwIfUndelivered(): void + { + if ($this->undelivered === []) { + return; + } + + $failures = implode('; ', $this->undelivered); + $this->undelivered = []; + + throw new MessagingException("Kafka rejected produced record(s): {$failures}.", 'MESSAGING_KAFKA_DELIVERY_FAILED'); } public function close(): void @@ -105,6 +143,8 @@ private function consumer(): KafkaConsumer private function producer(): Producer { - return $this->producer ??= $this->producers->producer(); + return $this->producer ??= $this->producers->producer(function (Producer $producer, Message $message): void { + $this->delivered($message->err, $message->errstr(), $message->topic_name); + }); } } diff --git a/packages/eda-kafka/tests/Fixtures/FakeKafkaRecordClient.php b/packages/eda-kafka/tests/Fixtures/FakeKafkaRecordClient.php index 638d7a4f..ca6ffe61 100644 --- a/packages/eda-kafka/tests/Fixtures/FakeKafkaRecordClient.php +++ b/packages/eda-kafka/tests/Fixtures/FakeKafkaRecordClient.php @@ -29,6 +29,12 @@ final class FakeKafkaRecordClient implements KafkaRecordClient public ?Throwable $produceFailure = null; + /** Thrown by flush(): what RdKafkaRecordClient does when a delivery report says a produced record failed. */ + public ?Throwable $flushFailure = null; + + /** @var list the timeout of every consume() call, in order */ + public array $consumeCalls = []; + /** * @param array $headers */ @@ -44,6 +50,8 @@ public function subscribe(array $topics): void public function consume(int $timeoutMs): ?KafkaRecord { + $this->consumeCalls[] = $timeoutMs; + return array_shift($this->incoming); } @@ -64,6 +72,9 @@ public function produce(string $topic, string $value, ?string $key, array $heade public function flush(int $timeoutMs): void { $this->flushes++; + if ($this->flushFailure !== null) { + throw $this->flushFailure; + } } public function close(): void diff --git a/packages/eda-kafka/tests/Messaging/KafkaMessageBrokerTest.php b/packages/eda-kafka/tests/Messaging/KafkaMessageBrokerTest.php index 1419d458..60cfb900 100644 --- a/packages/eda-kafka/tests/Messaging/KafkaMessageBrokerTest.php +++ b/packages/eda-kafka/tests/Messaging/KafkaMessageBrokerTest.php @@ -161,6 +161,73 @@ function w4KafkaConsumer(ArrayObject $consumers, string $group): FakeKafkaRecord ->and(w4KafkaConsumer($consumers, 'firefly')->committed)->toBe([]); }); +it('leaves the offset uncommitted when the dead letter is produced but its delivery report says it failed', function () { + [$broker, $consumers, $producer] = w4KafkaBroker(); + $broker->subscribeListener('orders', static function (): void { + throw new RuntimeException('no'); + }, null, new ListenerOptions); + $broker->poll(1); + w4KafkaConsumer($consumers, 'firefly')->incoming[] = FakeKafkaRecordClient::record('orders', 'v'); + $producer->flushFailure = new MessagingException('Kafka rejected produced record(s): [orders.DLT] Broker: Topic authorization failed.', 'MESSAGING_KAFKA_DELIVERY_FAILED'); + + expect(fn () => $broker->poll(10))->toThrow(MessagingException::class, 'left uncommitted') + ->and($producer->produced)->toHaveCount(1) + ->and(w4KafkaConsumer($consumers, 'firefly')->committed)->toBe([]); +}); + +it('fails publish() when the delivery report says the broker rejected the message', function () { + [$broker, , $producer] = w4KafkaBroker(); + $producer->flushFailure = new MessagingException('Kafka rejected produced record(s).', 'MESSAGING_KAFKA_DELIVERY_FAILED'); + + expect(fn () => $broker->publish('orders', 'v'))->toThrow(MessagingException::class, 'rejected'); +}); + +it('round-robins across consumer groups so a backlog on one never starves another', function () { + [$broker, $consumers] = w4KafkaBroker(); + $seen = []; + $broker->subscribeListener('a', function (Message $message) use (&$seen): void { + $seen[] = $message->value; + }, 'g1', new ListenerOptions); + $broker->subscribeListener('b', function (Message $message) use (&$seen): void { + $seen[] = $message->value; + }, 'g2', new ListenerOptions); + $broker->poll(1); + $g1 = w4KafkaConsumer($consumers, 'g1'); + $g2 = w4KafkaConsumer($consumers, 'g2'); + $g1->consumeCalls = $g2->consumeCalls = []; + foreach ([1, 2, 3] as $n) { + $g1->incoming[] = FakeKafkaRecordClient::record('a', "a{$n}", offset: $n); + $g2->incoming[] = FakeKafkaRecordClient::record('b', "b{$n}", offset: $n); + } + + for ($i = 0; $i < 6; $i++) { + expect($broker->poll(10))->toBeTrue(); + } + + expect($seen)->toBe(['a1', 'b1', 'a2', 'b2', 'a3', 'b3']) + ->and($g1->consumeCalls)->toHaveCount(3) + ->and($g2->consumeCalls)->toHaveCount(3) + ->and($g1->committed)->toHaveCount(3) + ->and($g2->committed)->toHaveCount(3); +}); + +it('still reaches the next group when the one asked first has nothing', function () { + [$broker, $consumers] = w4KafkaBroker(); + $seen = []; + $broker->subscribeListener('a', static function (): void {}, 'g1', new ListenerOptions); + $broker->subscribeListener('b', function (Message $message) use (&$seen): void { + $seen[] = $message->value; + }, 'g2', new ListenerOptions); + $broker->poll(1); + w4KafkaConsumer($consumers, 'g2')->incoming[] = FakeKafkaRecordClient::record('b', 'b1'); + w4KafkaConsumer($consumers, 'g2')->incoming[] = FakeKafkaRecordClient::record('b', 'b2', offset: 1); + + expect($broker->poll(10))->toBeTrue() + ->and($broker->poll(10))->toBeTrue() + ->and($broker->poll(10))->toBeFalse() + ->and($seen)->toBe(['b1', 'b2']); +}); + it('closes every consumer and flushes the producer on stop', function () { [$broker, $consumers, $producer] = w4KafkaBroker(); $broker->subscribeListener('orders', static function (): void {}, null, new ListenerOptions); diff --git a/packages/eda-kafka/tests/Messaging/RdKafkaRecordClientTest.php b/packages/eda-kafka/tests/Messaging/RdKafkaRecordClientTest.php index 0077362c..fa6740f9 100644 --- a/packages/eda-kafka/tests/Messaging/RdKafkaRecordClientTest.php +++ b/packages/eda-kafka/tests/Messaging/RdKafkaRecordClientTest.php @@ -2,7 +2,17 @@ declare(strict_types=1); +use Firefly\Eda\Kafka\KafkaConsumerFactory; +use Firefly\Eda\Kafka\KafkaProducerFactory; use Firefly\Eda\Kafka\Messaging\RdKafkaRecordClient; +use Firefly\Messaging\Exception\MessagingException; +use RdKafka\Message; +use RdKafka\Producer; + +function w4RecordClient(): RdKafkaRecordClient +{ + return new RdKafkaRecordClient(new KafkaConsumerFactory('127.0.0.1:59999', 'firefly-dr-probe'), new KafkaProducerFactory('127.0.0.1:59999')); +} it('maps a consumed payload into a record, keeping only string-valued headers', function () { $record = RdKafkaRecordClient::record('orders', 'v', 'k', ['a' => '1', 'b' => 2, 'c' => null], 3, 41, 'handle'); @@ -19,3 +29,32 @@ it('maps a tombstone (null payload) to an empty value', function () { expect(RdKafkaRecordClient::record('orders', null, null, null, 0, 0, null)->value)->toBe(''); }); + +it('throws from the next flush() when a delivery report said a produced record failed, then forgets it', function () { + $client = w4RecordClient(); + $client->delivered(0, 'Success', 'orders'); + $client->flush(10); + + $client->delivered(29, 'Broker: Topic authorization failed', 'orders.DLT'); + + expect(fn () => $client->flush(10))->toThrow(MessagingException::class, '[orders.DLT] Broker: Topic authorization failed (librdkafka error 29)'); + $client->flush(10); +}); + +/* + * The wiring half, against the real ext-rdkafka and no broker: a message queued for an unroutable bootstrap address + * and then purged from the queue fails delivery with ERR__PURGE_QUEUE, which librdkafka reports ONLY to dr_msg_cb — + * while flush() itself answers NO_ERROR, because the out-queue is empty. That is exactly the gap a callback-less + * producer leaves open. + */ +it('hands every delivery report to the callback given to KafkaProducerFactory, failures flush() does not report', function () { + $reports = []; + $producer = (new KafkaProducerFactory('127.0.0.1:59999'))->producer(function (Producer $producer, Message $message) use (&$reports): void { + $reports[] = $message->err; + }); + $producer->newTopic('orders')->producev(RD_KAFKA_PARTITION_UA, 0, 'v'); + $producer->purge(RD_KAFKA_PURGE_F_QUEUE); + + expect($producer->flush(2000))->toBe(RD_KAFKA_RESP_ERR_NO_ERROR) + ->and($reports)->toBe([RD_KAFKA_RESP_ERR__PURGE_QUEUE]); +})->skip(! extension_loaded('rdkafka'), 'needs ext-rdkafka'); From 2caa9700d910896e901d2efe86c9a5c0c27a67cc Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 19:54:07 -0700 Subject: [PATCH 24/47] test(eda-kafka): prove the messaging listener container against a real Kafka, dead-letter topic included --- .../KafkaMessagingRoundTripTest.php | 118 ++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100644 packages/eda-kafka/tests/Integration/KafkaMessagingRoundTripTest.php diff --git a/packages/eda-kafka/tests/Integration/KafkaMessagingRoundTripTest.php b/packages/eda-kafka/tests/Integration/KafkaMessagingRoundTripTest.php new file mode 100644 index 00000000..8e0c1bc5 --- /dev/null +++ b/packages/eda-kafka/tests/Integration/KafkaMessagingRoundTripTest.php @@ -0,0 +1,118 @@ + new RdKafkaRecordClient(new KafkaConsumerFactory($brokers, $g), new KafkaProducerFactory($brokers)), + new RdKafkaRecordClient(new KafkaConsumerFactory($brokers), new KafkaProducerFactory($brokers)), + new RetryPolicy($maxAttempts, new LinearBackOff(0.0)), + new ListenerInvoker(static function (float $seconds): void {}), + $group, + ); +} + +/** + * Polls until $done says so or 45s pass — a fresh consumer group needs a rebalance before it is assigned. $done + * must read the test's state BY REFERENCE (`function () use (&$seen)`): an arrow fn captures by value, would never + * see the listener's writes, and would turn every run into a 45-second wait that still passes. + */ +function w4KafkaPollUntil(KafkaMessageBroker $broker, Closure $done): void +{ + // Seconds, not a loop count: each poll() returns as soon as a record arrives. + $deadline = microtime(true) + 45; + while (! $done() && microtime(true) < $deadline) { + $broker->poll(1000); + } +} + +/** The next record a fresh consumer of $group reads from $topic within 45s, or null. */ +function w4KafkaFirstRecord(string $group, string $topic): ?KafkaRecord +{ + $brokers = (string) getenv('FIREFLY_KAFKA_BROKERS'); + $client = new RdKafkaRecordClient(new KafkaConsumerFactory($brokers, $group), new KafkaProducerFactory($brokers)); + $client->subscribe([$topic]); + $record = null; + $deadline = microtime(true) + 45; + while (! $record instanceof KafkaRecord && microtime(true) < $deadline) { + $record = $client->consume(1000); + } + $client->close(); + + return $record; +} + +it('round-trips publish -> poll -> listener -> commit on a real Kafka, with key and headers', function () { + $topic = 'firefly.messaging.it.'.bin2hex(random_bytes(4)); + $group = 'firefly-it-'.bin2hex(random_bytes(4)); + $broker = w4KafkaMessaging($group); + $seen = []; + $broker->subscribeListener($topic, function (Message $message) use (&$seen): void { + $seen[] = [$message->value, $message->key, $message->headers['x-trace'] ?? null]; + }, null, new ListenerOptions); + + $broker->publish($topic, 'hello', 'k-1', ['x-trace' => 't-1']); + w4KafkaPollUntil($broker, function () use (&$seen): bool { + return $seen !== []; + }); + $broker->stop(); + + // The commit is proven, not assumed: a marker published after it is the FIRST thing a fresh consumer of the + // same group reads — had the 'hello' offset stayed uncommitted, Kafka would hand 'hello' back first. + $broker->publish($topic, 'marker'); + $next = w4KafkaFirstRecord($group, $topic); + + expect($seen)->toBe([['hello', 'k-1', 't-1']]) + ->and($next?->value)->toBe('marker'); +})->skip(! extension_loaded('rdkafka') || getenv('FIREFLY_KAFKA_BROKERS') === false, 'Set FIREFLY_KAFKA_BROKERS (and load ext-rdkafka) to run the Kafka messaging round trip.')->group('integration'); + +it('retries a failing listener, then produces the record to .DLT with provenance headers', function () { + $topic = 'firefly.messaging.it.'.bin2hex(random_bytes(4)); + $broker = w4KafkaMessaging('firefly-it-'.bin2hex(random_bytes(4)), maxAttempts: 2); + $calls = 0; + $broker->subscribeListener($topic, function () use (&$calls): void { + $calls++; + + throw new RuntimeException('listener down'); + }, null, new ListenerOptions); + + $broker->publish($topic, 'payload', 'k-9'); + w4KafkaPollUntil($broker, function () use (&$calls): bool { + return $calls >= 2; + }); + $broker->stop(); + + $dead = w4KafkaFirstRecord('firefly-it-dlt-'.bin2hex(random_bytes(4)), $topic.'.DLT'); + + expect($calls)->toBe(2) + ->and($dead)->toBeInstanceOf(KafkaRecord::class) + ->and($dead?->value)->toBe('payload') + ->and($dead?->key)->toBe('k-9') + ->and($dead?->headers)->toMatchArray([ + 'x-original-topic' => $topic, + 'x-exception' => 'RuntimeException', + 'x-exception-message' => 'listener down', + 'x-delivery-attempts' => '2', + 'x-dlt-reason' => 'RuntimeException', + 'x-dlt-source-topic' => $topic, + 'x-dlt-source-offset' => '0', + ]); +})->skip(! extension_loaded('rdkafka') || getenv('FIREFLY_KAFKA_BROKERS') === false, 'Set FIREFLY_KAFKA_BROKERS (and load ext-rdkafka) to run the Kafka messaging round trip.')->group('integration'); From b06db57d1340475cfd20e6a1c050be5ee4eb59ac Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 19:57:26 -0700 Subject: [PATCH 25/47] feat(eda-rabbitmq): a RabbitMQ listener container for firefly/messaging with retries, back-off and a dead-letter exchange --- deptrac.yaml | 6 +- .../cache/firefly-eda-rabbitmq-components.php | 31 ++- .../cache/firefly-eda-rabbitmq-context.php | 28 ++- packages/eda-rabbitmq/composer.json | 6 +- .../src/Messaging/AmqpDelivery.php | 20 ++ .../src/Messaging/AmqpMessagingChannel.php | 39 +++ .../Messaging/PhpAmqpLibMessagingChannel.php | 125 ++++++++++ .../src/Messaging/RabbitMqMessageBroker.php | 222 ++++++++++++++++++ .../RabbitMqMessagingAutoConfiguration.php | 47 ++++ .../Fixtures/FakeAmqpMessagingChannel.php | 106 +++++++++ .../Messaging/RabbitMqMessageBrokerTest.php | 137 +++++++++++ ...RabbitMqMessagingAutoConfigurationTest.php | 37 +++ 12 files changed, 799 insertions(+), 5 deletions(-) create mode 100644 packages/eda-rabbitmq/src/Messaging/AmqpDelivery.php create mode 100644 packages/eda-rabbitmq/src/Messaging/AmqpMessagingChannel.php create mode 100644 packages/eda-rabbitmq/src/Messaging/PhpAmqpLibMessagingChannel.php create mode 100644 packages/eda-rabbitmq/src/Messaging/RabbitMqMessageBroker.php create mode 100644 packages/eda-rabbitmq/src/Messaging/RabbitMqMessagingAutoConfiguration.php create mode 100644 packages/eda-rabbitmq/tests/Fixtures/FakeAmqpMessagingChannel.php create mode 100644 packages/eda-rabbitmq/tests/Messaging/RabbitMqMessageBrokerTest.php create mode 100644 packages/eda-rabbitmq/tests/Messaging/RabbitMqMessagingAutoConfigurationTest.php diff --git a/deptrac.yaml b/deptrac.yaml index 479e1162..674bc99b 100644 --- a/deptrac.yaml +++ b/deptrac.yaml @@ -241,7 +241,10 @@ deptrac: # broker its own package, the SchedulingPostgres precedent) so the core eda package stays free of the # php-amqplib dependency. It reads Config indirectly via #[ConditionalOnProperty], carries Container/Context # attributes, extends AutoConfigure's base, implements Eda's EventPublisher, and supplies an Actuator - # HealthIndicator. It is depended on by NONE. + # HealthIndicator. It is depended on by NONE. The => Messaging edge is the raw-bytes messaging adapter + # (src/Messaging): RabbitMqMessageBroker implements Messaging's ListenerContainerBroker over this package's + # RabbitMqConnectionFactory, so both transports share one connection configuration. Core Messaging still has no + # edge to or from Eda. EdaRabbitmq: - Kernel - Container @@ -250,6 +253,7 @@ deptrac: - AutoConfigure - Eda - Actuator + - Messaging # EdaPostgres is the GENUINE same-transaction Postgres OUTBOX adapter for Eda's EventPublisher port: its own # package (every broker its own package, the SchedulingPostgres precedent) so the core eda package stays free of diff --git a/packages/eda-rabbitmq/cache/firefly-eda-rabbitmq-components.php b/packages/eda-rabbitmq/cache/firefly-eda-rabbitmq-components.php index 0bee1848..c0b95281 100644 --- a/packages/eda-rabbitmq/cache/firefly-eda-rabbitmq-components.php +++ b/packages/eda-rabbitmq/cache/firefly-eda-rabbitmq-components.php @@ -6,6 +6,35 @@ return [ 0 => [ + 'class' => 'Firefly\\Eda\\Rabbitmq\\Messaging\\RabbitMqMessagingAutoConfiguration', + 'stereotype' => 'configuration', + 'name' => null, + 'scope' => 'Singleton', + 'primary' => false, + 'order' => 900, + 'qualifier' => null, + 'interfaces' => [ + ], + 'beans' => [ + 0 => [ + 'method' => 'messageBroker', + 'returns' => 'Firefly\\Messaging\\MessageBrokerPort', + 'name' => null, + 'scope' => 'Singleton', + 'primary' => false, + 'order' => 0, + 'lazy' => false, + 'dependencies' => [ + 0 => 'Firefly\\Config\\Config', + 1 => 'Illuminate\\Container\\Container', + ], + ], + ], + 'lazy' => false, + 'dependencies' => [ + ], + ], + 1 => [ 'class' => 'Firefly\\Eda\\Rabbitmq\\RabbitMqAutoConfiguration', 'stereotype' => 'configuration', 'name' => null, @@ -84,7 +113,7 @@ 'dependencies' => [ ], ], - 1 => [ + 2 => [ 'class' => 'Firefly\\Eda\\Rabbitmq\\RabbitMqHealthIndicator', 'stereotype' => 'component', 'name' => null, diff --git a/packages/eda-rabbitmq/cache/firefly-eda-rabbitmq-context.php b/packages/eda-rabbitmq/cache/firefly-eda-rabbitmq-context.php index 62b09626..f0bd9e91 100644 --- a/packages/eda-rabbitmq/cache/firefly-eda-rabbitmq-context.php +++ b/packages/eda-rabbitmq/cache/firefly-eda-rabbitmq-context.php @@ -6,6 +6,32 @@ return [ 0 => [ + 'class' => 'Firefly\\Eda\\Rabbitmq\\Messaging\\RabbitMqMessagingAutoConfiguration', + 'postConstruct' => [ + ], + 'preDestroy' => [ + ], + 'listeners' => [ + ], + 'conditions' => [ + ], + 'beanConditions' => [ + 0 => [ + 'method' => 'messageBroker', + 'conditions' => [ + 0 => [ + 'type' => 'Firefly\\Context\\Condition\\Attributes\\ConditionalOnProperty', + 'args' => [ + 0 => 'firefly.messaging.provider', + 1 => 'rabbitmq', + 2 => false, + ], + ], + ], + ], + ], + ], + 1 => [ 'class' => 'Firefly\\Eda\\Rabbitmq\\RabbitMqAutoConfiguration', 'postConstruct' => [ ], @@ -89,7 +115,7 @@ ], ], ], - 1 => [ + 2 => [ 'class' => 'Firefly\\Eda\\Rabbitmq\\RabbitMqHealthIndicator', 'postConstruct' => [ ], diff --git a/packages/eda-rabbitmq/composer.json b/packages/eda-rabbitmq/composer.json index 719fe349..6b4291f7 100644 --- a/packages/eda-rabbitmq/composer.json +++ b/packages/eda-rabbitmq/composer.json @@ -1,6 +1,6 @@ { "name": "firefly/eda-rabbitmq", - "description": "LaraFly eda RabbitMQ adapter: a RabbitMqEventPublisher + long-running consumer over php-amqplib (AMQP 0-9-1), auto-configured behind firefly.eda.provider=rabbitmq, with a DLX for exhausted retries and a RabbitMqHealthIndicator.", + "description": "LaraFly eda RabbitMQ adapter: a RabbitMqEventPublisher + long-running consumer over php-amqplib (AMQP 0-9-1), auto-configured behind firefly.eda.provider=rabbitmq, with a DLX for exhausted retries and a RabbitMqHealthIndicator; also the RabbitMQ adapter for firefly/messaging (firefly.messaging.provider=rabbitmq).", "type": "library", "license": "Apache-2.0", "homepage": "https://github.com/fireflyframework/fireflyframework-php", @@ -28,10 +28,12 @@ "firefly/context": "*@dev", "firefly/eda": "*@dev", "firefly/kernel": "*@dev", + "firefly/messaging": "*@dev", "illuminate/console": "^13.0", "illuminate/contracts": "^13.0", "illuminate/support": "^13.0", - "php-amqplib/php-amqplib": "^3.7" + "php-amqplib/php-amqplib": "^3.7", + "psr/log": "^3.0" }, "extra": { "laravel": { diff --git a/packages/eda-rabbitmq/src/Messaging/AmqpDelivery.php b/packages/eda-rabbitmq/src/Messaging/AmqpDelivery.php new file mode 100644 index 00000000..b3425f6a --- /dev/null +++ b/packages/eda-rabbitmq/src/Messaging/AmqpDelivery.php @@ -0,0 +1,20 @@ + $headers + */ + public function __construct( + public string $queue, + public string $routingKey, + public string $body, + public array $headers, + public int $deliveryTag, + ) {} +} diff --git a/packages/eda-rabbitmq/src/Messaging/AmqpMessagingChannel.php b/packages/eda-rabbitmq/src/Messaging/AmqpMessagingChannel.php new file mode 100644 index 00000000..050e1d55 --- /dev/null +++ b/packages/eda-rabbitmq/src/Messaging/AmqpMessagingChannel.php @@ -0,0 +1,39 @@ + $arguments */ + public function declareQueue(string $queue, array $arguments): void; + + public function bindQueue(string $queue, string $exchange, string $routingKey): void; + + public function qos(int $prefetch): void; + + /** @param array $headers */ + public function publish(string $exchange, string $routingKey, string $body, array $headers): void; + + /** @param Closure(AmqpDelivery): void $callback */ + public function consume(string $queue, Closure $callback): void; + + public function wait(float $timeoutSeconds): bool; + + public function ack(int $deliveryTag): void; + + public function nack(int $deliveryTag, bool $requeue): void; + + public function close(): void; +} diff --git a/packages/eda-rabbitmq/src/Messaging/PhpAmqpLibMessagingChannel.php b/packages/eda-rabbitmq/src/Messaging/PhpAmqpLibMessagingChannel.php new file mode 100644 index 00000000..acec6133 --- /dev/null +++ b/packages/eda-rabbitmq/src/Messaging/PhpAmqpLibMessagingChannel.php @@ -0,0 +1,125 @@ +channel()->exchange_declare($exchange, $type, false, true, false); + } + + public function declareQueue(string $queue, array $arguments): void + { + $this->channel()->queue_declare($queue, false, true, false, false, false, new AMQPTable($arguments)); + } + + public function bindQueue(string $queue, string $exchange, string $routingKey): void + { + $this->channel()->queue_bind($queue, $exchange, $routingKey); + } + + public function qos(int $prefetch): void + { + $this->channel()->basic_qos(0, $prefetch, false); + } + + public function publish(string $exchange, string $routingKey, string $body, array $headers): void + { + $message = new AMQPMessage($body, [ + 'delivery_mode' => AMQPMessage::DELIVERY_MODE_PERSISTENT, + 'message_id' => bin2hex(random_bytes(16)), + 'application_headers' => new AMQPTable($headers), + ]); + + $this->channel()->basic_publish($message, $exchange, $routingKey); + } + + public function consume(string $queue, Closure $callback): void + { + $this->channel()->basic_consume($queue, '', false, false, false, false, static function (AMQPMessage $message) use ($queue, $callback): void { + $callback(new AmqpDelivery($queue, (string) $message->getRoutingKey(), $message->getBody(), self::headers($message), (int) $message->getDeliveryTag())); + }); + } + + public function wait(float $timeoutSeconds): bool + { + try { + $this->channel()->wait(null, false, $timeoutSeconds); + + return true; + } catch (AMQPTimeoutException) { + return false; + } + } + + public function ack(int $deliveryTag): void + { + $this->channel()->basic_ack($deliveryTag); + } + + public function nack(int $deliveryTag, bool $requeue): void + { + $this->channel()->basic_nack($deliveryTag, false, $requeue); + } + + public function close(): void + { + $this->channel?->close(); + $this->connection?->close(); + $this->channel = null; + $this->connection = null; + } + + /** + * @return array + */ + private static function headers(AMQPMessage $message): array + { + if (! $message->has('application_headers')) { + return []; + } + + $table = $message->get('application_headers'); + $headers = []; + foreach ($table instanceof AMQPTable ? $table->getNativeData() : [] as $name => $value) { + if (is_scalar($value)) { + $headers[(string) $name] = (string) $value; + } + } + + return $headers; + } + + private function channel(): AMQPChannel + { + if ($this->channel === null) { + $this->connection = $this->connections->connect(); + $this->channel = $this->connection->channel(); + } + + return $this->channel; + } +} diff --git a/packages/eda-rabbitmq/src/Messaging/RabbitMqMessageBroker.php b/packages/eda-rabbitmq/src/Messaging/RabbitMqMessageBroker.php new file mode 100644 index 00000000..b1a47c76 --- /dev/null +++ b/packages/eda-rabbitmq/src/Messaging/RabbitMqMessageBroker.php @@ -0,0 +1,222 @@ +` — bound to its topic and + * declared with x-dead-letter-exchange = the DLX (`.dlx` unless configured) and + * x-dead-letter-routing-key = the queue, so a rejected message lands in `.dlq`, bound on the DLX; + * * manual acknowledgement with a bounded prefetch. + * + * A delivery is acknowledged after its listeners handled it. A listener that throws is attempted again under the + * RetryPolicy (its own ListenerOptions first). After the last attempt: a listener deadLetterTopic is honoured by + * republishing to the exchange under that routing key, with the diagnostic headers, and acknowledging; otherwise + * the delivery is REJECTED without requeue and the broker's dead-letter exchange moves it to `.dlq` with its + * x-death history — RabbitMQ's own DLX, the eda adapter's choice too. With dead letters off it is acknowledged and + * logged. A republish that fails throws out of poll() unacknowledged: the channel dies with the worker and the + * broker redelivers. + * + * One queue serves one topic; two listeners of one group on one topic share the queue (and its DLX route), which is + * why one failing listener rejects the delivery for both — give listeners their own group when that matters. + * The message key travels in the `x-message-key` header (AMQP has no key). + */ +final class RabbitMqMessageBroker implements ListenerContainerBroker +{ + public const string KEY_HEADER = 'x-message-key'; + + /** @var array}> */ + private array $queues = []; + + private bool $exchangeDeclared = false; + + private bool $consuming = false; + + /** @var list */ + private array $pending = []; + + public function __construct( + private readonly AmqpMessagingChannel $channel, + private readonly RetryPolicy $policy, + private readonly ListenerInvoker $invoker, + private readonly string $exchange = 'firefly.messages', + private readonly string $deadLetterExchange = '', + private readonly int $prefetch = 10, + private readonly bool $deadLetterEnabled = true, + private readonly ?LoggerInterface $logger = null, + ) {} + + public static function queueNameOf(string $topic, ?string $group): string + { + return $group ?? 'firefly.'.$topic; + } + + public function publish(string $topic, string $value, ?string $key = null, array $headers = []): void + { + $this->declareExchange(); + $this->channel->publish($this->exchange, $topic, $value, $key === null ? $headers : array_merge($headers, [self::KEY_HEADER => $key])); + } + + public function subscribe(string $topic, callable $handler, ?string $group = null): void + { + $this->subscribeListener($topic, $handler, $group, new ListenerOptions); + } + + public function subscribeListener(string $topic, callable $handler, ?string $group, ListenerOptions $options): void + { + if ($this->consuming) { + throw new MessagingException('The RabbitMQ messaging broker is already consuming; subscribe every listener before the first poll().'); + } + + $queue = self::queueNameOf($topic, $group); + if (isset($this->queues[$queue]) && $this->queues[$queue]['topic'] !== $topic) { + throw new MessagingException("Queue [{$queue}] already consumes topic [{$this->queues[$queue]['topic']}]; a queue serves one topic, so give [{$topic}] its own group."); + } + + $this->queues[$queue]['topic'] = $topic; + $this->queues[$queue]['listeners'][] = ['handler' => $handler, 'options' => $options]; + } + + public function poll(int $timeoutMs): bool + { + if ($this->queues === []) { + return false; + } + + $this->startConsuming(); + + if ($this->pending === [] && ! $this->channel->wait($timeoutMs / 1000)) { + return false; + } + + $delivery = array_shift($this->pending); + if ($delivery === null) { + return false; + } + + $this->dispatch($delivery); + + return true; + } + + public function start(): void {} + + public function stop(): void + { + $this->channel->close(); + $this->consuming = false; + $this->exchangeDeclared = false; + $this->pending = []; + } + + private function deadLetterExchange(): string + { + return $this->deadLetterExchange !== '' ? $this->deadLetterExchange : $this->exchange.'.dlx'; + } + + private function declareExchange(): void + { + if (! $this->exchangeDeclared) { + $this->channel->declareExchange($this->exchange, 'direct'); + $this->exchangeDeclared = true; + } + } + + private function startConsuming(): void + { + if ($this->consuming) { + return; + } + + $this->declareExchange(); + $dlx = $this->deadLetterExchange(); + $this->channel->declareExchange($dlx, 'direct'); + $this->channel->qos($this->prefetch); + + foreach ($this->queues as $queue => $subscription) { + $queue = (string) $queue; + $this->channel->declareQueue($queue, ['x-dead-letter-exchange' => $dlx, 'x-dead-letter-routing-key' => $queue]); + $this->channel->bindQueue($queue, $this->exchange, $subscription['topic']); + $this->channel->declareQueue($queue.'.dlq', []); + $this->channel->bindQueue($queue.'.dlq', $dlx, $queue); + $this->channel->consume($queue, function (AmqpDelivery $delivery): void { + $this->pending[] = $delivery; + }); + } + + $this->consuming = true; + } + + private function dispatch(AmqpDelivery $delivery): void + { + $subscription = $this->queues[$delivery->queue] ?? null; + if ($subscription === null) { + $this->channel->nack($delivery->deliveryTag, false); + + return; + } + + $headers = $delivery->headers; + $key = $headers[self::KEY_HEADER] ?? null; + unset($headers[self::KEY_HEADER]); + $message = new Message($subscription['topic'], $delivery->body, $key, $headers); + + $exhausted = null; + foreach ($subscription['listeners'] as $listener) { + $outcome = $this->invoker->invoke($listener['handler'], $message, $this->policy->withOptions($listener['options'])); + if ($outcome->handled) { + continue; + } + + if ($listener['options']->deadLetterTopic !== null) { + $this->republish($listener['options']->deadLetterTopic, $message, $outcome); + + continue; + } + + $exhausted ??= $outcome; + } + + if ($exhausted === null) { + $this->channel->ack($delivery->deliveryTag); + + return; + } + + $cause = $exhausted->failure?->getMessage() ?? ''; + if ($this->deadLetterEnabled) { + $this->logger?->error("RabbitMQ message on [{$delivery->queue}] failed {$exhausted->attempts} attempt(s); rejected to [{$delivery->queue}.dlq]: {$cause}"); + $this->channel->nack($delivery->deliveryTag, false); + + return; + } + + $this->logger?->error("RabbitMQ message on [{$delivery->queue}] failed {$exhausted->attempts} attempt(s) and dead letters are off; acknowledged and skipped: {$cause}"); + $this->channel->ack($delivery->deliveryTag); + } + + private function republish(string $topic, Message $message, DeliveryOutcome $outcome): void + { + $headers = array_merge($message->headers, DeadLetterHeaders::for($message, $outcome)); + if ($message->key !== null) { + $headers[self::KEY_HEADER] = $message->key; + } + + $this->channel->publish($this->exchange, $topic, $message->value, $headers); + $this->logger?->error("RabbitMQ message on [{$message->topic}] failed {$outcome->attempts} attempt(s); republished to [{$topic}]."); + } +} diff --git a/packages/eda-rabbitmq/src/Messaging/RabbitMqMessagingAutoConfiguration.php b/packages/eda-rabbitmq/src/Messaging/RabbitMqMessagingAutoConfiguration.php new file mode 100644 index 00000000..e06bd282 --- /dev/null +++ b/packages/eda-rabbitmq/src/Messaging/RabbitMqMessagingAutoConfiguration.php @@ -0,0 +1,47 @@ +bound(LoggerInterface::class) ? $container->make(LoggerInterface::class) : null; + $logger = $logger instanceof LoggerInterface ? $logger : null; + + return new RabbitMqMessageBroker( + new PhpAmqpLibMessagingChannel(new RabbitMqConnectionFactory($config)), + RetryPolicy::fromConfig($config), + new ListenerInvoker(logger: $logger), + $config->string('firefly.messaging.rabbitmq.exchange', 'firefly.messages'), + $config->string('firefly.messaging.rabbitmq.dead-letter-exchange', ''), + $config->int('firefly.messaging.rabbitmq.prefetch', 10), + $config->bool('firefly.messaging.listener.dead-letter.enabled', true), + $logger, + ); + } +} diff --git a/packages/eda-rabbitmq/tests/Fixtures/FakeAmqpMessagingChannel.php b/packages/eda-rabbitmq/tests/Fixtures/FakeAmqpMessagingChannel.php new file mode 100644 index 00000000..4a9d7648 --- /dev/null +++ b/packages/eda-rabbitmq/tests/Fixtures/FakeAmqpMessagingChannel.php @@ -0,0 +1,106 @@ + exchange => type */ + public array $exchanges = []; + + /** @var array> queue => arguments */ + public array $queues = []; + + /** @var list queue, exchange, routing key */ + public array $bindings = []; + + public ?int $prefetch = null; + + /** @var list}> */ + public array $published = []; + + /** @var list */ + public array $acked = []; + + /** @var list */ + public array $nacked = []; + + public bool $closed = false; + + /** @var array */ + private array $consumers = []; + + /** @var list */ + private array $inbound = []; + + /** + * @param array $headers + */ + public function deliver(string $queue, string $routingKey, string $body, array $headers = [], int $tag = 1): void + { + $this->inbound[] = new AmqpDelivery($queue, $routingKey, $body, $headers, $tag); + } + + public function declareExchange(string $exchange, string $type): void + { + $this->exchanges[$exchange] = $type; + } + + public function declareQueue(string $queue, array $arguments): void + { + $this->queues[$queue] = $arguments; + } + + public function bindQueue(string $queue, string $exchange, string $routingKey): void + { + $this->bindings[] = [$queue, $exchange, $routingKey]; + } + + public function qos(int $prefetch): void + { + $this->prefetch = $prefetch; + } + + public function publish(string $exchange, string $routingKey, string $body, array $headers): void + { + $this->published[] = ['exchange' => $exchange, 'routingKey' => $routingKey, 'body' => $body, 'headers' => $headers]; + } + + public function consume(string $queue, Closure $callback): void + { + $this->consumers[$queue] = $callback; + } + + public function wait(float $timeoutSeconds): bool + { + $delivery = array_shift($this->inbound); + if ($delivery === null) { + return false; + } + + ($this->consumers[$delivery->queue])($delivery); + + return true; + } + + public function ack(int $deliveryTag): void + { + $this->acked[] = $deliveryTag; + } + + public function nack(int $deliveryTag, bool $requeue): void + { + $this->nacked[] = [$deliveryTag, $requeue]; + } + + public function close(): void + { + $this->closed = true; + } +} diff --git a/packages/eda-rabbitmq/tests/Messaging/RabbitMqMessageBrokerTest.php b/packages/eda-rabbitmq/tests/Messaging/RabbitMqMessageBrokerTest.php new file mode 100644 index 00000000..ab54e308 --- /dev/null +++ b/packages/eda-rabbitmq/tests/Messaging/RabbitMqMessageBrokerTest.php @@ -0,0 +1,137 @@ +publish('orders', 'bytes', 'k-1', ['x-trace' => 't']); + $broker->publish('orders', 'more'); + + expect($broker)->toBeInstanceOf(ListenerContainerBroker::class) + ->and($channel->exchanges)->toBe(['firefly.messages' => 'direct']) + ->and($channel->published[0])->toBe(['exchange' => 'firefly.messages', 'routingKey' => 'orders', 'body' => 'bytes', 'headers' => ['x-trace' => 't', 'x-message-key' => 'k-1']]) + ->and($channel->published[1]['headers'])->toBe([]); +}); + +it('declares a queue per subscription, its dead-letter route and its .dlq on the first poll', function () { + $channel = new FakeAmqpMessagingChannel; + $broker = w4RabbitBroker($channel); + $broker->subscribeListener('orders', static function (): void {}, null, new ListenerOptions); + $broker->subscribeListener('invoices', static function (): void {}, 'billing', new ListenerOptions); + + expect($broker->poll(10))->toBeFalse() + ->and($channel->exchanges)->toBe(['firefly.messages' => 'direct', 'firefly.messages.dlx' => 'direct']) + ->and($channel->queues)->toBe([ + 'firefly.orders' => ['x-dead-letter-exchange' => 'firefly.messages.dlx', 'x-dead-letter-routing-key' => 'firefly.orders'], + 'firefly.orders.dlq' => [], + 'billing' => ['x-dead-letter-exchange' => 'firefly.messages.dlx', 'x-dead-letter-routing-key' => 'billing'], + 'billing.dlq' => [], + ]) + ->and($channel->bindings)->toBe([ + ['firefly.orders', 'firefly.messages', 'orders'], + ['firefly.orders.dlq', 'firefly.messages.dlx', 'firefly.orders'], + ['billing', 'firefly.messages', 'invoices'], + ['billing.dlq', 'firefly.messages.dlx', 'billing'], + ]) + ->and($channel->prefetch)->toBe(10) + ->and(fn () => $broker->subscribeListener('late', static function (): void {}, null, new ListenerOptions))->toThrow(MessagingException::class, 'already consuming'); +}); + +it('refuses two topics on one queue', function () { + $broker = w4RabbitBroker(new FakeAmqpMessagingChannel); + $broker->subscribeListener('a', static function (): void {}, 'shared', new ListenerOptions); + + expect(fn () => $broker->subscribeListener('b', static function (): void {}, 'shared', new ListenerOptions))->toThrow(MessagingException::class, 'one topic'); +}); + +it('delivers to the listeners of the queue and acks', function () { + $channel = new FakeAmqpMessagingChannel; + $broker = w4RabbitBroker($channel); + $seen = []; + $broker->subscribeListener('orders', function (Message $message) use (&$seen): void { + $seen[] = [$message->topic, $message->value, $message->key, $message->headers]; + }, null, new ListenerOptions); + $channel->deliver('firefly.orders', 'orders', 'v', ['x-message-key' => 'k', 'h' => '1'], 5); + + expect($broker->poll(10))->toBeTrue() + ->and($seen)->toBe([['orders', 'v', 'k', ['h' => '1']]]) + ->and($channel->acked)->toBe([5]) + ->and($channel->nacked)->toBe([]); +}); + +it('retries, then rejects without requeue so the broker routes the message to the .dlq', function () { + $channel = new FakeAmqpMessagingChannel; + $broker = w4RabbitBroker($channel, maxAttempts: 3); + $calls = 0; + $broker->subscribeListener('orders', function () use (&$calls): void { + $calls++; + + throw new RuntimeException('down'); + }, null, new ListenerOptions); + $channel->deliver('firefly.orders', 'orders', 'v', [], 9); + + $broker->poll(10); + + expect($calls)->toBe(3)->and($channel->nacked)->toBe([[9, false]])->and($channel->acked)->toBe([]); +}); + +it("republishes to the listener's own dead-letter topic with the diagnostic headers, then acks", function () { + $channel = new FakeAmqpMessagingChannel; + $broker = w4RabbitBroker($channel); + $broker->subscribeListener('orders', static function (): void { + throw new DomainException('bad order'); + }, null, new ListenerOptions(1, null, 'orders.parked')); + $channel->deliver('firefly.orders', 'orders', 'v', ['x-message-key' => 'k'], 3); + + $broker->poll(10); + + expect($channel->published)->toHaveCount(1) + ->and($channel->published[0]['routingKey'])->toBe('orders.parked') + ->and($channel->published[0]['headers'])->toMatchArray([ + 'x-message-key' => 'k', + 'x-original-topic' => 'orders', + 'x-exception' => 'DomainException', + 'x-exception-message' => 'bad order', + 'x-delivery-attempts' => '1', + ]) + ->and($channel->acked)->toBe([3]); +}); + +it('acks and logs after the last attempt when dead letters are off', function () { + $channel = new FakeAmqpMessagingChannel; + $broker = w4RabbitBroker($channel, deadLetters: false); + $broker->subscribeListener('orders', static function (): void { + throw new RuntimeException('no'); + }, null, new ListenerOptions); + $channel->deliver('firefly.orders', 'orders', 'v', [], 4); + + $broker->poll(10); + + expect($channel->acked)->toBe([4])->and($channel->nacked)->toBe([]); +}); + +it('closes the channel on stop', function () { + $channel = new FakeAmqpMessagingChannel; + w4RabbitBroker($channel)->stop(); + + expect($channel->closed)->toBeTrue() + ->and(RabbitMqMessageBroker::queueNameOf('orders', null))->toBe('firefly.orders') + ->and(RabbitMqMessageBroker::queueNameOf('orders', 'billing'))->toBe('billing'); +}); diff --git a/packages/eda-rabbitmq/tests/Messaging/RabbitMqMessagingAutoConfigurationTest.php b/packages/eda-rabbitmq/tests/Messaging/RabbitMqMessagingAutoConfigurationTest.php new file mode 100644 index 00000000..2e211df5 --- /dev/null +++ b/packages/eda-rabbitmq/tests/Messaging/RabbitMqMessagingAutoConfigurationTest.php @@ -0,0 +1,37 @@ +getAttributes(ConditionalOnProperty::class)[0]->newInstance(); + + expect($condition->name)->toBe('firefly.messaging.provider')->and($condition->havingValue)->toBe('rabbitmq'); +}); + +it('builds the RabbitMQ broker without opening a connection', function () { + $config = new Config(new Repository(['firefly' => ['messaging' => ['provider' => 'rabbitmq'], 'eda' => ['rabbitmq' => ['host' => '127.0.0.1', 'port' => 1]]]])); + + expect((new RabbitMqMessagingAutoConfiguration)->messageBroker($config, new Container))->toBeInstanceOf(RabbitMqMessageBroker::class); +}); + +it('wins over the core in-memory broker through the real boot', function () { + $app = fireflyApplication( + config: ['firefly' => ['messaging' => ['provider' => 'rabbitmq'], 'eda' => ['rabbitmq' => ['host' => '127.0.0.1', 'port' => 1]]]], + providers: [MessagingServiceProvider::class, MessagingWiringProvider::class, EdaRabbitmqServiceProvider::class], + bindings: [MessageListenerManifest::class => new MessageListenerManifest([])], + ); + + expect($app->make(MessageBrokerPort::class))->toBeInstanceOf(RabbitMqMessageBroker::class); +}); From c390521e8703c18694dd71d960c273e6ab42665b Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 20:03:57 -0700 Subject: [PATCH 26/47] docs(config): document firefly.messaging.rabbitmq exchange, dead-letter-exchange and prefetch in the skeleton --- skeleton/config/firefly.php | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/skeleton/config/firefly.php b/skeleton/config/firefly.php index 3593b5fc..ed4056fb 100644 --- a/skeleton/config/firefly.php +++ b/skeleton/config/firefly.php @@ -2559,6 +2559,8 @@ | 'queue' — Laravel queue; both keys UNSET mean the application's own defaults. | 'kafka' — Kafka through firefly/eda-kafka (needs ext-rdkafka); a listener container driven by | `php artisan firefly:messaging:consume`, with the retry and dead-letter keys below. + | 'rabbitmq' — RabbitMQ through firefly/eda-rabbitmq, on the firefly.eda.rabbitmq.* connection; a + | listener container consumed the same way, with the same retry and dead-letter keys. | */ @@ -2598,7 +2600,8 @@ /* | listener.dead-letter.enabled — what a listener container broker does with a delivery whose | attempts ran out. true: it is published to the listener's own deadLetterTopic, or to the - | broker's default route (Kafka: ``), with x-original-topic, x-exception, + | broker's default route (Kafka: ``; RabbitMQ: the queue is rejected without + | requeue and its dead-letter exchange moves it into `.dlq`), with x-original-topic, x-exception, | x-exception-message and x-delivery-attempts headers, and only then acknowledged. false: it is | logged and skipped — an explicit choice to lose it. A listener that names its own | deadLetterTopic is always dead-lettered there. @@ -2628,6 +2631,26 @@ 'group-id' => env('FIREFLY_MESSAGING_KAFKA_GROUP_ID', 'firefly'), 'dlt-suffix' => env('FIREFLY_MESSAGING_KAFKA_DLT_SUFFIX', '.DLT'), ], + + /* + | rabbitmq — read only when provider is 'rabbitmq'. The connection is the event bus's + | (firefly.eda.rabbitmq.host|port|user|password|vhost); nothing connects at boot. A topic is a + | routing key on one durable DIRECT exchange; each subscription consumes one durable queue (the + | listener's group, else `firefly.`) declared with this dead-letter exchange, so a rejected + | delivery lands in `.dlq`. Deliveries are acknowledged by hand after their listeners + | handled them, with a bounded prefetch. + | + | exchange — the exchange messages are published to and queues are bound on. + | dead-letter-exchange — '' means `.dlx`. + | prefetch — unacknowledged deliveries the broker hands this worker at once. + | + | Default: exchange 'firefly.messages', dead-letter-exchange '' (= 'firefly.messages.dlx'), prefetch 10. + */ + 'rabbitmq' => [ + 'exchange' => env('FIREFLY_MESSAGING_RABBITMQ_EXCHANGE', 'firefly.messages'), + 'dead-letter-exchange' => env('FIREFLY_MESSAGING_RABBITMQ_DEAD_LETTER_EXCHANGE', ''), + 'prefetch' => env('FIREFLY_MESSAGING_RABBITMQ_PREFETCH', 10), + ], ], /* From 4fee709aadee437b28f9d124beb4a201d1326add Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 20:03:57 -0700 Subject: [PATCH 27/47] test(admin): opt the paged conditions harness into the default-off health indicators so its backed-off panel holds only the seeded rows --- .../AdminTableConditionsPagedCapstoneTestCase.php | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/packages/admin/tests/Support/AdminTableConditionsPagedCapstoneTestCase.php b/packages/admin/tests/Support/AdminTableConditionsPagedCapstoneTestCase.php index c69b14e5..f7438975 100644 --- a/packages/admin/tests/Support/AdminTableConditionsPagedCapstoneTestCase.php +++ b/packages/admin/tests/Support/AdminTableConditionsPagedCapstoneTestCase.php @@ -17,7 +17,8 @@ * somewhere else — the exact bug the carrying exists to prevent, with nothing failing. * * So this case changes TWO things and nothing else, the same two AdminTablePagedCapstoneTestCase changes - * for Routes. It OFFERS a page size of 2 — `firefly.admin.table.page-sizes`, seeded through + * for Routes (plus opting in the actuator's default-off health indicators, so the framework itself backs + * nothing off and the Backed-off side holds this fixture's rows alone). It OFFERS a page size of 2 — `firefly.admin.table.page-sizes`, seeded through * configOverrides() rather than set inside a test, because AdminSettings (and the TableSettings on it) is * built once by AdminRouteRegistrar during the boot passes and bound as an instance, so a `config()->set()` * from a test body arrives after the object that would have read it. And it grows the backed-off side to @@ -38,6 +39,14 @@ protected function configOverrides(): array return [ ...parent::configOverrides(), 'firefly.admin.table.page-sizes' => '2,25,50', + // The actuator's opt-in cache, queue and Redis health indicators are #[ConditionalOnProperty] + // components that default OFF, so a bare testbench backs all three off — ambient rows on the very + // panel this case pins at exactly its five seeded rows, and a second `Redis` match for `?neg_q=`. + // Opting them in moves them to the Applied side, whose count this case never pins; nothing here + // asks them for a health check, so neither a cache, a queue nor a Redis server is needed. + 'firefly.management.endpoint.health.cache.enabled' => 'true', + 'firefly.management.endpoint.health.queue.enabled' => 'true', + 'firefly.management.endpoint.health.redis.enabled' => 'true', ]; } From 1d08116c946a1baa081af2feab962ed6447ba997 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 20:08:48 -0700 Subject: [PATCH 28/47] fix(eda-rabbitmq): declare and bind a parking queue for every listener dead-letter topic so the direct exchange never drops a dead letter as unroutable --- .../src/Messaging/RabbitMqMessageBroker.php | 43 +++++++-- ...bbitMqMessageBrokerDeadLetterTopicTest.php | 90 +++++++++++++++++++ .../Messaging/RabbitMqMessageBrokerTest.php | 28 ++++++ skeleton/config/firefly.php | 4 +- 4 files changed, 158 insertions(+), 7 deletions(-) create mode 100644 packages/eda-rabbitmq/tests/Integration/RabbitMqMessageBrokerDeadLetterTopicTest.php diff --git a/packages/eda-rabbitmq/src/Messaging/RabbitMqMessageBroker.php b/packages/eda-rabbitmq/src/Messaging/RabbitMqMessageBroker.php index b1a47c76..d77dff34 100644 --- a/packages/eda-rabbitmq/src/Messaging/RabbitMqMessageBroker.php +++ b/packages/eda-rabbitmq/src/Messaging/RabbitMqMessageBroker.php @@ -28,8 +28,15 @@ * republishing to the exchange under that routing key, with the diagnostic headers, and acknowledging; otherwise * the delivery is REJECTED without requeue and the broker's dead-letter exchange moves it to `.dlq` with its * x-death history — RabbitMQ's own DLX, the eda adapter's choice too. With dead letters off it is acknowledged and - * logged. A republish that fails throws out of poll() unacknowledged: the channel dies with the worker and the - * broker redelivers. + * logged. + * + * A direct exchange silently drops a message no queue is bound for, so a deadLetterTopic always gets a home before + * the first delivery: unless a subscription of this broker already consumes that topic, a durable parking queue + * `firefly.` is declared and bound under it — the very queue (same arguments, same `.dlq`) a + * group-less listener of that topic would declare later, so subscribing to the parked topic afterwards drains what + * was parked instead of clashing with it. A republish that fails on the socket throws out of poll() + * unacknowledged: the channel dies with the worker and the broker redelivers. Deleting the parking queue by hand + * while its listener fails is the one way left to lose a parked message, as with any unbound routing key. * * One queue serves one topic; two listeners of one group on one topic share the queue (and its DLX route), which is * why one failing listener rejects the delivery for both — give listeners their own group when that matters. @@ -147,20 +154,44 @@ private function startConsuming(): void $this->channel->declareExchange($dlx, 'direct'); $this->channel->qos($this->prefetch); + $consumedTopics = []; foreach ($this->queues as $queue => $subscription) { $queue = (string) $queue; - $this->channel->declareQueue($queue, ['x-dead-letter-exchange' => $dlx, 'x-dead-letter-routing-key' => $queue]); - $this->channel->bindQueue($queue, $this->exchange, $subscription['topic']); - $this->channel->declareQueue($queue.'.dlq', []); - $this->channel->bindQueue($queue.'.dlq', $dlx, $queue); + $this->declareBoundQueue($queue, $subscription['topic'], $dlx); $this->channel->consume($queue, function (AmqpDelivery $delivery): void { $this->pending[] = $delivery; }); + $consumedTopics[$subscription['topic']] = true; + } + + foreach ($this->queues as $subscription) { + foreach ($subscription['listeners'] as $listener) { + $parked = $listener['options']->deadLetterTopic; + if ($parked === null || isset($consumedTopics[$parked])) { + continue; + } + + $this->declareBoundQueue(self::queueNameOf($parked, null), $parked, $dlx); + $consumedTopics[$parked] = true; + } } $this->consuming = true; } + /** + * A durable queue bound to its topic on the exchange, with its dead-letter route into `.dlq` on the DLX — + * one shape for a subscription's queue and a dead-letter topic's parking queue, so the two never disagree on + * arguments (RabbitMQ refuses a redeclare with different ones). + */ + private function declareBoundQueue(string $queue, string $topic, string $dlx): void + { + $this->channel->declareQueue($queue, ['x-dead-letter-exchange' => $dlx, 'x-dead-letter-routing-key' => $queue]); + $this->channel->bindQueue($queue, $this->exchange, $topic); + $this->channel->declareQueue($queue.'.dlq', []); + $this->channel->bindQueue($queue.'.dlq', $dlx, $queue); + } + private function dispatch(AmqpDelivery $delivery): void { $subscription = $this->queues[$delivery->queue] ?? null; diff --git a/packages/eda-rabbitmq/tests/Integration/RabbitMqMessageBrokerDeadLetterTopicTest.php b/packages/eda-rabbitmq/tests/Integration/RabbitMqMessageBrokerDeadLetterTopicTest.php new file mode 100644 index 00000000..37488e74 --- /dev/null +++ b/packages/eda-rabbitmq/tests/Integration/RabbitMqMessageBrokerDeadLetterTopicTest.php @@ -0,0 +1,90 @@ + ['eda' => ['rabbitmq' => [ + 'host' => $parts['host'] ?? '127.0.0.1', + 'port' => (int) ($parts['port'] ?? 5672), + 'user' => $parts['user'] ?? 'guest', + 'password' => $parts['password'] ?? 'guest', + 'vhost' => '/', + ]]], + ]))); +} + +it("parks an exhausted delivery on the listener's dead-letter topic even when nobody consumes that topic", function () { + $factory = w4RabbitDeadLetterFactory(); + $suffix = bin2hex(random_bytes(4)); + $exchange = 'firefly.messages.test.'.$suffix; + $topic = 'orders.'.$suffix; + $parked = 'orders.parked.'.$suffix; + $parkingQueue = RabbitMqMessageBroker::queueNameOf($parked, null); + + $broker = new RabbitMqMessageBroker( + new PhpAmqpLibMessagingChannel($factory), + new RetryPolicy(1, new LinearBackOff(0.0)), + new ListenerInvoker(static function (float $s): void {}), + exchange: $exchange, + ); + $broker->subscribeListener($topic, static function (): void { + throw new DomainException('bad order'); + }, null, new ListenerOptions(1, null, $parked)); + + // The first poll declares the topology — the subscription's queue AND the dead-letter topic's parking queue. + $broker->poll(100); + $broker->publish($topic, 'payload', 'k-1'); + + $deadline = time() + 15; + $handled = false; + while (! $handled && time() < $deadline) { + $handled = $broker->poll(1000); + } + $broker->stop(); + + $connection = $factory->connect(); + $channel = $connection->channel(); + $message = null; + $deadline = time() + 10; + while ($message === null && time() < $deadline) { + $message = $channel->basic_get($parkingQueue, true); + if ($message === null) { + usleep(200_000); + } + } + + foreach ([$parkingQueue, $parkingQueue.'.dlq', 'firefly.'.$topic, 'firefly.'.$topic.'.dlq'] as $queue) { + $channel->queue_delete($queue); + } + $channel->exchange_delete($exchange); + $channel->exchange_delete($exchange.'.dlx'); + $channel->close(); + $connection->close(); + + expect($handled)->toBeTrue() + ->and($message)->toBeInstanceOf(AMQPMessage::class) + ->and($message?->getBody())->toBe('payload'); +})->skip(getenv('FIREFLY_RABBITMQ_DSN') === false, 'Set FIREFLY_RABBITMQ_DSN to run the @group integration RabbitMQ dead-letter topic test.')->group('integration'); diff --git a/packages/eda-rabbitmq/tests/Messaging/RabbitMqMessageBrokerTest.php b/packages/eda-rabbitmq/tests/Messaging/RabbitMqMessageBrokerTest.php index ab54e308..62b506b4 100644 --- a/packages/eda-rabbitmq/tests/Messaging/RabbitMqMessageBrokerTest.php +++ b/packages/eda-rabbitmq/tests/Messaging/RabbitMqMessageBrokerTest.php @@ -114,6 +114,34 @@ function w4RabbitBroker(FakeAmqpMessagingChannel $channel, int $maxAttempts = 1, ->and($channel->acked)->toBe([3]); }); +it("gives a listener's dead-letter topic a bound parking queue, so the direct exchange cannot drop the dead letter", function () { + $channel = new FakeAmqpMessagingChannel; + $broker = w4RabbitBroker($channel); + $broker->subscribeListener('orders', static function (): void {}, null, new ListenerOptions(1, null, 'orders.parked')); + $broker->subscribeListener('invoices', static function (): void {}, 'billing', new ListenerOptions(1, null, 'orders.parked')); + + $broker->poll(10); + + expect($channel->queues)->toHaveKey('firefly.orders.parked') + ->and($channel->queues['firefly.orders.parked'])->toBe(['x-dead-letter-exchange' => 'firefly.messages.dlx', 'x-dead-letter-routing-key' => 'firefly.orders.parked']) + ->and($channel->queues['firefly.orders.parked.dlq'])->toBe([]) + ->and($channel->bindings)->toContain(['firefly.orders.parked', 'firefly.messages', 'orders.parked']) + ->and($channel->bindings)->toContain(['firefly.orders.parked.dlq', 'firefly.messages.dlx', 'firefly.orders.parked']) + ->and(array_filter($channel->bindings, static fn (array $binding): bool => $binding[0] === 'firefly.orders.parked'))->toHaveCount(1); +}); + +it('declares no parking queue for a dead-letter topic a subscription already consumes', function () { + $channel = new FakeAmqpMessagingChannel; + $broker = w4RabbitBroker($channel); + $broker->subscribeListener('orders', static function (): void {}, null, new ListenerOptions(1, null, 'orders.parked')); + $broker->subscribeListener('orders.parked', static function (): void {}, 'parking', new ListenerOptions); + + $broker->poll(10); + + expect($channel->queues)->not->toHaveKey('firefly.orders.parked') + ->and($channel->bindings)->toContain(['parking', 'firefly.messages', 'orders.parked']); +}); + it('acks and logs after the last attempt when dead letters are off', function () { $channel = new FakeAmqpMessagingChannel; $broker = w4RabbitBroker($channel, deadLetters: false); diff --git a/skeleton/config/firefly.php b/skeleton/config/firefly.php index ed4056fb..74e52a1f 100644 --- a/skeleton/config/firefly.php +++ b/skeleton/config/firefly.php @@ -2604,7 +2604,9 @@ | requeue and its dead-letter exchange moves it into `.dlq`), with x-original-topic, x-exception, | x-exception-message and x-delivery-attempts headers, and only then acknowledged. false: it is | logged and skipped — an explicit choice to lose it. A listener that names its own - | deadLetterTopic is always dead-lettered there. + | deadLetterTopic is always dead-lettered there (RabbitMQ declares and binds a durable + | `firefly.` parking queue for it unless a subscription already consumes that topic, + | so the direct exchange never drops the dead letter as unroutable). | | Default: true. */ From bb7deb513aeab0c90d42caea8ac0243b5dbf48e9 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 20:13:54 -0700 Subject: [PATCH 29/47] test(eda-rabbitmq): prove the messaging listener container against a real RabbitMQ, dead-letter exchange included --- .../RabbitMqMessagingRoundTripTest.php | 165 ++++++++++++++++++ 1 file changed, 165 insertions(+) create mode 100644 packages/eda-rabbitmq/tests/Integration/RabbitMqMessagingRoundTripTest.php diff --git a/packages/eda-rabbitmq/tests/Integration/RabbitMqMessagingRoundTripTest.php b/packages/eda-rabbitmq/tests/Integration/RabbitMqMessagingRoundTripTest.php new file mode 100644 index 00000000..71675555 --- /dev/null +++ b/packages/eda-rabbitmq/tests/Integration/RabbitMqMessagingRoundTripTest.php @@ -0,0 +1,165 @@ + ['eda' => ['rabbitmq' => [ + 'host' => $parts['host'] ?? '127.0.0.1', + 'port' => (int) ($parts['port'] ?? 5672), + 'user' => $parts['user'] ?? 'guest', + 'password' => $parts['password'] ?? 'guest', + 'vhost' => '/', + ]]]]))); +} + +function w4RabbitMessaging(string $exchange, int $maxAttempts = 1): RabbitMqMessageBroker +{ + return new RabbitMqMessageBroker( + new PhpAmqpLibMessagingChannel(w4RabbitConnection()), + new RetryPolicy($maxAttempts, new LinearBackOff(0.0)), + new ListenerInvoker(static function (float $seconds): void {}), + $exchange, + ); +} + +/** + * Polls until $done answers true or 15 seconds pass. $done must read the test's state BY REFERENCE (a `use (&$x)` + * closure): an arrow function captures by value and would never see the listener's writes. + */ +function w4RabbitPollUntil(RabbitMqMessageBroker $broker, Closure $done): void +{ + $deadline = microtime(true) + 15; + while (! $done() && microtime(true) < $deadline) { + $broker->poll(500); + } +} + +/** + * Removes what one test declared on the shared broker — every queue is durable, so without this each run would + * leave its randomly named queues and exchanges behind. + * + * @param list $topics + */ +function w4RabbitTearDown(string $exchange, array $topics): void +{ + $connection = w4RabbitConnection()->connect(); + $channel = $connection->channel(); + foreach ($topics as $topic) { + $queue = RabbitMqMessageBroker::queueNameOf($topic, null); + $channel->queue_delete($queue); + $channel->queue_delete($queue.'.dlq'); + } + $channel->exchange_delete($exchange); + $channel->exchange_delete($exchange.'.dlx'); + $channel->close(); + $connection->close(); +} + +it('round-trips publish -> poll -> listener -> ack on a real RabbitMQ, with key and headers', function () { + $suffix = bin2hex(random_bytes(4)); + $topic = 'orders.'.$suffix; + $exchange = 'firefly.messages.it.'.$suffix; + $broker = w4RabbitMessaging($exchange); + $seen = []; + $broker->subscribeListener($topic, function (Message $message) use (&$seen): void { + $seen[] = [$message->value, $message->key, $message->headers['x-trace'] ?? null]; + }, null, new ListenerOptions); + + $broker->poll(100); // declares the exchange, the queue and its binding before anything is published + $broker->publish($topic, 'hello', 'k-1', ['x-trace' => 't-1']); + w4RabbitPollUntil($broker, static function () use (&$seen): bool { + return $seen !== []; + }); + $broker->stop(); + w4RabbitTearDown($exchange, [$topic]); + + expect($seen)->toBe([['hello', 'k-1', 't-1']]); +})->skip(getenv('FIREFLY_RABBITMQ_DSN') === false, 'Set FIREFLY_RABBITMQ_DSN to run the RabbitMQ messaging round trip.')->group('integration'); + +it('retries, then rejects the message into .dlq through the dead-letter exchange', function () { + $suffix = bin2hex(random_bytes(4)); + $topic = 'orders.'.$suffix; + $exchange = 'firefly.messages.it.'.$suffix; + $broker = w4RabbitMessaging($exchange, maxAttempts: 2); + $calls = 0; + $broker->subscribeListener($topic, function () use (&$calls): void { + $calls++; + + throw new RuntimeException('listener down'); + }, null, new ListenerOptions); + + $broker->poll(100); + $broker->publish($topic, 'payload'); + w4RabbitPollUntil($broker, static function () use (&$calls): bool { + return $calls >= 2; + }); + $broker->stop(); + + $probe = new PhpAmqpLibMessagingChannel(w4RabbitConnection()); + $dead = []; + $probe->consume(RabbitMqMessageBroker::queueNameOf($topic, null).'.dlq', function (AmqpDelivery $delivery) use (&$dead, $probe): void { + $dead[] = $delivery->body; + $probe->ack($delivery->deliveryTag); + }); + $deadline = microtime(true) + 10; + while ($dead === [] && microtime(true) < $deadline) { + $probe->wait(1.0); + } + $probe->close(); + w4RabbitTearDown($exchange, [$topic]); + + expect($calls)->toBe(2)->and($dead)->toBe(['payload']); +})->skip(getenv('FIREFLY_RABBITMQ_DSN') === false, 'Set FIREFLY_RABBITMQ_DSN to run the RabbitMQ messaging round trip.')->group('integration'); + +it("republishes to the listener's own dead-letter topic with the diagnostic headers", function () { + $suffix = bin2hex(random_bytes(4)); + $topic = 'orders.'.$suffix; + $parked = 'parked.'.$suffix; + $exchange = 'firefly.messages.it.'.$suffix; + $broker = w4RabbitMessaging($exchange); + $received = []; + $broker->subscribeListener($topic, static function (): void { + throw new DomainException('bad order'); + }, null, new ListenerOptions(1, null, $parked)); + $broker->subscribeListener($parked, function (Message $message) use (&$received): void { + $received[] = $message->headers; + }, null, new ListenerOptions); + + $broker->poll(100); + $broker->publish($topic, 'payload'); + w4RabbitPollUntil($broker, static function () use (&$received): bool { + return $received !== []; + }); + $broker->stop(); + w4RabbitTearDown($exchange, [$topic, $parked]); + + expect($received[0] ?? [])->toMatchArray([ + 'x-original-topic' => $topic, + 'x-exception' => 'DomainException', + 'x-exception-message' => 'bad order', + 'x-delivery-attempts' => '1', + ]); +})->skip(getenv('FIREFLY_RABBITMQ_DSN') === false, 'Set FIREFLY_RABBITMQ_DSN to run the RabbitMQ messaging round trip.')->group('integration'); From 460644314bcf2c2a6fd823984dd6de1f410191a5 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 20:18:57 -0700 Subject: [PATCH 30/47] feat(webhooks): register the firefly/webhooks package with validated provider settings behind firefly.webhooks.enabled --- composer.json | 9 +- deptrac.yaml | 20 ++ docs/modules.md | 4 +- packages/firefly/composer.json | 3 +- packages/installer/src/CapabilityCatalog.php | 1 + packages/webhooks/.gitattributes | 2 + packages/webhooks/LICENSE | 204 ++++++++++++++++++ packages/webhooks/README.md | 10 + .../cache/firefly-webhooks-components.php | 36 ++++ .../cache/firefly-webhooks-context.php | 40 ++++ packages/webhooks/composer.json | 67 ++++++ .../webhooks/src/Provider/SignatureKind.php | 39 ++++ .../src/Provider/WebhookProviderSettings.php | 104 +++++++++ .../src/WebhooksAutoConfiguration.php | 30 +++ .../webhooks/src/WebhooksServiceProvider.php | 25 +++ packages/webhooks/src/WebhooksSettings.php | 105 +++++++++ .../webhooks/src/WebhooksWiringProvider.php | 20 ++ .../tests/CompiledManifestFreshnessTest.php | 30 +++ packages/webhooks/tests/PackageBootsTest.php | 23 ++ .../webhooks/tests/WebhooksSettingsTest.php | 69 ++++++ 20 files changed, 836 insertions(+), 5 deletions(-) create mode 100644 packages/webhooks/.gitattributes create mode 100644 packages/webhooks/LICENSE create mode 100644 packages/webhooks/README.md create mode 100644 packages/webhooks/cache/firefly-webhooks-components.php create mode 100644 packages/webhooks/cache/firefly-webhooks-context.php create mode 100644 packages/webhooks/composer.json create mode 100644 packages/webhooks/src/Provider/SignatureKind.php create mode 100644 packages/webhooks/src/Provider/WebhookProviderSettings.php create mode 100644 packages/webhooks/src/WebhooksAutoConfiguration.php create mode 100644 packages/webhooks/src/WebhooksServiceProvider.php create mode 100644 packages/webhooks/src/WebhooksSettings.php create mode 100644 packages/webhooks/src/WebhooksWiringProvider.php create mode 100644 packages/webhooks/tests/CompiledManifestFreshnessTest.php create mode 100644 packages/webhooks/tests/PackageBootsTest.php create mode 100644 packages/webhooks/tests/WebhooksSettingsTest.php diff --git a/composer.json b/composer.json index 1d1f918d..0355f252 100644 --- a/composer.json +++ b/composer.json @@ -85,6 +85,7 @@ "Firefly\\Tests\\": "tests/", "Firefly\\Validation\\Tests\\": "packages/validation/tests/", "Firefly\\Web\\Tests\\": "packages/web/tests/", + "Firefly\\Webhooks\\Tests\\": "packages/webhooks/tests/", "Lumen\\Tests\\": "samples/lumen/tests/", "Firefly\\Kernel\\Tests\\": "packages/kernel/tests/", "Lumen\\": "samples/lumen/src/" @@ -149,7 +150,8 @@ "firefly/security-oauth2-server": "self.version", "firefly/testing": "self.version", "firefly/validation": "self.version", - "firefly/web": "self.version" + "firefly/web": "self.version", + "firefly/webhooks": "self.version" }, "autoload": { "psr-4": { @@ -181,7 +183,8 @@ "Firefly\\Security\\OAuth2\\Server\\": "packages/security-oauth2-server/src/", "Firefly\\Testing\\": "packages/testing/src/", "Firefly\\Validation\\": "packages/validation/src/", - "Firefly\\Web\\": "packages/web/src/" + "Firefly\\Web\\": "packages/web/src/", + "Firefly\\Webhooks\\": "packages/webhooks/src/" }, "files": [ "packages/security-oauth2-client/src/Registration/casters.php", @@ -260,6 +263,8 @@ "Firefly\\Security\\OAuth2\\Client\\SecurityOAuth2ClientWiringProvider", "Firefly\\FeatureFlags\\FeatureFlagsServiceProvider", "Firefly\\FeatureFlags\\FeatureFlagsWiringProvider", + "Firefly\\Webhooks\\WebhooksServiceProvider", + "Firefly\\Webhooks\\WebhooksWiringProvider", "Firefly\\Cli\\Boot\\FireflyCacheServiceProvider" ] } diff --git a/deptrac.yaml b/deptrac.yaml index 674bc99b..874d82f0 100644 --- a/deptrac.yaml +++ b/deptrac.yaml @@ -102,6 +102,10 @@ deptrac: collectors: - type: directory value: packages/feature-flags/src/.* + - name: Webhooks + collectors: + - type: directory + value: packages/webhooks/src/.* - name: OpenApi collectors: - type: directory @@ -502,6 +506,22 @@ deptrac: - Actuator - Resilience + # Webhooks is the inbound-webhook capability: an endpoint per provider mounted on the router by a BootPass + # (the FlagdSyncRouteRegistrar shape), signature validators, an idempotency store on the cache or any database + # connection (illuminate, unlayered), and a publish to Eda's EventPublisher (=> Eda). Every refusal is rendered + # by Web's ProblemDetailsRenderer (=> Web); replay windows parse through Resilience's Duration (=> Resilience, + # the Scheduling precedent). It reads Config, carries Container/Context attributes, extends AutoConfigure's base + # and reuses Kernel exceptions. NOTHING depends on it. + Webhooks: + - Kernel + - Container + - Config + - Context + - AutoConfigure + - Web + - Eda + - Resilience + # OpenApi is a top-of-stack capability (like Web/Cqrs/Security/Actuator): it generates an OpenAPI 3.1 # document from manifests that already exist, so it is nearly all READS. It reads Web's RouteManifest/ # RouteDescriptor (paths, verbs, statuses, binding plans) and Validation's ConstraintManifest + Rule diff --git a/docs/modules.md b/docs/modules.md index 69241792..7fdb64ee 100644 --- a/docs/modules.md +++ b/docs/modules.md @@ -1,12 +1,12 @@ # Modules -LaraFly publishes one `fireflyframework/larafly` library containing the **30 component directories** under +LaraFly publishes one `fireflyframework/larafly` library containing the **31 component directories** under `packages/*`, each with its own tests or metadata. The **33 guides** below explain their public surfaces. Several components carry more than one guide: `firefly/data` answers for *Data & Repositories*, *Relational Data* and *Transactions*. Laravel discovers the root library's providers, and configuration selects optional features. A bean you declare wins over the framework default. Two idioms deliver that outcome, and -which one you are looking at matters the moment you go reading the source. **18 of the 30 packages** carry a +which one you are looking at matters the moment you go reading the source. **19 of the 31 packages** carry a `#[Configuration]` class whose `#[Bean]` methods sit behind `#[ConditionalOnMissingBean]`, so a bean of yours makes the framework's back off silently — `packages/data/src/DataAutoConfiguration.php` is the model. The **other 12** have no such class to go looking for; `firefly/web`, the one nearly every reader installs, guards diff --git a/packages/firefly/composer.json b/packages/firefly/composer.json index faee7fec..3c13d468 100644 --- a/packages/firefly/composer.json +++ b/packages/firefly/composer.json @@ -45,7 +45,8 @@ "firefly/security-oauth2-client": "*@dev", "firefly/security-oauth2-server": "*@dev", "firefly/validation": "*@dev", - "firefly/web": "*@dev" + "firefly/web": "*@dev", + "firefly/webhooks": "*@dev" }, "extra": { "branch-alias": { diff --git a/packages/installer/src/CapabilityCatalog.php b/packages/installer/src/CapabilityCatalog.php index 395f3719..05ff46b8 100644 --- a/packages/installer/src/CapabilityCatalog.php +++ b/packages/installer/src/CapabilityCatalog.php @@ -44,6 +44,7 @@ public static function all(): array new Capability('admin', 'firefly/admin', 'Server-rendered dashboard over the actuator (a Spring Boot Admin analog)'), new Capability('openapi', 'firefly/openapi', 'OpenAPI 3.1 document generated from the route and constraint manifests, plus a viewer'), new Capability('feature-flags', 'firefly/feature-flags', 'Feature flags on OpenFeature: flagd documents, targeting and percentage rollouts, a writable store, admin and actuator controls'), + new Capability('webhooks', 'firefly/webhooks', 'Inbound webhooks: an endpoint per provider, HMAC/Stripe/GitHub/Twilio signatures, a replay window and idempotency, published to the event bus', requires: ['eda']), new Capability('eda-kafka', 'firefly/eda-kafka', 'Kafka publisher/consumer over ext-rdkafka', requires: ['eda'], adapter: true), new Capability('eda-rabbitmq', 'firefly/eda-rabbitmq', 'RabbitMQ publisher/consumer over php-amqplib', requires: ['eda'], adapter: true), diff --git a/packages/webhooks/.gitattributes b/packages/webhooks/.gitattributes new file mode 100644 index 00000000..538b69a4 --- /dev/null +++ b/packages/webhooks/.gitattributes @@ -0,0 +1,2 @@ +/tests export-ignore +/.gitattributes export-ignore diff --git a/packages/webhooks/LICENSE b/packages/webhooks/LICENSE new file mode 100644 index 00000000..2240005d --- /dev/null +++ b/packages/webhooks/LICENSE @@ -0,0 +1,204 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +Copyright 2026 Firefly Software Solutions Inc. diff --git a/packages/webhooks/README.md b/packages/webhooks/README.md new file mode 100644 index 00000000..c58f9b87 --- /dev/null +++ b/packages/webhooks/README.md @@ -0,0 +1,10 @@ +# firefly/webhooks + +LaraFly's inbound webhooks. Every provider configured under `firefly.webhooks.providers` gets an endpoint, +`POST /webhooks/{provider}`, whose request is size-checked, signature-verified (generic HMAC, Stripe, GitHub, +Twilio, or deliberately none), held to a replay window where the provider signs a timestamp, deduplicated by its +delivery id on an idempotency store (memory, the Laravel cache, or a portable table on any database connection), +and published to the `firefly/eda` event bus as `webhook.{provider}.{type}` on `webhooks.{provider}`. A refusal is +a problem+json document with a stable code. Off until `firefly.webhooks.enabled` is true. + +Apache-2.0 © Firefly Software Solutions Inc. diff --git a/packages/webhooks/cache/firefly-webhooks-components.php b/packages/webhooks/cache/firefly-webhooks-components.php new file mode 100644 index 00000000..33b9d4cb --- /dev/null +++ b/packages/webhooks/cache/firefly-webhooks-components.php @@ -0,0 +1,36 @@ + [ + 'class' => 'Firefly\\Webhooks\\WebhooksAutoConfiguration', + 'stereotype' => 'configuration', + 'name' => null, + 'scope' => 'Singleton', + 'primary' => false, + 'order' => 1000, + 'qualifier' => null, + 'interfaces' => [ + ], + 'beans' => [ + 0 => [ + 'method' => 'webhooksSettings', + 'returns' => 'Firefly\\Webhooks\\WebhooksSettings', + 'name' => null, + 'scope' => 'Singleton', + 'primary' => false, + 'order' => 0, + 'lazy' => false, + 'dependencies' => [ + 0 => 'Firefly\\Config\\Config', + ], + ], + ], + 'lazy' => false, + 'dependencies' => [ + ], + ], +]; diff --git a/packages/webhooks/cache/firefly-webhooks-context.php b/packages/webhooks/cache/firefly-webhooks-context.php new file mode 100644 index 00000000..e93b2c9b --- /dev/null +++ b/packages/webhooks/cache/firefly-webhooks-context.php @@ -0,0 +1,40 @@ + [ + 'class' => 'Firefly\\Webhooks\\WebhooksAutoConfiguration', + 'postConstruct' => [ + ], + 'preDestroy' => [ + ], + 'listeners' => [ + ], + 'conditions' => [ + ], + 'beanConditions' => [ + 0 => [ + 'method' => 'webhooksSettings', + 'conditions' => [ + 0 => [ + 'type' => 'Firefly\\Context\\Condition\\Attributes\\ConditionalOnProperty', + 'args' => [ + 0 => 'firefly.webhooks.enabled', + 1 => 'true', + 2 => false, + ], + ], + 1 => [ + 'type' => 'Firefly\\Context\\Condition\\Attributes\\ConditionalOnMissingBean', + 'args' => [ + 0 => 'Firefly\\Webhooks\\WebhooksSettings', + ], + ], + ], + ], + ], + ], +]; diff --git a/packages/webhooks/composer.json b/packages/webhooks/composer.json new file mode 100644 index 00000000..046ca412 --- /dev/null +++ b/packages/webhooks/composer.json @@ -0,0 +1,67 @@ +{ + "name": "firefly/webhooks", + "description": "LaraFly inbound webhooks: one endpoint per configured provider, HMAC/Stripe/GitHub/Twilio/NoOp signature validators with a replay window, an idempotency store on memory, the Laravel cache or any database connection, and every accepted webhook published to the firefly/eda event bus; refusals answer problem+json.", + "type": "library", + "license": "Apache-2.0", + "homepage": "https://github.com/fireflyframework/fireflyframework-php", + "authors": [ + { + "name": "Firefly Software Solutions Inc.", + "homepage": "https://github.com/fireflyframework" + } + ], + "keywords": [ + "firefly", + "laravel", + "webhooks" + ], + "support": { + "issues": "https://github.com/fireflyframework/fireflyframework-php/issues", + "source": "https://github.com/fireflyframework/fireflyframework-php/tree/main/packages/webhooks" + }, + "require": { + "php": "^8.3", + "firefly/autoconfigure": "*@dev", + "firefly/config": "*@dev", + "firefly/container": "*@dev", + "firefly/context": "*@dev", + "firefly/eda": "*@dev", + "firefly/kernel": "*@dev", + "firefly/resilience": "*@dev", + "firefly/web": "*@dev", + "illuminate/cache": "^13.0", + "illuminate/console": "^13.0", + "illuminate/container": "^13.0", + "illuminate/contracts": "^13.0", + "illuminate/database": "^13.0", + "illuminate/http": "^13.0", + "illuminate/routing": "^13.0", + "illuminate/support": "^13.0", + "psr/log": "^3.0" + }, + "extra": { + "laravel": { + "providers": [ + "Firefly\\Webhooks\\WebhooksServiceProvider", + "Firefly\\Webhooks\\WebhooksWiringProvider" + ] + }, + "branch-alias": { + "dev-main": "26.x-dev" + } + }, + "autoload": { + "psr-4": { + "Firefly\\Webhooks\\": "src/" + } + }, + "autoload-dev": { + "psr-4": { + "Firefly\\Webhooks\\Tests\\": "tests/" + } + }, + "minimum-stability": "stable", + "config": { + "sort-packages": true + } +} diff --git a/packages/webhooks/src/Provider/SignatureKind.php b/packages/webhooks/src/Provider/SignatureKind.php new file mode 100644 index 00000000..77b68cbe --- /dev/null +++ b/packages/webhooks/src/Provider/SignatureKind.php @@ -0,0 +1,39 @@ + 'github', 'secret' => …]` is a complete configuration; every + * default can be overridden per provider. + * + * * hmac — a generic HMAC over the raw body (or "." when a timestamp header is configured). + * * stripe — `Stripe-Signature: t=…,v1=…`, HMAC-SHA256 over ".", with a replay window. + * * github — `X-Hub-Signature-256: sha256=…`, HMAC-SHA256 over the body. + * * twilio — `X-Twilio-Signature`, base64(HMAC-SHA1) over the URL plus the sorted form parameters. + * * none — no verification at all; an explicit choice for a sender on a private network. + */ +enum SignatureKind: string +{ + case Hmac = 'hmac'; + case Stripe = 'stripe'; + case GitHub = 'github'; + case Twilio = 'twilio'; + case None = 'none'; + + /** + * @return array{signature-header: string, signature-prefix: string, idempotency-header: string, idempotency-field: string, event-type-header: string, event-type-field: string} + */ + public function defaults(): array + { + return match ($this) { + self::Hmac => ['signature-header' => 'X-Signature', 'signature-prefix' => 'sha256=', 'idempotency-header' => 'X-Idempotency-Key', 'idempotency-field' => '', 'event-type-header' => '', 'event-type-field' => 'type'], + self::Stripe => ['signature-header' => 'Stripe-Signature', 'signature-prefix' => '', 'idempotency-header' => '', 'idempotency-field' => 'id', 'event-type-header' => '', 'event-type-field' => 'type'], + self::GitHub => ['signature-header' => 'X-Hub-Signature-256', 'signature-prefix' => 'sha256=', 'idempotency-header' => 'X-GitHub-Delivery', 'idempotency-field' => '', 'event-type-header' => 'X-GitHub-Event', 'event-type-field' => ''], + self::Twilio => ['signature-header' => 'X-Twilio-Signature', 'signature-prefix' => '', 'idempotency-header' => 'I-Twilio-Idempotency-Token', 'idempotency-field' => '', 'event-type-header' => '', 'event-type-field' => 'EventType'], + self::None => ['signature-header' => '', 'signature-prefix' => '', 'idempotency-header' => 'X-Idempotency-Key', 'idempotency-field' => '', 'event-type-header' => '', 'event-type-field' => 'type'], + }; + } +} diff --git a/packages/webhooks/src/Provider/WebhookProviderSettings.php b/packages/webhooks/src/Provider/WebhookProviderSettings.php new file mode 100644 index 00000000..1599484d --- /dev/null +++ b/packages/webhooks/src/Provider/WebhookProviderSettings.php @@ -0,0 +1,104 @@ + $raw + */ + public static function fromArray(string $id, array $raw, float $defaultTolerance): self + { + if (preg_match('/^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/', $id) !== 1) { + throw new ConfigurationException("firefly.webhooks.providers.{$id}: a provider id is 1 to 64 letters, digits, dots, underscores or dashes, starting with a letter or digit."); + } + + $where = "firefly.webhooks.providers.{$id}"; + $kindName = self::text($raw, 'validator', 'hmac'); + $kind = SignatureKind::tryFrom($kindName) + ?? throw new ConfigurationException("{$where}.validator must be one of hmac, stripe, github, twilio or none; got '{$kindName}'."); + $defaults = $kind->defaults(); + + $secret = self::text($raw, 'secret', ''); + if ($kind !== SignatureKind::None && $secret === '') { + throw new ConfigurationException("{$where}.secret is empty: a {$kind->value} validator cannot verify anything without it."); + } + + $algorithm = self::text($raw, 'algorithm', 'sha256'); + if (! in_array($algorithm, hash_hmac_algos(), true)) { + throw new ConfigurationException("{$where}.algorithm '{$algorithm}' is not an HMAC algorithm this PHP supports."); + } + + $encoding = self::text($raw, 'encoding', 'hex'); + if (! in_array($encoding, ['hex', 'base64'], true)) { + throw new ConfigurationException("{$where}.encoding must be 'hex' or 'base64'; got '{$encoding}'."); + } + + return new self( + id: $id, + validator: $kind, + secret: $secret, + signatureHeader: self::text($raw, 'signature-header', $defaults['signature-header']), + signaturePrefix: self::text($raw, 'signature-prefix', $defaults['signature-prefix']), + algorithm: $algorithm, + encoding: $encoding, + timestampHeader: self::text($raw, 'timestamp-header', ''), + tolerance: WebhooksSettings::seconds($raw['tolerance'] ?? null, $defaultTolerance, "{$where}.tolerance"), + idempotencyHeader: self::text($raw, 'idempotency-header', $defaults['idempotency-header']), + idempotencyField: self::text($raw, 'idempotency-field', $defaults['idempotency-field']), + eventTypeHeader: self::text($raw, 'event-type-header', $defaults['event-type-header']), + eventTypeField: self::text($raw, 'event-type-field', $defaults['event-type-field']), + destination: self::optional($raw, 'destination'), + publicUrl: self::optional($raw, 'public-url'), + ); + } + + /** + * @param array $raw + */ + private static function text(array $raw, string $key, string $default): string + { + $value = $raw[$key] ?? null; + + return is_string($value) ? $value : $default; + } + + /** + * @param array $raw + */ + private static function optional(array $raw, string $key): ?string + { + $value = $raw[$key] ?? null; + + return is_string($value) && $value !== '' ? $value : null; + } +} diff --git a/packages/webhooks/src/WebhooksAutoConfiguration.php b/packages/webhooks/src/WebhooksAutoConfiguration.php new file mode 100644 index 00000000..3ac1c8f9 --- /dev/null +++ b/packages/webhooks/src/WebhooksAutoConfiguration.php @@ -0,0 +1,30 @@ + */ + public const array STORES = ['memory', 'cache', 'database']; + + /** + * @param array $providers + */ + public function __construct( + public bool $enabled, + public string $path, + public int $maxBodyBytes, + public string $destination, + public string $eventType, + public float $replayTolerance, + public string $idempotencyStore, + public float $idempotencyTtl, + public string $idempotencyCacheStore, + public ?string $idempotencyConnection, + public array $providers, + ) {} + + public static function fromConfig(Config $config): self + { + $tolerance = self::seconds($config->get('firefly.webhooks.replay.tolerance'), 300.0, 'firefly.webhooks.replay.tolerance'); + + $store = $config->string('firefly.webhooks.idempotency.store', 'cache'); + if (! in_array($store, self::STORES, true)) { + throw new ConfigurationException("firefly.webhooks.idempotency.store must be memory, cache or database; got '{$store}'."); + } + + $maxBody = $config->int('firefly.webhooks.max-body-bytes', 1048576); + if ($maxBody < 1) { + throw new ConfigurationException("firefly.webhooks.max-body-bytes must be a positive number of bytes; got {$maxBody}."); + } + + $providers = []; + foreach ($config->array('firefly.webhooks.providers', []) as $id => $raw) { + if (! is_array($raw)) { + throw new ConfigurationException("firefly.webhooks.providers.{$id} must be an array of provider settings."); + } + $providers[(string) $id] = WebhookProviderSettings::fromArray((string) $id, $raw, $tolerance); + } + + $connection = $config->get('firefly.webhooks.idempotency.connection'); + + return new self( + enabled: $config->bool('firefly.webhooks.enabled', false), + path: trim($config->string('firefly.webhooks.path', 'webhooks'), '/'), + maxBodyBytes: $maxBody, + destination: $config->string('firefly.webhooks.destination', 'webhooks.{provider}'), + eventType: $config->string('firefly.webhooks.event-type', 'webhook.{provider}.{type}'), + replayTolerance: $tolerance, + idempotencyStore: $store, + idempotencyTtl: self::seconds($config->get('firefly.webhooks.idempotency.ttl'), 86400.0, 'firefly.webhooks.idempotency.ttl'), + idempotencyCacheStore: $config->string('firefly.webhooks.idempotency.cache-store', ''), + idempotencyConnection: is_string($connection) && $connection !== '' ? $connection : null, + providers: $providers, + ); + } + + public function provider(string $id): ?WebhookProviderSettings + { + return $this->providers[$id] ?? null; + } + + /** A duration setting in seconds: null takes the default; anything that is not a non-negative duration refuses. */ + public static function seconds(mixed $value, float $default, string $key): float + { + if ($value === null) { + return $default; + } + + if ((is_int($value) || is_float($value)) && $value >= 0) { + return (float) $value; + } + + if (is_string($value)) { + try { + return Duration::parse($value); + } catch (Throwable $e) { + throw new ConfigurationException("{$key} must be a duration such as 90s, 5m or 24h; got '{$value}'.", previous: $e); + } + } + + throw new ConfigurationException("{$key} must be a duration such as 90s, 5m or 24h, or a number of seconds."); + } +} diff --git a/packages/webhooks/src/WebhooksWiringProvider.php b/packages/webhooks/src/WebhooksWiringProvider.php new file mode 100644 index 00000000..d9acb61b --- /dev/null +++ b/packages/webhooks/src/WebhooksWiringProvider.php @@ -0,0 +1,20 @@ + + */ + public function passes(): array + { + return []; + } +} diff --git a/packages/webhooks/tests/CompiledManifestFreshnessTest.php b/packages/webhooks/tests/CompiledManifestFreshnessTest.php new file mode 100644 index 00000000..4ff6abc0 --- /dev/null +++ b/packages/webhooks/tests/CompiledManifestFreshnessTest.php @@ -0,0 +1,30 @@ + dirname(__DIR__).'/src']; + $cache = dirname(__DIR__).'/cache'; + + $freshComponents = sys_get_temp_dir().'/firefly-webhooks-fresh-c-'.bin2hex(random_bytes(6)).'.php'; + $freshContext = sys_get_temp_dir().'/firefly-webhooks-fresh-x-'.bin2hex(random_bytes(6)).'.php'; + + try { + (new AutoConfigManifestCompiler)->write($src, $freshComponents, $freshContext); + + expect(ComponentManifest::load($freshComponents))->toEqual(ComponentManifest::load($cache.'/firefly-webhooks-components.php')) + ->and(ContextManifest::load($freshContext))->toEqual(ContextManifest::load($cache.'/firefly-webhooks-context.php')); + } finally { + @unlink($freshComponents); + @unlink($freshContext); + } +}); diff --git a/packages/webhooks/tests/PackageBootsTest.php b/packages/webhooks/tests/PackageBootsTest.php new file mode 100644 index 00000000..490237db --- /dev/null +++ b/packages/webhooks/tests/PackageBootsTest.php @@ -0,0 +1,23 @@ + []], providers: [WebhooksServiceProvider::class, WebhooksWiringProvider::class]); + + expect($app->make(ApplicationContext::class))->toBeInstanceOf(ApplicationContext::class) + ->and($app->bound(WebhooksSettings::class))->toBeFalse(); +}); + +it('refuses the boot when an enabled provider has no secret', function () { + expect(fn () => fireflyApplication( + config: ['firefly' => ['webhooks' => ['enabled' => true, 'providers' => ['stripe' => ['validator' => 'stripe']]]]], + providers: [WebhooksServiceProvider::class, WebhooksWiringProvider::class], + ))->toThrow(ConfigurationException::class, 'firefly.webhooks.providers.stripe.secret'); +}); diff --git a/packages/webhooks/tests/WebhooksSettingsTest.php b/packages/webhooks/tests/WebhooksSettingsTest.php new file mode 100644 index 00000000..ac4e1410 --- /dev/null +++ b/packages/webhooks/tests/WebhooksSettingsTest.php @@ -0,0 +1,69 @@ + $webhooks a firefly.webhooks subtree, as a config file or a dataset spells it + */ +function w4WebhooksSettings(array $webhooks): WebhooksSettings +{ + return WebhooksSettings::fromConfig(new Config(new Repository(['firefly' => ['webhooks' => $webhooks]]))); +} + +it('has safe defaults: off, cache idempotency for a day, a five-minute replay window', function () { + $settings = w4WebhooksSettings([]); + + expect($settings->enabled)->toBeFalse() + ->and($settings->path)->toBe('webhooks') + ->and($settings->maxBodyBytes)->toBe(1048576) + ->and($settings->destination)->toBe('webhooks.{provider}') + ->and($settings->eventType)->toBe('webhook.{provider}.{type}') + ->and($settings->replayTolerance)->toBe(300.0) + ->and($settings->idempotencyStore)->toBe('cache') + ->and($settings->idempotencyTtl)->toBe(86400.0) + ->and($settings->providers)->toBe([]); +}); + +it('fills each provider kind with the headers that provider actually sends', function () { + $settings = w4WebhooksSettings(['providers' => [ + 'stripe' => ['validator' => 'stripe', 'secret' => 's'], + 'github' => ['validator' => 'github', 'secret' => 'g'], + 'twilio' => ['validator' => 'twilio', 'secret' => 't', 'public-url' => 'https://api.example.com/webhooks/twilio'], + 'acme' => ['secret' => 'a', 'timestamp-header' => 'X-Timestamp', 'tolerance' => '90s', 'destination' => 'partners.acme'], + 'open' => ['validator' => 'none'], + ]]); + + expect($settings->provider('stripe')?->signatureHeader)->toBe('Stripe-Signature') + ->and($settings->provider('stripe')?->idempotencyField)->toBe('id') + ->and($settings->provider('github')?->signatureHeader)->toBe('X-Hub-Signature-256') + ->and($settings->provider('github')?->idempotencyHeader)->toBe('X-GitHub-Delivery') + ->and($settings->provider('github')?->eventTypeHeader)->toBe('X-GitHub-Event') + ->and($settings->provider('twilio')?->idempotencyHeader)->toBe('I-Twilio-Idempotency-Token') + ->and($settings->provider('twilio')?->publicUrl)->toBe('https://api.example.com/webhooks/twilio') + ->and($settings->provider('acme')?->validator)->toBe(SignatureKind::Hmac) + ->and($settings->provider('acme')?->signatureHeader)->toBe('X-Signature') + ->and($settings->provider('acme')?->signaturePrefix)->toBe('sha256=') + ->and($settings->provider('acme')?->tolerance)->toBe(90.0) + ->and($settings->provider('acme')?->destination)->toBe('partners.acme') + ->and($settings->provider('open')?->validator)->toBe(SignatureKind::None) + ->and($settings->provider('missing'))->toBeNull(); +}); + +it('refuses at boot what could never verify anything', function (array $webhooks, string $message) { + expect(fn () => w4WebhooksSettings($webhooks))->toThrow(ConfigurationException::class, $message); +})->with([ + 'unknown validator' => [['providers' => ['x' => ['validator' => 'paypal', 'secret' => 's']]], 'firefly.webhooks.providers.x.validator'], + 'missing secret' => [['providers' => ['x' => ['validator' => 'stripe']]], 'firefly.webhooks.providers.x.secret'], + 'bad id' => [['providers' => ['../x' => ['secret' => 's']]], 'provider id'], + 'bad algorithm' => [['providers' => ['x' => ['secret' => 's', 'algorithm' => 'md42']]], 'firefly.webhooks.providers.x.algorithm'], + 'bad encoding' => [['providers' => ['x' => ['secret' => 's', 'encoding' => 'hex64']]], 'firefly.webhooks.providers.x.encoding'], + 'unknown store' => [['idempotency' => ['store' => 'redis']], 'firefly.webhooks.idempotency.store'], + 'bad tolerance' => [['replay' => ['tolerance' => 'soon']], 'firefly.webhooks.replay.tolerance'], + 'empty body limit' => [['max-body-bytes' => 0], 'firefly.webhooks.max-body-bytes'], +]); From abcf36407a4bb0fedb6307a9064121421f0efc67 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 20:24:11 -0700 Subject: [PATCH 31/47] feat(webhooks): HMAC, Stripe, GitHub, Twilio and NoOp signature validators with a replay window --- .../Exception/WebhookRejectedException.php | 42 ++++++ .../Signature/GitHubSignatureValidator.php | 27 ++++ .../src/Signature/HmacSignatureValidator.php | 75 ++++++++++ .../src/Signature/NoOpSignatureValidator.php | 13 ++ .../webhooks/src/Signature/ReplayWindow.php | 21 +++ .../src/Signature/SignatureValidator.php | 17 +++ .../Signature/SignatureValidatorFactory.php | 29 ++++ .../Signature/StripeSignatureValidator.php | 69 +++++++++ .../Signature/TwilioSignatureValidator.php | 72 +++++++++ packages/webhooks/src/WebhookRequest.php | 64 ++++++++ .../Signature/SignatureValidatorsTest.php | 141 ++++++++++++++++++ .../tests/Support/WebhookRequests.php | 25 ++++ .../webhooks/tests/WebhookRequestTest.php | 30 ++++ 13 files changed, 625 insertions(+) create mode 100644 packages/webhooks/src/Exception/WebhookRejectedException.php create mode 100644 packages/webhooks/src/Signature/GitHubSignatureValidator.php create mode 100644 packages/webhooks/src/Signature/HmacSignatureValidator.php create mode 100644 packages/webhooks/src/Signature/NoOpSignatureValidator.php create mode 100644 packages/webhooks/src/Signature/ReplayWindow.php create mode 100644 packages/webhooks/src/Signature/SignatureValidator.php create mode 100644 packages/webhooks/src/Signature/SignatureValidatorFactory.php create mode 100644 packages/webhooks/src/Signature/StripeSignatureValidator.php create mode 100644 packages/webhooks/src/Signature/TwilioSignatureValidator.php create mode 100644 packages/webhooks/src/WebhookRequest.php create mode 100644 packages/webhooks/tests/Signature/SignatureValidatorsTest.php create mode 100644 packages/webhooks/tests/Support/WebhookRequests.php create mode 100644 packages/webhooks/tests/WebhookRequestTest.php diff --git a/packages/webhooks/src/Exception/WebhookRejectedException.php b/packages/webhooks/src/Exception/WebhookRejectedException.php new file mode 100644 index 00000000..ba3a2b41 --- /dev/null +++ b/packages/webhooks/src/Exception/WebhookRejectedException.php @@ -0,0 +1,42 @@ + $provider]); + } + + public static function tooLarge(string $provider, int $limit): self + { + return new self("The [{$provider}] webhook body is larger than firefly.webhooks.max-body-bytes ({$limit} bytes).", 'WEBHOOK_PAYLOAD_TOO_LARGE', 413, ErrorCategory::Validation, ErrorSeverity::Warning, extensions: ['provider' => $provider]); + } + + public static function signatureInvalid(string $provider, string $reason): self + { + return new self("The [{$provider}] webhook signature is invalid: {$reason}.", 'WEBHOOK_SIGNATURE_INVALID', 401, ErrorCategory::Security, ErrorSeverity::Warning, extensions: ['provider' => $provider]); + } + + public static function replayed(string $provider, int $timestamp, float $tolerance): self + { + return new self("The [{$provider}] webhook was signed at {$timestamp}, outside the {$tolerance}s replay window.", 'WEBHOOK_TIMESTAMP_OUT_OF_TOLERANCE', 401, ErrorCategory::Security, ErrorSeverity::Warning, extensions: ['provider' => $provider]); + } + + public static function invalidPayload(string $provider, string $reason): self + { + return new self("The [{$provider}] webhook body cannot be read: {$reason}.", 'WEBHOOK_PAYLOAD_INVALID', 400, ErrorCategory::Validation, ErrorSeverity::Warning, extensions: ['provider' => $provider]); + } +} diff --git a/packages/webhooks/src/Signature/GitHubSignatureValidator.php b/packages/webhooks/src/Signature/GitHubSignatureValidator.php new file mode 100644 index 00000000..c4d535c3 --- /dev/null +++ b/packages/webhooks/src/Signature/GitHubSignatureValidator.php @@ -0,0 +1,27 @@ +`. A named HMAC validator, + * as PyFly's is. GitHub signs no timestamp, so replays are stopped by the idempotency store keyed on + * `X-GitHub-Delivery` instead. + */ +final class GitHubSignatureValidator implements SignatureValidator +{ + private readonly HmacSignatureValidator $hmac; + + public function __construct(string $secret, string $header = 'X-Hub-Signature-256') + { + $this->hmac = new HmacSignatureValidator($secret, $header, 'sha256=', 'sha256', 'hex'); + } + + public function verify(WebhookRequest $request): void + { + $this->hmac->verify($request); + } +} diff --git a/packages/webhooks/src/Signature/HmacSignatureValidator.php b/packages/webhooks/src/Signature/HmacSignatureValidator.php new file mode 100644 index 00000000..95689d2b --- /dev/null +++ b/packages/webhooks/src/Signature/HmacSignatureValidator.php @@ -0,0 +1,75 @@ +`, hex or base64, with any algorithm hash_hmac() supports. With a timestamp header the signed + * payload is "." and the timestamp must sit inside the replay window — the scheme firefly/callbacks + * sends, so a LaraFly receiver verifies a LaraFly sender with `timestamp-header: X-Firefly-Timestamp`. Compared with + * hash_equals(), so the comparison leaks no timing. + */ +final class HmacSignatureValidator implements SignatureValidator +{ + /** @var Closure(): int */ + private readonly Closure $clock; + + /** + * @param (Closure(): int)|null $clock unix seconds + */ + public function __construct( + private readonly string $secret, + private readonly string $header = 'X-Signature', + private readonly string $prefix = 'sha256=', + private readonly string $algorithm = 'sha256', + private readonly string $encoding = 'hex', + private readonly string $timestampHeader = '', + private readonly float $tolerance = 300.0, + ?Closure $clock = null, + ) { + $this->clock = $clock ?? static fn (): int => time(); + } + + public function verify(WebhookRequest $request): void + { + $signature = $request->header($this->header); + if ($signature === null || $signature === '') { + throw WebhookRejectedException::signatureInvalid($request->provider, "the {$this->header} header is missing"); + } + + if ($this->prefix !== '') { + if (! str_starts_with($signature, $this->prefix)) { + throw WebhookRejectedException::signatureInvalid($request->provider, "the signature does not start with {$this->prefix}"); + } + $signature = substr($signature, strlen($this->prefix)); + } + + $signed = $request->body; + if ($this->timestampHeader !== '') { + $timestamp = $request->header($this->timestampHeader); + if ($timestamp === null || preg_match('/^\d{1,12}$/', $timestamp) !== 1) { + throw WebhookRejectedException::signatureInvalid($request->provider, "the {$this->timestampHeader} header is missing or not a unix timestamp"); + } + + ReplayWindow::assertWithin((int) $timestamp, $this->tolerance, ($this->clock)(), $request->provider); + $signed = $timestamp.'.'.$request->body; + } + + if (! hash_equals($this->expected($signed), $signature)) { + throw WebhookRejectedException::signatureInvalid($request->provider, 'the signature does not match the body'); + } + } + + public function expected(string $signed): string + { + $raw = hash_hmac($this->algorithm, $signed, $this->secret, true); + + return $this->encoding === 'base64' ? base64_encode($raw) : bin2hex($raw); + } +} diff --git a/packages/webhooks/src/Signature/NoOpSignatureValidator.php b/packages/webhooks/src/Signature/NoOpSignatureValidator.php new file mode 100644 index 00000000..8ad6ded1 --- /dev/null +++ b/packages/webhooks/src/Signature/NoOpSignatureValidator.php @@ -0,0 +1,13 @@ + 0 && abs($now - $timestamp) > $tolerance) { + throw WebhookRejectedException::replayed($provider, $timestamp, $tolerance); + } + } +} diff --git a/packages/webhooks/src/Signature/SignatureValidator.php b/packages/webhooks/src/Signature/SignatureValidator.php new file mode 100644 index 00000000..18e19397 --- /dev/null +++ b/packages/webhooks/src/Signature/SignatureValidator.php @@ -0,0 +1,17 @@ +validator) { + SignatureKind::None => new NoOpSignatureValidator, + SignatureKind::Hmac => new HmacSignatureValidator($provider->secret, $provider->signatureHeader, $provider->signaturePrefix, $provider->algorithm, $provider->encoding, $provider->timestampHeader, $provider->tolerance, $this->clock), + SignatureKind::Stripe => new StripeSignatureValidator($provider->secret, $provider->signatureHeader, $provider->tolerance, $this->clock), + SignatureKind::GitHub => new GitHubSignatureValidator($provider->secret, $provider->signatureHeader), + SignatureKind::Twilio => new TwilioSignatureValidator($provider->secret, $provider->signatureHeader), + }; + } +} diff --git a/packages/webhooks/src/Signature/StripeSignatureValidator.php b/packages/webhooks/src/Signature/StripeSignatureValidator.php new file mode 100644 index 00000000..ced05566 --- /dev/null +++ b/packages/webhooks/src/Signature/StripeSignatureValidator.php @@ -0,0 +1,69 @@ +,v1=[,v1=…]`, HMAC-SHA256 over ".". + * Any v1 may match — Stripe sends one per active secret while a secret is being rolled — and t must sit inside the + * replay window (Stripe's own default tolerance is the same five minutes). + */ +final class StripeSignatureValidator implements SignatureValidator +{ + /** @var Closure(): int */ + private readonly Closure $clock; + + /** + * @param (Closure(): int)|null $clock unix seconds + */ + public function __construct( + private readonly string $secret, + private readonly string $header = 'Stripe-Signature', + private readonly float $tolerance = 300.0, + ?Closure $clock = null, + ) { + $this->clock = $clock ?? static fn (): int => time(); + } + + public function verify(WebhookRequest $request): void + { + $value = $request->header($this->header); + if ($value === null || $value === '') { + throw WebhookRejectedException::signatureInvalid($request->provider, "the {$this->header} header is missing"); + } + + $timestamp = null; + $signatures = []; + foreach (explode(',', $value) as $pair) { + [$key, $part] = array_pad(explode('=', trim($pair), 2), 2, ''); + if ($key === 't') { + $timestamp = $part; + } elseif ($key === 'v1') { + $signatures[] = $part; + } + } + + if ($timestamp === null || preg_match('/^\d{1,12}$/', $timestamp) !== 1) { + throw WebhookRejectedException::signatureInvalid($request->provider, 'the header has no t= timestamp'); + } + if ($signatures === []) { + throw WebhookRejectedException::signatureInvalid($request->provider, 'the header has no v1= signature'); + } + + ReplayWindow::assertWithin((int) $timestamp, $this->tolerance, ($this->clock)(), $request->provider); + + $expected = hash_hmac('sha256', $timestamp.'.'.$request->body, $this->secret); + foreach ($signatures as $signature) { + if (hash_equals($expected, $signature)) { + return; + } + } + + throw WebhookRejectedException::signatureInvalid($request->provider, 'no v1 signature matches the body'); + } +} diff --git a/packages/webhooks/src/Signature/TwilioSignatureValidator.php b/packages/webhooks/src/Signature/TwilioSignatureValidator.php new file mode 100644 index 00000000..386eb4bc --- /dev/null +++ b/packages/webhooks/src/Signature/TwilioSignatureValidator.php @@ -0,0 +1,72 @@ +))`, where the + * signed string is the full URL followed by every POST parameter, sorted by name, as name + value. A JSON callback + * signs the URL alone and carries the body's SHA-256 in the `bodySHA256` query parameter, which is checked first. + * Twilio signs no timestamp: replays are stopped by the idempotency store keyed on `I-Twilio-Idempotency-Token`. + * The URL must be the one Twilio called — set the provider's public-url behind a proxy that rewrites the host. + */ +final class TwilioSignatureValidator implements SignatureValidator +{ + public function __construct( + private readonly string $authToken, + private readonly string $header = 'X-Twilio-Signature', + ) {} + + public function verify(WebhookRequest $request): void + { + $signature = $request->header($this->header); + if ($signature === null || $signature === '') { + throw WebhookRejectedException::signatureInvalid($request->provider, "the {$this->header} header is missing"); + } + + if (str_contains($request->contentType, 'json')) { + parse_str((string) parse_url($request->url, PHP_URL_QUERY), $query); + $bodyHash = $query['bodySHA256'] ?? null; + if (! is_string($bodyHash) || ! hash_equals(hash('sha256', $request->body), strtolower($bodyHash))) { + throw WebhookRejectedException::signatureInvalid($request->provider, 'the bodySHA256 parameter does not match the body'); + } + $signed = $request->url; + } else { + $signed = self::signedForm($request->url, $request->form); + } + + if (! hash_equals(base64_encode(hash_hmac('sha1', $signed, $this->authToken, true)), $signature)) { + throw WebhookRejectedException::signatureInvalid($request->provider, 'the signature does not match the URL and parameters'); + } + } + + /** + * The URL followed by name + value for every parameter, names sorted. A repeated parameter contributes each + * distinct value once, values sorted — what twilio-php's RequestValidator signs, so a multi-valued callback + * verifies here exactly as it does there. + * + * @param array $params + */ + public static function signedForm(string $url, array $params): string + { + ksort($params, SORT_STRING); + $signed = $url; + foreach ($params as $name => $value) { + $items = []; + foreach (is_array($value) ? $value : [$value] as $item) { + $items[] = is_scalar($item) ? (string) $item : ''; + } + $items = array_unique($items); + sort($items, SORT_STRING); + foreach ($items as $item) { + $signed .= $name.$item; + } + } + + return $signed; + } +} diff --git a/packages/webhooks/src/WebhookRequest.php b/packages/webhooks/src/WebhookRequest.php new file mode 100644 index 00000000..70c7af51 --- /dev/null +++ b/packages/webhooks/src/WebhookRequest.php @@ -0,0 +1,64 @@ + $headers lower-cased name => first value + * @param array $form + */ + public function __construct( + public string $provider, + public string $method, + public string $url, + public array $headers, + public string $body, + public array $form, + public string $contentType, + public int $receivedAt, + public ?string $sourceIp = null, + ) {} + + public function header(string $name): ?string + { + return $this->headers[strtolower($name)] ?? null; + } + + public static function fromHttp(Request $request, string $provider, ?string $publicUrl, int $now): self + { + $headers = []; + foreach ($request->headers->all() as $name => $values) { + $first = $values[0] ?? null; + if (is_string($first)) { + $headers[strtolower((string) $name)] = $first; + } + } + + $uri = $request->getRequestUri(); + $query = ($position = strpos($uri, '?')) !== false ? substr($uri, $position) : ''; + $url = $publicUrl !== null ? rtrim($publicUrl, '/').$query : $request->getSchemeAndHttpHost().$uri; + + $contentType = strtolower(trim(explode(';', (string) $request->headers->get('Content-Type', ''))[0])); + + $form = []; + if ($contentType === 'application/x-www-form-urlencoded') { + foreach ($request->request->all() as $name => $value) { + $form[(string) $name] = $value; + } + } + + return new self($provider, $request->getMethod(), $url, $headers, (string) $request->getContent(), $form, $contentType, $now, $request->ip()); + } +} diff --git a/packages/webhooks/tests/Signature/SignatureValidatorsTest.php b/packages/webhooks/tests/Signature/SignatureValidatorsTest.php new file mode 100644 index 00000000..e55e342f --- /dev/null +++ b/packages/webhooks/tests/Signature/SignatureValidatorsTest.php @@ -0,0 +1,141 @@ + W4_NOW; +} + +function w4Rejection(Closure $verify): WebhookRejectedException +{ + try { + $verify(); + } catch (WebhookRejectedException $e) { + return $e; + } + + throw new RuntimeException('the request was accepted'); +} + +it('accepts a generic HMAC-SHA256 over the body and rejects a tampered body', function () { + $body = '{"type":"order.created","id":1}'; + $signature = 'sha256='.hash_hmac('sha256', $body, 'secret'); + $validator = new HmacSignatureValidator('secret'); + + $validator->verify(WebhookRequests::make('acme', $body, ['X-Signature' => $signature])); + + $rejected = w4Rejection(fn () => $validator->verify(WebhookRequests::make('acme', $body.' ', ['X-Signature' => $signature]))); + expect($rejected->errorCode())->toBe('WEBHOOK_SIGNATURE_INVALID') + ->and($rejected->httpStatus())->toBe(401) + ->and($rejected->extensions())->toBe(['provider' => 'acme']); +}); + +it('rejects a missing header and a missing prefix without comparing anything', function () { + $validator = new HmacSignatureValidator('secret'); + + expect(w4Rejection(fn () => $validator->verify(WebhookRequests::make('acme', '{}')))->getMessage())->toContain('X-Signature header is missing') + ->and(w4Rejection(fn () => $validator->verify(WebhookRequests::make('acme', '{}', ['X-Signature' => hash_hmac('sha256', '{}', 'secret')])))->getMessage())->toContain('sha256='); +}); + +it('signs "." when a timestamp header is configured, inside the replay window only', function () { + $body = '{"a":1}'; + $validator = new HmacSignatureValidator('secret', 'X-Firefly-Signature', 'sha256=', 'sha256', 'hex', 'X-Firefly-Timestamp', 300.0, w4Clock()); + $sign = static fn (int $t): array => ['X-Firefly-Timestamp' => (string) $t, 'X-Firefly-Signature' => 'sha256='.hash_hmac('sha256', $t.'.'.$body, 'secret')]; + + $validator->verify(WebhookRequests::make('acme', $body, $sign(W4_NOW - 299))); + + $replayed = w4Rejection(fn () => $validator->verify(WebhookRequests::make('acme', $body, $sign(W4_NOW - 301)))); + expect($replayed->errorCode())->toBe('WEBHOOK_TIMESTAMP_OUT_OF_TOLERANCE')->and($replayed->httpStatus())->toBe(401); +}); + +it('supports base64 encodings and other algorithms', function () { + $body = 'raw'; + $validator = new HmacSignatureValidator('secret', 'X-Sig', '', 'sha512', 'base64'); + + $validator->verify(WebhookRequests::make('acme', $body, ['X-Sig' => base64_encode(hash_hmac('sha512', $body, 'secret', true))])); + + expect(true)->toBeTrue(); +}); + +it('verifies Stripe-Signature with any matching v1 and enforces the tolerance', function () { + $body = '{"id":"evt_1","type":"payment_intent.succeeded"}'; + $t = W4_NOW - 10; + $good = hash_hmac('sha256', $t.'.'.$body, 'whsec'); + $validator = new StripeSignatureValidator('whsec', 'Stripe-Signature', 300.0, w4Clock()); + + $validator->verify(WebhookRequests::make('stripe', $body, ['Stripe-Signature' => "t={$t},v1=deadbeef,v1={$good}"])); + + $stale = W4_NOW - 1000; + $staleSig = hash_hmac('sha256', $stale.'.'.$body, 'whsec'); + expect(w4Rejection(fn () => $validator->verify(WebhookRequests::make('stripe', $body, ['Stripe-Signature' => "t={$stale},v1={$staleSig}"])))->errorCode())->toBe('WEBHOOK_TIMESTAMP_OUT_OF_TOLERANCE') + ->and(w4Rejection(fn () => $validator->verify(WebhookRequests::make('stripe', $body, ['Stripe-Signature' => "v1={$good}"])))->getMessage())->toContain('t=') + ->and(w4Rejection(fn () => $validator->verify(WebhookRequests::make('stripe', $body, ['Stripe-Signature' => "t={$t},v1=00"])))->errorCode())->toBe('WEBHOOK_SIGNATURE_INVALID'); +}); + +it('verifies GitHub X-Hub-Signature-256', function () { + $body = '{"zen":"Keep it logically awesome."}'; + $validator = new GitHubSignatureValidator('gh-secret'); + + $validator->verify(WebhookRequests::make('github', $body, ['X-Hub-Signature-256' => 'sha256='.hash_hmac('sha256', $body, 'gh-secret')])); + + expect(w4Rejection(fn () => $validator->verify(WebhookRequests::make('github', $body, ['X-Hub-Signature-256' => 'sha1=abc'])))->errorCode())->toBe('WEBHOOK_SIGNATURE_INVALID'); +}); + +it('verifies Twilio form callbacks over the URL plus the sorted parameters', function () { + $url = 'https://hooks.example.com/webhooks/twilio?foo=1'; + $form = ['To' => '+18005551212', 'CallSid' => 'CA123', 'Digits' => '1234']; + $expected = base64_encode(hash_hmac('sha1', $url.'CallSidCA123Digits1234To+18005551212', 'token', true)); + $validator = new TwilioSignatureValidator('token'); + + $validator->verify(WebhookRequests::make('twilio', http_build_query($form), ['X-Twilio-Signature' => $expected], $form, $url, 'application/x-www-form-urlencoded')); + + expect(TwilioSignatureValidator::signedForm($url, $form))->toBe($url.'CallSidCA123Digits1234To+18005551212') + ->and(w4Rejection(fn () => $validator->verify(WebhookRequests::make('twilio', 'x', ['X-Twilio-Signature' => $expected], ['To' => 'other'], $url, 'application/x-www-form-urlencoded')))->errorCode())->toBe('WEBHOOK_SIGNATURE_INVALID'); +}); + +it('signs a repeated Twilio parameter the way twilio-php does: values de-duplicated and sorted', function () { + $url = 'https://hooks.example.com/webhooks/twilio'; + + expect(TwilioSignatureValidator::signedForm($url, ['To' => '+1', 'Media' => ['b', 'a', 'b']]))->toBe($url.'MediaaMediabTo+1'); +}); + +it('verifies Twilio JSON callbacks through the bodySHA256 parameter', function () { + $body = '{"EventType":"onMessageAdded"}'; + $url = 'https://hooks.example.com/webhooks/twilio?bodySHA256='.hash('sha256', $body); + $signature = base64_encode(hash_hmac('sha1', $url, 'token', true)); + $validator = new TwilioSignatureValidator('token'); + + $validator->verify(WebhookRequests::make('twilio', $body, ['X-Twilio-Signature' => $signature], [], $url)); + + expect(w4Rejection(fn () => $validator->verify(WebhookRequests::make('twilio', $body.'x', ['X-Twilio-Signature' => $signature], [], $url)))->getMessage())->toContain('bodySHA256'); +}); + +it('accepts anything with the NoOp validator, and the factory picks the validator for each kind', function () { + (new NoOpSignatureValidator)->verify(WebhookRequests::make('open', 'anything')); + $factory = new SignatureValidatorFactory(w4Clock()); + $kinds = []; + foreach (['hmac', 'stripe', 'github', 'twilio', 'none'] as $kind) { + $kinds[$kind] = $factory->for(WebhookProviderSettings::fromArray('p', ['validator' => $kind, 'secret' => 's'], 300.0))::class; + } + + expect($kinds)->toBe([ + 'hmac' => HmacSignatureValidator::class, + 'stripe' => StripeSignatureValidator::class, + 'github' => GitHubSignatureValidator::class, + 'twilio' => TwilioSignatureValidator::class, + 'none' => NoOpSignatureValidator::class, + ]); +}); diff --git a/packages/webhooks/tests/Support/WebhookRequests.php b/packages/webhooks/tests/Support/WebhookRequests.php new file mode 100644 index 00000000..36d972a5 --- /dev/null +++ b/packages/webhooks/tests/Support/WebhookRequests.php @@ -0,0 +1,25 @@ + $headers + * @param array $form + */ + public static function make(string $provider, string $body, array $headers = [], array $form = [], string $url = 'https://hooks.example.com/webhooks/x', string $contentType = 'application/json'): WebhookRequest + { + $lower = []; + foreach ($headers as $name => $value) { + $lower[strtolower($name)] = $value; + } + + return new WebhookRequest($provider, 'POST', $url, $lower, $body, $form, $contentType, time()); + } +} diff --git a/packages/webhooks/tests/WebhookRequestTest.php b/packages/webhooks/tests/WebhookRequestTest.php new file mode 100644 index 00000000..700f17f7 --- /dev/null +++ b/packages/webhooks/tests/WebhookRequestTest.php @@ -0,0 +1,30 @@ + '+1'], [], [], [ + 'HTTP_X_TWILIO_SIGNATURE' => 'sig', + 'CONTENT_TYPE' => 'application/x-www-form-urlencoded; charset=utf-8', + 'REMOTE_ADDR' => '10.0.0.5', + ], 'To=%2B1'); + + $webhook = WebhookRequest::fromHttp($request, 'twilio', null, 1_760_000_000); + + expect($webhook->url)->toBe('https://api.example.com/webhooks/twilio?b=2&a=1') + ->and($webhook->header('X-Twilio-Signature'))->toBe('sig') + ->and($webhook->body)->toBe('To=%2B1') + ->and($webhook->form)->toBe(['To' => '+1']) + ->and($webhook->contentType)->toBe('application/x-www-form-urlencoded') + ->and($webhook->sourceIp)->toBe('10.0.0.5') + ->and($webhook->receivedAt)->toBe(1_760_000_000); +}); + +it('signs against the configured public URL when a proxy rewrites the host', function () { + $request = Request::create('http://10.0.0.7:8080/webhooks/twilio?x=1', 'POST'); + + expect(WebhookRequest::fromHttp($request, 'twilio', 'https://api.example.com/webhooks/twilio', 0)->url)->toBe('https://api.example.com/webhooks/twilio?x=1'); +}); From 1b627d378645e29105e743e16dd7247bf277f5ca Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 20:27:09 -0700 Subject: [PATCH 32/47] fix(webhooks): choose the Twilio signed string by the URL, so a replayed JSON signature cannot carry a forged body --- .../Signature/TwilioSignatureValidator.php | 17 ++++++++--- .../Signature/SignatureValidatorsTest.php | 29 +++++++++++++++++++ 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/packages/webhooks/src/Signature/TwilioSignatureValidator.php b/packages/webhooks/src/Signature/TwilioSignatureValidator.php index 386eb4bc..88aa21db 100644 --- a/packages/webhooks/src/Signature/TwilioSignatureValidator.php +++ b/packages/webhooks/src/Signature/TwilioSignatureValidator.php @@ -11,6 +11,9 @@ * Twilio (validator `twilio`): `X-Twilio-Signature: base64(HMAC-SHA1(auth token, ))`, where the * signed string is the full URL followed by every POST parameter, sorted by name, as name + value. A JSON callback * signs the URL alone and carries the body's SHA-256 in the `bodySHA256` query parameter, which is checked first. + * The branch is chosen by the URL, as twilio-php's RequestValidator does, never by the Content-Type an attacker can + * rewrite: a URL carrying `bodySHA256` always has its body checked; otherwise only a form-encoded request is signed + * as URL + parameters, and any other request must have an empty body, since nothing would cover its bytes. * Twilio signs no timestamp: replays are stopped by the idempotency store keyed on `I-Twilio-Idempotency-Token`. * The URL must be the one Twilio called — set the provider's public-url behind a proxy that rewrites the host. */ @@ -28,15 +31,21 @@ public function verify(WebhookRequest $request): void throw WebhookRejectedException::signatureInvalid($request->provider, "the {$this->header} header is missing"); } - if (str_contains($request->contentType, 'json')) { - parse_str((string) parse_url($request->url, PHP_URL_QUERY), $query); - $bodyHash = $query['bodySHA256'] ?? null; + parse_str((string) parse_url($request->url, PHP_URL_QUERY), $query); + $contentType = strtolower(trim(explode(';', $request->contentType)[0])); + + if (array_key_exists('bodySHA256', $query)) { + $bodyHash = $query['bodySHA256']; if (! is_string($bodyHash) || ! hash_equals(hash('sha256', $request->body), strtolower($bodyHash))) { throw WebhookRejectedException::signatureInvalid($request->provider, 'the bodySHA256 parameter does not match the body'); } $signed = $request->url; - } else { + } elseif ($contentType === 'application/x-www-form-urlencoded') { $signed = self::signedForm($request->url, $request->form); + } elseif ($request->body === '') { + $signed = $request->url; + } else { + throw WebhookRejectedException::signatureInvalid($request->provider, 'a body that is neither form-encoded nor covered by bodySHA256 is not signed'); } if (! hash_equals(base64_encode(hash_hmac('sha1', $signed, $this->authToken, true)), $signature)) { diff --git a/packages/webhooks/tests/Signature/SignatureValidatorsTest.php b/packages/webhooks/tests/Signature/SignatureValidatorsTest.php index e55e342f..ce87bd31 100644 --- a/packages/webhooks/tests/Signature/SignatureValidatorsTest.php +++ b/packages/webhooks/tests/Signature/SignatureValidatorsTest.php @@ -123,6 +123,35 @@ function w4Rejection(Closure $verify): WebhookRejectedException expect(w4Rejection(fn () => $validator->verify(WebhookRequests::make('twilio', $body.'x', ['X-Twilio-Signature' => $signature], [], $url)))->getMessage())->toContain('bodySHA256'); }); +it('checks bodySHA256 whatever the content type, so a replayed JSON signature cannot carry a forged body', function () { + $body = '{"EventType":"onMessageAdded"}'; + $url = 'https://hooks.example.com/webhooks/twilio?bodySHA256='.hash('sha256', $body); + $signature = base64_encode(hash_hmac('sha1', $url, 'token', true)); + $validator = new TwilioSignatureValidator('token'); + + $validator->verify(WebhookRequests::make('twilio', $body, ['X-Twilio-Signature' => $signature], [], $url, '')); + + foreach (['', 'text/plain', 'application/octet-stream', 'application/x-www-form-urlencoded'] as $contentType) { + $forged = WebhookRequests::make('twilio', '{"EventType":"forged"}', ['X-Twilio-Signature' => $signature], [], $url, $contentType); + + expect(w4Rejection(fn () => $validator->verify($forged))->getMessage())->toContain('bodySHA256'); + } +}); + +it('rejects a Twilio request whose body is neither form-encoded nor covered by bodySHA256', function () { + $url = 'https://hooks.example.com/webhooks/twilio'; + $signature = base64_encode(hash_hmac('sha1', $url, 'token', true)); + $validator = new TwilioSignatureValidator('token'); + + $validator->verify(WebhookRequests::make('twilio', '', ['X-Twilio-Signature' => $signature], [], $url, '')); + + foreach (['', 'text/plain', 'application/json'] as $contentType) { + $request = WebhookRequests::make('twilio', '{"EventType":"forged"}', ['X-Twilio-Signature' => $signature], [], $url, $contentType); + + expect(w4Rejection(fn () => $validator->verify($request))->errorCode())->toBe('WEBHOOK_SIGNATURE_INVALID'); + } +}); + it('accepts anything with the NoOp validator, and the factory picks the validator for each kind', function () { (new NoOpSignatureValidator)->verify(WebhookRequests::make('open', 'anything')); $factory = new SignatureValidatorFactory(w4Clock()); From 26ee8fe0a9ebc249ca952a98fb8886b581b00d8e Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 20:31:09 -0700 Subject: [PATCH 33/47] fix(webhooks): parse Twilio form parameters from the raw signed body, so TrimStrings and ConvertEmptyStringsToNull cannot break the signature --- packages/webhooks/src/WebhookRequest.php | 37 +++++++++++++--- .../webhooks/tests/WebhookRequestTest.php | 42 +++++++++++++++++++ 2 files changed, 73 insertions(+), 6 deletions(-) diff --git a/packages/webhooks/src/WebhookRequest.php b/packages/webhooks/src/WebhookRequest.php index 70c7af51..968b5be8 100644 --- a/packages/webhooks/src/WebhookRequest.php +++ b/packages/webhooks/src/WebhookRequest.php @@ -11,7 +11,11 @@ * cover. The URL is the EXACT one the sender signed: scheme + host + the raw request URI (Laravel's fullUrl() sorts * the query string, which breaks Twilio's signature), or the provider's configured public-url plus the query when a * proxy rewrites the host. Header names are lower-cased; a repeated header keeps its first value. The form - * parameters are only read for application/x-www-form-urlencoded bodies. + * parameters are only read for application/x-www-form-urlencoded bodies, and they are parsed from the raw body the + * sender signed — never from Laravel's request bag, which the global TrimStrings and ConvertEmptyStringsToNull + * middleware have already rewritten (Twilio signs `Body=Hello ` with its trailing space) and which keeps only the last + * value of a repeated name. Names are kept verbatim (no PHP `a.b` => `a_b` or `a[]` mangling); a name sent once maps + * to its string, a repeated name to the list of its values in arrival order. */ final readonly class WebhookRequest { @@ -52,13 +56,34 @@ public static function fromHttp(Request $request, string $provider, ?string $pub $contentType = strtolower(trim(explode(';', (string) $request->headers->get('Content-Type', ''))[0])); - $form = []; - if ($contentType === 'application/x-www-form-urlencoded') { - foreach ($request->request->all() as $name => $value) { - $form[(string) $name] = $value; + $body = (string) $request->getContent(); + $form = $contentType === 'application/x-www-form-urlencoded' ? self::parseForm($body) : []; + + return new self($provider, $request->getMethod(), $url, $headers, $body, $form, $contentType, $now, $request->ip()); + } + + /** + * Decodes an application/x-www-form-urlencoded body exactly as sent: `+` and percent escapes are decoded, + * nothing is trimmed or nulled, and every value of a repeated name is kept. + * + * @return array> + */ + public static function parseForm(string $body): array + { + $values = []; + foreach (explode('&', $body) as $pair) { + if ($pair === '') { + continue; } + $parts = explode('=', $pair, 2); + $values[urldecode($parts[0])][] = urldecode($parts[1] ?? ''); + } + + $form = []; + foreach ($values as $name => $list) { + $form[(string) $name] = count($list) === 1 ? $list[0] : $list; } - return new self($provider, $request->getMethod(), $url, $headers, (string) $request->getContent(), $form, $contentType, $now, $request->ip()); + return $form; } } diff --git a/packages/webhooks/tests/WebhookRequestTest.php b/packages/webhooks/tests/WebhookRequestTest.php index 700f17f7..0f27842b 100644 --- a/packages/webhooks/tests/WebhookRequestTest.php +++ b/packages/webhooks/tests/WebhookRequestTest.php @@ -2,8 +2,12 @@ declare(strict_types=1); +use Firefly\Webhooks\Signature\TwilioSignatureValidator; use Firefly\Webhooks\WebhookRequest; +use Illuminate\Foundation\Http\Middleware\ConvertEmptyStringsToNull; +use Illuminate\Foundation\Http\Middleware\TrimStrings; use Illuminate\Http\Request; +use Symfony\Component\HttpFoundation\Response; it('captures the exact URL, lower-cased headers, the raw body and form parameters', function () { $request = Request::create('https://api.example.com/webhooks/twilio?b=2&a=1', 'POST', ['To' => '+1'], [], [], [ @@ -28,3 +32,41 @@ expect(WebhookRequest::fromHttp($request, 'twilio', 'https://api.example.com/webhooks/twilio', 0)->url)->toBe('https://api.example.com/webhooks/twilio?x=1'); }); + +it('reads Twilio form parameters from the raw body, so the global TrimStrings and ConvertEmptyStringsToNull middleware cannot break the signature', function () { + $url = 'https://api.example.com/webhooks/twilio'; + $raw = 'Body=Hello+&From=%2B1&MediaUrl=&Tag=b&Tag=a'; + $signed = $url.'BodyHello From+1MediaUrlTagaTagb'; + $request = Request::create($url, 'POST', ['Body' => 'Hello ', 'From' => '+1', 'MediaUrl' => '', 'Tag' => 'a'], [], [], [ + 'HTTP_X_TWILIO_SIGNATURE' => base64_encode(hash_hmac('sha1', $signed, 'token', true)), + 'CONTENT_TYPE' => 'application/x-www-form-urlencoded', + ], $raw); + + $passed = null; + (new TrimStrings)->handle($request, function (Request $trimmed) use (&$passed) { + return (new ConvertEmptyStringsToNull)->handle($trimmed, function (Request $converted) use (&$passed) { + $passed = $converted; + + return new Response; + }); + }); + + if (! $passed instanceof Request) { + throw new RuntimeException('the middleware did not pass the request on'); + } + + expect($passed->request->get('Body'))->toBe('Hello') + ->and($passed->request->get('MediaUrl'))->toBeNull(); + + $webhook = WebhookRequest::fromHttp($passed, 'twilio', null, 0); + + expect($webhook->form)->toBe(['Body' => 'Hello ', 'From' => '+1', 'MediaUrl' => '', 'Tag' => ['b', 'a']]) + ->and(TwilioSignatureValidator::signedForm($url, $webhook->form))->toBe($signed); + + (new TwilioSignatureValidator('token'))->verify($webhook); +}); + +it('keeps form parameter names verbatim and skips empty pairs', function () { + expect(WebhookRequest::parseForm('a.b=1&&c%5B%5D=2&flag'))->toBe(['a.b' => '1', 'c[]' => '2', 'flag' => '']) + ->and(WebhookRequest::parseForm(''))->toBe([]); +}); From 8c061a8ccfda5b3dedb81cd988d28a62193c51a0 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 20:36:30 -0700 Subject: [PATCH 34/47] feat(webhooks): atomic idempotency stores on memory, the cache or a portable receipts table proven on every database --- .../cache/firefly-webhooks-components.php | 13 +++ .../cache/firefly-webhooks-context.php | 19 ++++ ..._create_firefly_webhook_receipts_table.php | 31 +++++++ .../Console/PruneWebhookReceiptsCommand.php | 44 +++++++++ .../CacheWebhookIdempotencyStore.php | 36 ++++++++ .../DatabaseWebhookIdempotencyStore.php | 88 ++++++++++++++++++ .../InMemoryWebhookIdempotencyStore.php | 47 ++++++++++ .../Idempotency/WebhookIdempotencyStore.php | 20 ++++ .../src/Idempotency/WebhookSchema.php | 59 ++++++++++++ .../src/WebhooksAutoConfiguration.php | 37 ++++++++ .../webhooks/src/WebhooksWiringProvider.php | 25 ++++- .../IdempotencyStoreSelectionTest.php | 30 ++++++ .../Idempotency/IdempotencyStoresTest.php | 91 +++++++++++++++++++ ...atabaseIdempotencyStoreIntegrationTest.php | 65 +++++++++++++ .../tests/Support/WebhookDatabases.php | 50 ++++++++++ .../Support/WebhooksCacheStoreTestCase.php | 28 ++++++ .../Support/WebhooksDatabaseTestCase.php | 29 ++++++ .../tests/WebhookReceiptsMigrationTest.php | 35 +++++++ 18 files changed, 746 insertions(+), 1 deletion(-) create mode 100644 packages/webhooks/database/migrations/2026_10_12_000000_create_firefly_webhook_receipts_table.php create mode 100644 packages/webhooks/src/Console/PruneWebhookReceiptsCommand.php create mode 100644 packages/webhooks/src/Idempotency/CacheWebhookIdempotencyStore.php create mode 100644 packages/webhooks/src/Idempotency/DatabaseWebhookIdempotencyStore.php create mode 100644 packages/webhooks/src/Idempotency/InMemoryWebhookIdempotencyStore.php create mode 100644 packages/webhooks/src/Idempotency/WebhookIdempotencyStore.php create mode 100644 packages/webhooks/src/Idempotency/WebhookSchema.php create mode 100644 packages/webhooks/tests/Idempotency/IdempotencyStoreSelectionTest.php create mode 100644 packages/webhooks/tests/Idempotency/IdempotencyStoresTest.php create mode 100644 packages/webhooks/tests/Idempotency/Integration/DatabaseIdempotencyStoreIntegrationTest.php create mode 100644 packages/webhooks/tests/Support/WebhookDatabases.php create mode 100644 packages/webhooks/tests/Support/WebhooksCacheStoreTestCase.php create mode 100644 packages/webhooks/tests/Support/WebhooksDatabaseTestCase.php create mode 100644 packages/webhooks/tests/WebhookReceiptsMigrationTest.php diff --git a/packages/webhooks/cache/firefly-webhooks-components.php b/packages/webhooks/cache/firefly-webhooks-components.php index 33b9d4cb..3bdc8d2a 100644 --- a/packages/webhooks/cache/firefly-webhooks-components.php +++ b/packages/webhooks/cache/firefly-webhooks-components.php @@ -28,6 +28,19 @@ 0 => 'Firefly\\Config\\Config', ], ], + 1 => [ + 'method' => 'webhookIdempotencyStore', + 'returns' => 'Firefly\\Webhooks\\Idempotency\\WebhookIdempotencyStore', + 'name' => null, + 'scope' => 'Singleton', + 'primary' => false, + 'order' => 0, + 'lazy' => false, + 'dependencies' => [ + 0 => 'Firefly\\Webhooks\\WebhooksSettings', + 1 => 'Illuminate\\Container\\Container', + ], + ], ], 'lazy' => false, 'dependencies' => [ diff --git a/packages/webhooks/cache/firefly-webhooks-context.php b/packages/webhooks/cache/firefly-webhooks-context.php index e93b2c9b..2a4c8dab 100644 --- a/packages/webhooks/cache/firefly-webhooks-context.php +++ b/packages/webhooks/cache/firefly-webhooks-context.php @@ -35,6 +35,25 @@ ], ], ], + 1 => [ + 'method' => 'webhookIdempotencyStore', + 'conditions' => [ + 0 => [ + 'type' => 'Firefly\\Context\\Condition\\Attributes\\ConditionalOnProperty', + 'args' => [ + 0 => 'firefly.webhooks.enabled', + 1 => 'true', + 2 => false, + ], + ], + 1 => [ + 'type' => 'Firefly\\Context\\Condition\\Attributes\\ConditionalOnMissingBean', + 'args' => [ + 0 => 'Firefly\\Webhooks\\Idempotency\\WebhookIdempotencyStore', + ], + ], + ], + ], ], ], ]; diff --git a/packages/webhooks/database/migrations/2026_10_12_000000_create_firefly_webhook_receipts_table.php b/packages/webhooks/database/migrations/2026_10_12_000000_create_firefly_webhook_receipts_table.php new file mode 100644 index 00000000..df4895e9 --- /dev/null +++ b/packages/webhooks/database/migrations/2026_10_12_000000_create_firefly_webhook_receipts_table.php @@ -0,0 +1,31 @@ +getConnection())); + } + + public function down(): void + { + WebhookSchema::drop(Schema::connection($this->getConnection())); + } +}; diff --git a/packages/webhooks/src/Console/PruneWebhookReceiptsCommand.php b/packages/webhooks/src/Console/PruneWebhookReceiptsCommand.php new file mode 100644 index 00000000..3c0ffe60 --- /dev/null +++ b/packages/webhooks/src/Console/PruneWebhookReceiptsCommand.php @@ -0,0 +1,44 @@ +bound(WebhookIdempotencyStore::class)) { + $this->error('firefly/webhooks is not enabled (firefly.webhooks.enabled is false).'); + + return self::FAILURE; + } + + $store = $container->make(WebhookIdempotencyStore::class); + if (! $store instanceof DatabaseWebhookIdempotencyStore) { + $this->info('The webhook idempotency store is not the database store; its receipts expire on their own.'); + + return self::SUCCESS; + } + + $this->info('Pruned '.$store->prune().' expired webhook receipt(s).'); + + return self::SUCCESS; + } +} diff --git a/packages/webhooks/src/Idempotency/CacheWebhookIdempotencyStore.php b/packages/webhooks/src/Idempotency/CacheWebhookIdempotencyStore.php new file mode 100644 index 00000000..dfdff7d5 --- /dev/null +++ b/packages/webhooks/src/Idempotency/CacheWebhookIdempotencyStore.php @@ -0,0 +1,36 @@ +cache->add($this->key($provider, $key), $eventId, max(1, (int) ceil($ttlSeconds))); + } + + public function release(string $provider, string $key): void + { + $this->cache->forget($this->key($provider, $key)); + } + + private function key(string $provider, string $key): string + { + return $this->prefix.hash('sha256', $provider."\0".$key); + } +} diff --git a/packages/webhooks/src/Idempotency/DatabaseWebhookIdempotencyStore.php b/packages/webhooks/src/Idempotency/DatabaseWebhookIdempotencyStore.php new file mode 100644 index 00000000..2d939e72 --- /dev/null +++ b/packages/webhooks/src/Idempotency/DatabaseWebhookIdempotencyStore.php @@ -0,0 +1,88 @@ +clock = $clock ?? static fn (): int => time(); + } + + public function claim(string $provider, string $key, string $eventId, float $ttlSeconds): bool + { + $now = ($this->clock)(); + $row = [ + 'provider' => $provider, + 'idempotency_key' => $key, + 'event_id' => $eventId, + 'claimed_at' => $now, + 'expires_at' => $now + max(1, (int) ceil($ttlSeconds)), + ]; + + if ($this->insert($row)) { + return true; + } + + $expired = $this->connection->table(WebhookSchema::TABLE) + ->where('provider', $provider) + ->where('idempotency_key', $key) + ->where('expires_at', '<=', $now) + ->delete(); + + return $expired > 0 && $this->insert($row); + } + + public function release(string $provider, string $key): void + { + $this->connection->table(WebhookSchema::TABLE)->where('provider', $provider)->where('idempotency_key', $key)->delete(); + } + + /** Deletes every expired receipt; what firefly:webhooks:prune runs. */ + public function prune(): int + { + return $this->connection->table(WebhookSchema::TABLE)->where('expires_at', '<=', ($this->clock)())->delete(); + } + + /** + * One INSERT attempt: true when the row went in, false when the primary key already holds a receipt. Impure by + * definition (it writes, and its answer depends on the table), so a second attempt after the expired-row delete + * is a fresh question, not a repeat of the first. + * + * @param array $row + * + * @phpstan-impure + */ + private function insert(array $row): bool + { + try { + $this->connection->transaction(fn (): bool => $this->connection->table(WebhookSchema::TABLE)->insert($row)); + + return true; + } catch (UniqueConstraintViolationException) { + return false; + } + } +} diff --git a/packages/webhooks/src/Idempotency/InMemoryWebhookIdempotencyStore.php b/packages/webhooks/src/Idempotency/InMemoryWebhookIdempotencyStore.php new file mode 100644 index 00000000..7b49bacd --- /dev/null +++ b/packages/webhooks/src/Idempotency/InMemoryWebhookIdempotencyStore.php @@ -0,0 +1,47 @@ + claim id => expires at */ + private array $expiries = []; + + /** @var Closure(): int */ + private readonly Closure $clock; + + /** + * @param (Closure(): int)|null $clock unix seconds + */ + public function __construct(?Closure $clock = null) + { + $this->clock = $clock ?? static fn (): int => time(); + } + + public function claim(string $provider, string $key, string $eventId, float $ttlSeconds): bool + { + $id = $provider."\0".$key; + $now = ($this->clock)(); + + if (isset($this->expiries[$id]) && $this->expiries[$id] > $now) { + return false; + } + + $this->expiries[$id] = $now + (int) ceil($ttlSeconds); + + return true; + } + + public function release(string $provider, string $key): void + { + unset($this->expiries[$provider."\0".$key]); + } +} diff --git a/packages/webhooks/src/Idempotency/WebhookIdempotencyStore.php b/packages/webhooks/src/Idempotency/WebhookIdempotencyStore.php new file mode 100644 index 00000000..83f48e99 --- /dev/null +++ b/packages/webhooks/src/Idempotency/WebhookIdempotencyStore.php @@ -0,0 +1,20 @@ +hasTable(self::TABLE)) { + return; + } + + $connection = $schema->getConnection(); + $schema->create(self::TABLE, static function (Blueprint $table) use ($connection): void { + self::binary($table->string('provider', 64), $connection); + self::binary($table->string('idempotency_key', 191), $connection); + $table->string('event_id', 36); + $table->bigInteger('claimed_at'); + $table->bigInteger('expires_at')->index('firefly_webhook_receipts_expires'); + $table->primary(['provider', 'idempotency_key'], 'firefly_webhook_receipts_pk'); + }); + } + + public static function drop(Builder $schema): void + { + $schema->dropIfExists(self::TABLE); + } + + public static function binary(ColumnDefinition $column, Connection $connection): void + { + if ($connection instanceof MySqlConnection) { + $modern = version_compare($connection->getServerVersion(), $connection->isMaria() ? '10.2.2' : '8.0.1', '>='); + $column->charset('utf8mb4')->collation($modern ? ($connection->isMaria() ? 'utf8mb4_nopad_bin' : 'utf8mb4_0900_bin') : 'utf8mb4_bin'); + } elseif ($connection instanceof SqlServerConnection) { + $column->collation('Latin1_General_100_BIN2'); + } + } +} diff --git a/packages/webhooks/src/WebhooksAutoConfiguration.php b/packages/webhooks/src/WebhooksAutoConfiguration.php index 3ac1c8f9..09969d70 100644 --- a/packages/webhooks/src/WebhooksAutoConfiguration.php +++ b/packages/webhooks/src/WebhooksAutoConfiguration.php @@ -10,6 +10,14 @@ use Firefly\Container\Attributes\Order; use Firefly\Context\Condition\Attributes\ConditionalOnMissingBean; use Firefly\Context\Condition\Attributes\ConditionalOnProperty; +use Firefly\Kernel\Exception\Framework\ConfigurationException; +use Firefly\Webhooks\Idempotency\CacheWebhookIdempotencyStore; +use Firefly\Webhooks\Idempotency\DatabaseWebhookIdempotencyStore; +use Firefly\Webhooks\Idempotency\InMemoryWebhookIdempotencyStore; +use Firefly\Webhooks\Idempotency\WebhookIdempotencyStore; +use Illuminate\Container\Container; +use Illuminate\Contracts\Cache\Factory as CacheFactory; +use Illuminate\Database\ConnectionResolverInterface; /** * The webhooks beans, every one behind firefly.webhooks.enabled (default false): installing the package is inert @@ -27,4 +35,33 @@ public function webhooksSettings(Config $config): WebhooksSettings { return WebhooksSettings::fromConfig($config); } + + /** + * firefly.webhooks.idempotency.store: `cache` (the default, on firefly.webhooks.idempotency.cache-store or the + * default store), `database` (firefly_webhook_receipts on firefly.webhooks.idempotency.connection or the + * default connection), or `memory` (one process only). + */ + #[Bean] + #[ConditionalOnProperty(name: 'firefly.webhooks.enabled', havingValue: 'true')] + #[ConditionalOnMissingBean(WebhookIdempotencyStore::class)] + public function webhookIdempotencyStore(WebhooksSettings $settings, Container $container): WebhookIdempotencyStore + { + if ($settings->idempotencyStore === 'memory') { + return new InMemoryWebhookIdempotencyStore; + } + + if ($settings->idempotencyStore === 'database') { + $databases = $container->bound('db') ? $container->make('db') : null; + if (! $databases instanceof ConnectionResolverInterface) { + throw new ConfigurationException('firefly.webhooks.idempotency.store is database but no database connection resolver (db) is bound.'); + } + + return new DatabaseWebhookIdempotencyStore($databases->connection($settings->idempotencyConnection)); + } + + /** @var CacheFactory $caches */ + $caches = $container->make('cache'); + + return new CacheWebhookIdempotencyStore($caches->store($settings->idempotencyCacheStore === '' ? null : $settings->idempotencyCacheStore)); + } } diff --git a/packages/webhooks/src/WebhooksWiringProvider.php b/packages/webhooks/src/WebhooksWiringProvider.php index d9acb61b..37c8eb9d 100644 --- a/packages/webhooks/src/WebhooksWiringProvider.php +++ b/packages/webhooks/src/WebhooksWiringProvider.php @@ -4,12 +4,20 @@ namespace Firefly\Webhooks; +use Firefly\Config\Config; use Firefly\Context\Boot\BootPass; use Firefly\Context\Boot\FireflyServiceProvider; +use Firefly\Webhooks\Console\PruneWebhookReceiptsCommand; -/** The boot-pass half of firefly/webhooks: the endpoint registrar, the receipts migration and the prune command. */ +/** + * The boot-pass half of firefly/webhooks: the endpoint registrar, the receipts migration (publishable with + * --tag=firefly-webhooks-migrations, and loaded automatically only when webhooks are enabled on the database + * store — an application that never chose it gets no table) and firefly:webhooks:prune. + */ final class WebhooksWiringProvider extends FireflyServiceProvider { + public const string MIGRATION = '2026_10_12_000000_create_firefly_webhook_receipts_table.php'; + /** * @return list */ @@ -17,4 +25,19 @@ public function passes(): array { return []; } + + public function boot(): void + { + if ($this->app->runningInConsole()) { + $this->commands([PruneWebhookReceiptsCommand::class]); + } + + $migrations = dirname(__DIR__).'/database/migrations'; + $this->publishes([$migrations.'/'.self::MIGRATION => $this->app->databasePath('migrations/'.self::MIGRATION)], 'firefly-webhooks-migrations'); + + $config = new Config($this->app->make('config')); + if ($config->bool('firefly.webhooks.enabled', false) && $config->string('firefly.webhooks.idempotency.store', 'cache') === 'database') { + $this->loadMigrationsFrom($migrations); + } + } } diff --git a/packages/webhooks/tests/Idempotency/IdempotencyStoreSelectionTest.php b/packages/webhooks/tests/Idempotency/IdempotencyStoreSelectionTest.php new file mode 100644 index 00000000..5e726bdb --- /dev/null +++ b/packages/webhooks/tests/Idempotency/IdempotencyStoreSelectionTest.php @@ -0,0 +1,30 @@ +toBe(0); + + $store = $this->app()->make(WebhookIdempotencyStore::class); + + expect($store)->toBeInstanceOf(CacheWebhookIdempotencyStore::class) + ->and(Schema::hasTable(WebhookSchema::TABLE))->toBeFalse() + ->and($store->claim('acme', 'evt_1', 'e-1', 60))->toBeTrue() + ->and($store->claim('acme', 'evt_1', 'e-2', 60))->toBeFalse(); +}); + +it('answers firefly:webhooks:prune without touching anything on the cache store', function () { + /** @var WebhooksCacheStoreTestCase $this */ + expect(Artisan::call('firefly:webhooks:prune'))->toBe(0) + ->and(Artisan::output())->toContain('not the database store'); +}); diff --git a/packages/webhooks/tests/Idempotency/IdempotencyStoresTest.php b/packages/webhooks/tests/Idempotency/IdempotencyStoresTest.php new file mode 100644 index 00000000..ab7a962c --- /dev/null +++ b/packages/webhooks/tests/Idempotency/IdempotencyStoresTest.php @@ -0,0 +1,91 @@ +addConnection(['driver' => 'sqlite', 'database' => ':memory:', 'prefix' => '']); + $connection = $capsule->getConnection(); + WebhookSchema::create($connection->getSchemaBuilder()); + + return $connection; +} + +dataset('w4 idempotency stores', [ + 'memory' => [static fn (): WebhookIdempotencyStore => new InMemoryWebhookIdempotencyStore], + 'cache' => [static fn (): WebhookIdempotencyStore => new CacheWebhookIdempotencyStore(new Repository(new ArrayStore))], + 'database (sqlite)' => [static fn (): WebhookIdempotencyStore => new DatabaseWebhookIdempotencyStore(w4SqliteReceipts())], +]); + +it('claims a delivery once, per provider, and lets a released claim be taken again', function (Closure $make) { + /** @var WebhookIdempotencyStore $store */ + $store = $make(); + + expect($store->claim('stripe', 'evt_1', 'e-1', 3600))->toBeTrue() + ->and($store->claim('stripe', 'evt_1', 'e-2', 3600))->toBeFalse() + ->and($store->claim('github', 'evt_1', 'e-3', 3600))->toBeTrue(); + + $store->release('stripe', 'evt_1'); + + expect($store->claim('stripe', 'evt_1', 'e-4', 3600))->toBeTrue(); +})->with('w4 idempotency stores'); + +it('treats keys that differ only in case as different deliveries', function (Closure $make) { + /** @var WebhookIdempotencyStore $store */ + $store = $make(); + + expect($store->claim('p', 'Key', 'e-1', 60))->toBeTrue() + ->and($store->claim('p', 'key', 'e-2', 60))->toBeTrue(); +})->with('w4 idempotency stores'); + +it('forgets a claim once its TTL is over (memory and database)', function () { + $now = 1_000; + $clock = static function () use (&$now): int { + return $now; + }; + $connection = w4SqliteReceipts(); + + foreach ([new InMemoryWebhookIdempotencyStore($clock), new DatabaseWebhookIdempotencyStore($connection, $clock)] as $store) { + $now = 1_000; + expect($store->claim('p', 'k', 'e-1', 60))->toBeTrue(); + $now = 1_059; + expect($store->claim('p', 'k', 'e-2', 60))->toBeFalse(); + $now = 1_061; + expect($store->claim('p', 'k', 'e-3', 60))->toBeTrue(); + } + + expect($connection->table(WebhookSchema::TABLE)->where('idempotency_key', 'k')->value('event_id'))->toBe('e-3'); +}); + +it('prunes expired receipts from the table and keeps live ones', function () { + $now = 5_000; + $clock = static function () use (&$now): int { + return $now; + }; + $connection = w4SqliteReceipts(); + $store = new DatabaseWebhookIdempotencyStore($connection, $clock); + $store->claim('p', 'old', 'e-1', 10); + $store->claim('p', 'live', 'e-2', 1000); + $now = 5_100; + + expect($store->prune())->toBe(1) + ->and($connection->table(WebhookSchema::TABLE)->pluck('idempotency_key')->all())->toBe(['live']); +}); + +it('creates the table idempotently', function () { + $connection = w4SqliteReceipts(); + WebhookSchema::create($connection->getSchemaBuilder()); + + expect($connection->getSchemaBuilder()->hasTable(WebhookSchema::TABLE))->toBeTrue(); +}); diff --git a/packages/webhooks/tests/Idempotency/Integration/DatabaseIdempotencyStoreIntegrationTest.php b/packages/webhooks/tests/Idempotency/Integration/DatabaseIdempotencyStoreIntegrationTest.php new file mode 100644 index 00000000..3ceac1a2 --- /dev/null +++ b/packages/webhooks/tests/Idempotency/Integration/DatabaseIdempotencyStoreIntegrationTest.php @@ -0,0 +1,65 @@ + ['pgsql', 'FIREFLY_PG_DSN'], + 'MySQL' => ['mysql', 'FIREFLY_MYSQL_DSN'], + 'MariaDB' => ['mysql', 'FIREFLY_MARIADB_DSN'], + 'SQL Server' => ['sqlsrv', 'FIREFLY_SQLSRV_DSN'], +] as $backend => [$driver, $variable]) { + it("claims once, case-sensitively, reclaims an expired receipt and prunes on {$backend}", function () use ($driver, $variable): void { + $connection = WebhookDatabases::connection($driver, $variable); + WebhookSchema::drop($connection->getSchemaBuilder()); + WebhookSchema::create($connection->getSchemaBuilder()); + WebhookSchema::create($connection->getSchemaBuilder()); + $now = 10_000; + $store = new DatabaseWebhookIdempotencyStore($connection, static function () use (&$now): int { + return $now; + }); + + try { + expect($store->claim('stripe', 'evt_Abc', 'e-1', 60))->toBeTrue() + ->and($store->claim('stripe', 'evt_Abc', 'e-2', 60))->toBeFalse() + ->and($store->claim('stripe', 'evt_abc', 'e-3', 60))->toBeTrue() + ->and($store->claim('github', 'evt_Abc', 'e-4', 60))->toBeTrue(); + + $now = 10_061; + expect($store->claim('stripe', 'evt_Abc', 'e-5', 60))->toBeTrue() + ->and($connection->table(WebhookSchema::TABLE)->where('provider', 'stripe')->where('idempotency_key', 'evt_Abc')->value('event_id'))->toBe('e-5') + ->and($store->prune())->toBe(2); + + $store->release('stripe', 'evt_Abc'); + expect($store->claim('stripe', 'evt_Abc', 'e-6', 60))->toBeTrue(); + } finally { + WebhookSchema::drop($connection->getSchemaBuilder()); + } + })->group('integration')->skip(getenv($variable) === false, "Set {$variable} to an isolated database."); + + it("answers a duplicate inside an open transaction without aborting it on {$backend}", function () use ($driver, $variable): void { + $connection = WebhookDatabases::connection($driver, $variable); + WebhookSchema::drop($connection->getSchemaBuilder()); + WebhookSchema::create($connection->getSchemaBuilder()); + $store = new DatabaseWebhookIdempotencyStore($connection); + + try { + $store->claim('p', 'k', 'e-1', 60); + $connection->beginTransaction(); + expect($store->claim('p', 'k', 'e-2', 60))->toBeFalse() + ->and($connection->transactionLevel())->toBe(1) + ->and($store->claim('p', 'k2', 'e-3', 60))->toBeTrue(); + $connection->commit(); + + expect($connection->table(WebhookSchema::TABLE)->where('idempotency_key', 'k2')->exists())->toBeTrue(); + } finally { + if ($connection->transactionLevel() > 0) { + $connection->rollBack(0); + } + WebhookSchema::drop($connection->getSchemaBuilder()); + } + })->group('integration')->skip(getenv($variable) === false, "Set {$variable} to an isolated database."); +} diff --git a/packages/webhooks/tests/Support/WebhookDatabases.php b/packages/webhooks/tests/Support/WebhookDatabases.php new file mode 100644 index 00000000..4e26632b --- /dev/null +++ b/packages/webhooks/tests/Support/WebhookDatabases.php @@ -0,0 +1,50 @@ +addConnection([ + 'driver' => $driver, + 'host' => $parts['host'] ?? '127.0.0.1', + 'port' => (int) ($parts['port'] ?? 0), + 'database' => $parts['dbname'] ?? '', + 'username' => $parts['user'] ?? '', + 'password' => $parts['password'] ?? '', + 'charset' => match ($driver) { + 'pgsql', 'sqlsrv' => 'utf8', + default => 'utf8mb4', + }, + 'prefix' => '', + 'schema' => 'public', + 'trust_server_certificate' => true, + ]); + + return $capsule->getConnection(); + } +} diff --git a/packages/webhooks/tests/Support/WebhooksCacheStoreTestCase.php b/packages/webhooks/tests/Support/WebhooksCacheStoreTestCase.php new file mode 100644 index 00000000..b54233a3 --- /dev/null +++ b/packages/webhooks/tests/Support/WebhooksCacheStoreTestCase.php @@ -0,0 +1,28 @@ + */ + protected function fireflyProviders(): array + { + return [WebhooksServiceProvider::class, WebhooksWiringProvider::class]; + } + + /** @return array */ + protected function configOverrides(): array + { + return [ + 'firefly.webhooks.enabled' => true, + 'firefly.webhooks.providers' => ['acme' => ['validator' => 'hmac', 'secret' => 'acme-secret']], + ]; + } +} diff --git a/packages/webhooks/tests/Support/WebhooksDatabaseTestCase.php b/packages/webhooks/tests/Support/WebhooksDatabaseTestCase.php new file mode 100644 index 00000000..13adb651 --- /dev/null +++ b/packages/webhooks/tests/Support/WebhooksDatabaseTestCase.php @@ -0,0 +1,29 @@ + */ + protected function fireflyProviders(): array + { + return [WebhooksServiceProvider::class, WebhooksWiringProvider::class]; + } + + /** @return array */ + protected function configOverrides(): array + { + return [ + 'firefly.webhooks.enabled' => true, + 'firefly.webhooks.idempotency.store' => 'database', + 'firefly.webhooks.providers' => ['acme' => ['validator' => 'hmac', 'secret' => 'acme-secret']], + ]; + } +} diff --git a/packages/webhooks/tests/WebhookReceiptsMigrationTest.php b/packages/webhooks/tests/WebhookReceiptsMigrationTest.php new file mode 100644 index 00000000..9897fad3 --- /dev/null +++ b/packages/webhooks/tests/WebhookReceiptsMigrationTest.php @@ -0,0 +1,35 @@ +toBe(0); + + expect($this->app()->make(WebhookIdempotencyStore::class))->toBeInstanceOf(DatabaseWebhookIdempotencyStore::class) + ->and(Schema::hasTable(WebhookSchema::TABLE))->toBeTrue(); +}); + +it('prunes expired receipts with php artisan firefly:webhooks:prune', function () { + /** @var WebhooksDatabaseTestCase $this */ + expect(Artisan::call('migrate'))->toBe(0); + DB::table(WebhookSchema::TABLE)->insert([ + ['provider' => 'acme', 'idempotency_key' => 'old', 'event_id' => 'e-1', 'claimed_at' => 1, 'expires_at' => 2], + ['provider' => 'acme', 'idempotency_key' => 'live', 'event_id' => 'e-2', 'claimed_at' => time(), 'expires_at' => time() + 3600], + ]); + + expect(Artisan::call('firefly:webhooks:prune'))->toBe(0) + ->and(Artisan::output())->toContain('Pruned 1 expired webhook receipt'); + + expect(DB::table(WebhookSchema::TABLE)->pluck('idempotency_key')->all())->toBe(['live']); +}); From 64ccb8b9cb95469be5dcf467a81b20f2ed0cb3bf Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 20:42:00 -0700 Subject: [PATCH 35/47] feat(webhooks): an endpoint per provider that verifies, deduplicates and publishes webhooks to the event bus, refusing with problem+json --- .../cache/firefly-webhooks-components.php | 27 +++ .../cache/firefly-webhooks-context.php | 38 ++++ .../webhooks/src/Http/WebhookController.php | 46 +++++ .../src/Http/WebhookRouteRegistrar.php | 54 +++++ packages/webhooks/src/WebhookProcessor.php | 188 ++++++++++++++++++ packages/webhooks/src/WebhookResult.php | 48 +++++ .../src/WebhooksAutoConfiguration.php | 25 +++ .../webhooks/src/WebhooksWiringProvider.php | 3 +- .../tests/CapstoneWebhooksDisabledTest.php | 16 ++ .../CapstoneWebhooksPublishFailureTest.php | 17 ++ .../webhooks/tests/CapstoneWebhooksTest.php | 95 +++++++++ ...ilingPublisherWebhooksCapstoneTestCase.php | 29 +++ .../Support/WebhooksCacheStoreTestCase.php | 5 +- .../Support/WebhooksCapstoneTestCase.php | 78 ++++++++ .../Support/WebhooksDatabaseTestCase.php | 5 +- .../WebhooksDisabledCapstoneTestCase.php | 14 ++ .../webhooks/tests/WebhookProcessorTest.php | 101 ++++++++++ 17 files changed, 784 insertions(+), 5 deletions(-) create mode 100644 packages/webhooks/src/Http/WebhookController.php create mode 100644 packages/webhooks/src/Http/WebhookRouteRegistrar.php create mode 100644 packages/webhooks/src/WebhookProcessor.php create mode 100644 packages/webhooks/src/WebhookResult.php create mode 100644 packages/webhooks/tests/CapstoneWebhooksDisabledTest.php create mode 100644 packages/webhooks/tests/CapstoneWebhooksPublishFailureTest.php create mode 100644 packages/webhooks/tests/CapstoneWebhooksTest.php create mode 100644 packages/webhooks/tests/Support/FailingPublisherWebhooksCapstoneTestCase.php create mode 100644 packages/webhooks/tests/Support/WebhooksCapstoneTestCase.php create mode 100644 packages/webhooks/tests/Support/WebhooksDisabledCapstoneTestCase.php create mode 100644 packages/webhooks/tests/WebhookProcessorTest.php diff --git a/packages/webhooks/cache/firefly-webhooks-components.php b/packages/webhooks/cache/firefly-webhooks-components.php index 3bdc8d2a..50e88665 100644 --- a/packages/webhooks/cache/firefly-webhooks-components.php +++ b/packages/webhooks/cache/firefly-webhooks-components.php @@ -41,6 +41,33 @@ 1 => 'Illuminate\\Container\\Container', ], ], + 2 => [ + 'method' => 'signatureValidatorFactory', + 'returns' => 'Firefly\\Webhooks\\Signature\\SignatureValidatorFactory', + 'name' => null, + 'scope' => 'Singleton', + 'primary' => false, + 'order' => 0, + 'lazy' => false, + 'dependencies' => [ + ], + ], + 3 => [ + 'method' => 'webhookProcessor', + 'returns' => 'Firefly\\Webhooks\\WebhookProcessor', + 'name' => null, + 'scope' => 'Singleton', + 'primary' => false, + 'order' => 0, + 'lazy' => false, + 'dependencies' => [ + 0 => 'Firefly\\Webhooks\\WebhooksSettings', + 1 => 'Firefly\\Webhooks\\Signature\\SignatureValidatorFactory', + 2 => 'Firefly\\Webhooks\\Idempotency\\WebhookIdempotencyStore', + 3 => 'Firefly\\Eda\\EventPublisher', + 4 => 'Illuminate\\Container\\Container', + ], + ], ], 'lazy' => false, 'dependencies' => [ diff --git a/packages/webhooks/cache/firefly-webhooks-context.php b/packages/webhooks/cache/firefly-webhooks-context.php index 2a4c8dab..1232a88b 100644 --- a/packages/webhooks/cache/firefly-webhooks-context.php +++ b/packages/webhooks/cache/firefly-webhooks-context.php @@ -54,6 +54,44 @@ ], ], ], + 2 => [ + 'method' => 'signatureValidatorFactory', + 'conditions' => [ + 0 => [ + 'type' => 'Firefly\\Context\\Condition\\Attributes\\ConditionalOnProperty', + 'args' => [ + 0 => 'firefly.webhooks.enabled', + 1 => 'true', + 2 => false, + ], + ], + 1 => [ + 'type' => 'Firefly\\Context\\Condition\\Attributes\\ConditionalOnMissingBean', + 'args' => [ + 0 => 'Firefly\\Webhooks\\Signature\\SignatureValidatorFactory', + ], + ], + ], + ], + 3 => [ + 'method' => 'webhookProcessor', + 'conditions' => [ + 0 => [ + 'type' => 'Firefly\\Context\\Condition\\Attributes\\ConditionalOnProperty', + 'args' => [ + 0 => 'firefly.webhooks.enabled', + 1 => 'true', + 2 => false, + ], + ], + 1 => [ + 'type' => 'Firefly\\Context\\Condition\\Attributes\\ConditionalOnMissingBean', + 'args' => [ + 0 => 'Firefly\\Webhooks\\WebhookProcessor', + ], + ], + ], + ], ], ], ]; diff --git a/packages/webhooks/src/Http/WebhookController.php b/packages/webhooks/src/Http/WebhookController.php new file mode 100644 index 00000000..40cd363c --- /dev/null +++ b/packages/webhooks/src/Http/WebhookController.php @@ -0,0 +1,46 @@ +processor->provider($provider); + $result = $this->processor->process(WebhookRequest::fromHttp($request, $provider, $settings->publicUrl, time())); + + return new JsonResponse($result->toArray(), $result->httpStatus()); + } catch (Throwable $e) { + if (! $e instanceof FireflyException || $e->httpStatus() >= 500) { + $this->exceptions->report($e); + } + + return $this->problems->render($e, $request); + } + } +} diff --git a/packages/webhooks/src/Http/WebhookRouteRegistrar.php b/packages/webhooks/src/Http/WebhookRouteRegistrar.php new file mode 100644 index 00000000..f909f0d1 --- /dev/null +++ b/packages/webhooks/src/Http/WebhookRouteRegistrar.php @@ -0,0 +1,54 @@ +container; + if (! $container->bound(WebhooksSettings::class)) { + return; + } + + /** @var WebhooksSettings $settings */ + $settings = $container->make(WebhooksSettings::class); + if (! $settings->enabled) { + return; + } + + /** @var Router $router */ + $router = $container->make('router'); + $router->post($settings->path.'/{provider}', static fn (Request $request, string $provider): Response => $container->make(WebhookController::class)($request, $provider)) + ->where('provider', '[A-Za-z0-9][A-Za-z0-9._-]*') + ->name(self::ROUTE_NAME); + } +} diff --git a/packages/webhooks/src/WebhookProcessor.php b/packages/webhooks/src/WebhookProcessor.php new file mode 100644 index 00000000..a602cd19 --- /dev/null +++ b/packages/webhooks/src/WebhookProcessor.php @@ -0,0 +1,188 @@ +settings->provider($id) ?? throw WebhookRejectedException::unknownProvider($id); + } + + public function process(WebhookRequest $request): WebhookResult + { + $provider = $this->provider($request->provider); + + if (strlen($request->body) > $this->settings->maxBodyBytes) { + throw WebhookRejectedException::tooLarge($provider->id, $this->settings->maxBodyBytes); + } + + $this->validators->for($provider)->verify($request); + + $body = $this->decode($request, $provider); + $type = $this->eventTypeOf($request, $provider, $body); + $key = $this->idempotencyKeyOf($request, $provider, $body); + $eventId = self::uuid4(); + + if ($key !== null && ! $this->idempotency->claim($provider->id, $key, $eventId, $this->settings->idempotencyTtl)) { + $this->logger?->info("Duplicate [{$provider->id}] webhook {$key} ignored."); + + return WebhookResult::duplicate($provider->id, $key); + } + + $replacements = ['{provider}' => $provider->id, '{type}' => $type]; + $eventType = strtr($this->settings->eventType, $replacements); + $destination = strtr($provider->destination ?? $this->settings->destination, $replacements); + + try { + $this->publisher->publish($destination, $eventType, [ + 'id' => $eventId, + 'provider' => $provider->id, + 'type' => $type, + 'receivedAt' => gmdate(DATE_ATOM, $request->receivedAt), + 'idempotencyKey' => $key, + 'headers' => self::publishableHeaders($request->headers), + 'body' => $body, + ], ['x-webhook-provider' => $provider->id, 'x-webhook-event-id' => $eventId]); + } catch (Throwable $e) { + if ($key !== null) { + $this->idempotency->release($provider->id, $key); + } + + throw new ServiceUnavailableException("The [{$provider->id}] webhook was verified but could not be published; the delivery should be retried.", 'WEBHOOK_PUBLISH_FAILED', $e); + } + + return WebhookResult::accepted($eventId, $provider->id, $eventType, $destination, $key); + } + + /** + * @return array + */ + private function decode(WebhookRequest $request, WebhookProviderSettings $provider): array + { + if ($request->body === '') { + return []; + } + + if ($request->contentType === 'application/x-www-form-urlencoded') { + return $request->form; + } + + $looksJson = str_contains($request->contentType, 'json') || ($request->contentType === '' && str_starts_with(ltrim($request->body), '{')); + if (! $looksJson) { + return mb_check_encoding($request->body, 'UTF-8') ? ['raw' => $request->body] : ['rawBase64' => base64_encode($request->body)]; + } + + try { + $decoded = json_decode($request->body, true, 512, JSON_THROW_ON_ERROR); + } catch (JsonException $e) { + throw WebhookRejectedException::invalidPayload($provider->id, 'the JSON does not parse ('.$e->getMessage().')'); + } + + if (! is_array($decoded) || ($decoded !== [] && array_is_list($decoded))) { + throw WebhookRejectedException::invalidPayload($provider->id, 'a JSON webhook body must be an object'); + } + + return $decoded; + } + + /** + * @param array $body + */ + private function eventTypeOf(WebhookRequest $request, WebhookProviderSettings $provider, array $body): string + { + $type = $provider->eventTypeHeader !== '' ? $request->header($provider->eventTypeHeader) : null; + $type ??= $provider->eventTypeField !== '' ? self::scalarAt($body, $provider->eventTypeField) : null; + + $type = preg_replace('/[^A-Za-z0-9._:\-]/', '_', (string) ($type ?? 'unknown')); + + return $type === null || $type === '' ? 'unknown' : substr($type, 0, 191); + } + + /** + * @param array $body + */ + private function idempotencyKeyOf(WebhookRequest $request, WebhookProviderSettings $provider, array $body): ?string + { + $key = $provider->idempotencyHeader !== '' ? $request->header($provider->idempotencyHeader) : null; + $key ??= $provider->idempotencyField !== '' ? self::scalarAt($body, $provider->idempotencyField) : null; + + if ($key === null || $key === '') { + return null; + } + + // A key longer than the column is hashed: the same delivery still maps to the same key. + return strlen($key) > 191 ? hash('sha256', $key) : $key; + } + + /** + * @param array $body + */ + private static function scalarAt(array $body, string $path): ?string + { + $current = $body; + foreach (explode('.', $path) as $segment) { + if (! is_array($current) || ! array_key_exists($segment, $current)) { + return null; + } + $current = $current[$segment]; + } + + return is_scalar($current) ? (string) $current : null; + } + + /** + * @param array $headers + * @return array + */ + private static function publishableHeaders(array $headers): array + { + unset($headers['authorization'], $headers['proxy-authorization'], $headers['cookie']); + + return $headers; + } + + private static function uuid4(): string + { + $bytes = random_bytes(16); + $bytes[6] = chr((ord($bytes[6]) & 0x0F) | 0x40); + $bytes[8] = chr((ord($bytes[8]) & 0x3F) | 0x80); + + return vsprintf('%s%s-%s-%s-%s-%s%s%s', str_split(bin2hex($bytes), 4)); + } +} diff --git a/packages/webhooks/src/WebhookResult.php b/packages/webhooks/src/WebhookResult.php new file mode 100644 index 00000000..2170dc21 --- /dev/null +++ b/packages/webhooks/src/WebhookResult.php @@ -0,0 +1,48 @@ +status === 'accepted' ? 202 : 200; + } + + /** + * @return array + */ + public function toArray(): array + { + return array_filter([ + 'status' => $this->status, + 'id' => $this->eventId, + 'provider' => $this->provider, + 'eventType' => $this->eventType, + 'destination' => $this->destination, + 'idempotencyKey' => $this->idempotencyKey, + ], static fn (?string $value): bool => $value !== null); + } +} diff --git a/packages/webhooks/src/WebhooksAutoConfiguration.php b/packages/webhooks/src/WebhooksAutoConfiguration.php index 09969d70..cd257d41 100644 --- a/packages/webhooks/src/WebhooksAutoConfiguration.php +++ b/packages/webhooks/src/WebhooksAutoConfiguration.php @@ -10,14 +10,17 @@ use Firefly\Container\Attributes\Order; use Firefly\Context\Condition\Attributes\ConditionalOnMissingBean; use Firefly\Context\Condition\Attributes\ConditionalOnProperty; +use Firefly\Eda\EventPublisher; use Firefly\Kernel\Exception\Framework\ConfigurationException; use Firefly\Webhooks\Idempotency\CacheWebhookIdempotencyStore; use Firefly\Webhooks\Idempotency\DatabaseWebhookIdempotencyStore; use Firefly\Webhooks\Idempotency\InMemoryWebhookIdempotencyStore; use Firefly\Webhooks\Idempotency\WebhookIdempotencyStore; +use Firefly\Webhooks\Signature\SignatureValidatorFactory; use Illuminate\Container\Container; use Illuminate\Contracts\Cache\Factory as CacheFactory; use Illuminate\Database\ConnectionResolverInterface; +use Psr\Log\LoggerInterface; /** * The webhooks beans, every one behind firefly.webhooks.enabled (default false): installing the package is inert @@ -64,4 +67,26 @@ public function webhookIdempotencyStore(WebhooksSettings $settings, Container $c return new CacheWebhookIdempotencyStore($caches->store($settings->idempotencyCacheStore === '' ? null : $settings->idempotencyCacheStore)); } + + #[Bean] + #[ConditionalOnProperty(name: 'firefly.webhooks.enabled', havingValue: 'true')] + #[ConditionalOnMissingBean(SignatureValidatorFactory::class)] + public function signatureValidatorFactory(): SignatureValidatorFactory + { + return new SignatureValidatorFactory(static fn (): int => time()); + } + + /** + * The processor every endpoint delegates to: it publishes to whichever EventPublisher firefly/eda bound + * (firefly.eda.provider), and logs ignored duplicates on the application logger when one is bound. + */ + #[Bean] + #[ConditionalOnProperty(name: 'firefly.webhooks.enabled', havingValue: 'true')] + #[ConditionalOnMissingBean(WebhookProcessor::class)] + public function webhookProcessor(WebhooksSettings $settings, SignatureValidatorFactory $validators, WebhookIdempotencyStore $idempotency, EventPublisher $publisher, Container $container): WebhookProcessor + { + $logger = $container->bound(LoggerInterface::class) ? $container->make(LoggerInterface::class) : null; + + return new WebhookProcessor($settings, $validators, $idempotency, $publisher, $logger instanceof LoggerInterface ? $logger : null); + } } diff --git a/packages/webhooks/src/WebhooksWiringProvider.php b/packages/webhooks/src/WebhooksWiringProvider.php index 37c8eb9d..a752fdd3 100644 --- a/packages/webhooks/src/WebhooksWiringProvider.php +++ b/packages/webhooks/src/WebhooksWiringProvider.php @@ -8,6 +8,7 @@ use Firefly\Context\Boot\BootPass; use Firefly\Context\Boot\FireflyServiceProvider; use Firefly\Webhooks\Console\PruneWebhookReceiptsCommand; +use Firefly\Webhooks\Http\WebhookRouteRegistrar; /** * The boot-pass half of firefly/webhooks: the endpoint registrar, the receipts migration (publishable with @@ -23,7 +24,7 @@ final class WebhooksWiringProvider extends FireflyServiceProvider */ public function passes(): array { - return []; + return [new WebhookRouteRegistrar]; } public function boot(): void diff --git a/packages/webhooks/tests/CapstoneWebhooksDisabledTest.php b/packages/webhooks/tests/CapstoneWebhooksDisabledTest.php new file mode 100644 index 00000000..883b62df --- /dev/null +++ b/packages/webhooks/tests/CapstoneWebhooksDisabledTest.php @@ -0,0 +1,16 @@ +postWebhook('/webhooks/open', '{"type":"ping"}')->assertNotFound(); + + expect($this->app()->bound(WebhookProcessor::class))->toBeFalse() + ->and($this->events->published)->toBe([]); +}); diff --git a/packages/webhooks/tests/CapstoneWebhooksPublishFailureTest.php b/packages/webhooks/tests/CapstoneWebhooksPublishFailureTest.php new file mode 100644 index 00000000..41d17aaf --- /dev/null +++ b/packages/webhooks/tests/CapstoneWebhooksPublishFailureTest.php @@ -0,0 +1,17 @@ + 'sha256='.hash_hmac('sha256', $body, 'acme-secret'), 'X-Idempotency-Key' => 'del-503']; + + $this->postWebhook('/webhooks/acme', $body, $headers)->assertStatus(503)->assertJsonPath('code', 'WEBHOOK_PUBLISH_FAILED'); + // Not "duplicate": the failed publish released its claim. + $this->postWebhook('/webhooks/acme', $body, $headers)->assertStatus(503)->assertJsonPath('code', 'WEBHOOK_PUBLISH_FAILED'); +}); diff --git a/packages/webhooks/tests/CapstoneWebhooksTest.php b/packages/webhooks/tests/CapstoneWebhooksTest.php new file mode 100644 index 00000000..6fb4914d --- /dev/null +++ b/packages/webhooks/tests/CapstoneWebhooksTest.php @@ -0,0 +1,95 @@ +postWebhook('/webhooks/acme', $body, ['X-Signature' => 'sha256='.hash_hmac('sha256', $body, 'acme-secret'), 'X-Idempotency-Key' => 'del-1']) + ->assertStatus(202) + ->assertJsonPath('status', 'accepted') + ->assertJsonPath('eventType', 'webhook.acme.order.created'); + + expect($this->events->published)->toHaveCount(1) + ->and($this->events->published[0]['destination'])->toBe('webhooks.acme') + ->and($this->events->published[0]['eventType'])->toBe('webhook.acme.order.created') + ->and($this->events->published[0]['payload']['body'])->toBe(['type' => 'order.created', 'id' => 7]) + ->and($this->events->published[0]['payload']['idempotencyKey'])->toBe('del-1'); +}); + +it('answers a redelivery 200 duplicate and publishes it once', function () { + /** @var WebhooksCapstoneTestCase $this */ + $body = '{"type":"order.created","id":7}'; + $headers = ['X-Signature' => 'sha256='.hash_hmac('sha256', $body, 'acme-secret'), 'X-Idempotency-Key' => 'del-2']; + + $this->postWebhook('/webhooks/acme', $body, $headers)->assertStatus(202); + $this->postWebhook('/webhooks/acme', $body, $headers)->assertStatus(200)->assertJsonPath('status', 'duplicate'); + + expect($this->events->published)->toHaveCount(1); +}); + +it('refuses with problem+json and a stable code', function (string $uri, string $body, array $headers, int $status, string $code) { + /** @var WebhooksCapstoneTestCase $this */ + /** @var array $headers */ + $response = $this->postWebhook($uri, $body, $headers)->assertStatus($status)->assertJsonPath('code', $code); + + expect((string) $response->headers->get('Content-Type'))->toContain('application/problem+json') + ->and($this->events->published)->toBe([]); +})->with([ + 'bad signature' => ['/webhooks/acme', '{"type":"x"}', ['X-Signature' => 'sha256=00'], 401, 'WEBHOOK_SIGNATURE_INVALID'], + 'unsigned' => ['/webhooks/acme', '{"type":"x"}', [], 401, 'WEBHOOK_SIGNATURE_INVALID'], + 'unknown provider' => ['/webhooks/nobody', '{}', [], 404, 'WEBHOOK_PROVIDER_NOT_FOUND'], + 'too large' => ['/webhooks/open', '{"pad":"'.str_repeat('x', 5000).'"}', [], 413, 'WEBHOOK_PAYLOAD_TOO_LARGE'], + 'malformed JSON' => ['/webhooks/open', '{"type":', [], 400, 'WEBHOOK_PAYLOAD_INVALID'], +]); + +it('names the provider in the problem document', function () { + /** @var WebhooksCapstoneTestCase $this */ + $this->postWebhook('/webhooks/acme', '{}', ['X-Signature' => 'sha256=00'])->assertStatus(401)->assertJsonPath('provider', 'acme'); +}); + +it('verifies Stripe, deduplicates on the event id, and refuses a stale timestamp', function () { + /** @var WebhooksCapstoneTestCase $this */ + $body = '{"id":"evt_1","type":"payment_intent.succeeded"}'; + $sign = static fn (int $t): string => "t={$t},v1=".hash_hmac('sha256', $t.'.'.$body, 'whsec_test'); + + $this->postWebhook('/webhooks/stripe', $body, ['Stripe-Signature' => $sign(time())]) + ->assertStatus(202)->assertJsonPath('eventType', 'webhook.stripe.payment_intent.succeeded'); + $this->postWebhook('/webhooks/stripe', $body, ['Stripe-Signature' => $sign(time())]) + ->assertStatus(200)->assertJsonPath('idempotencyKey', 'evt_1'); + $this->postWebhook('/webhooks/stripe', $body, ['Stripe-Signature' => $sign(time() - 3600)]) + ->assertStatus(401)->assertJsonPath('code', 'WEBHOOK_TIMESTAMP_OUT_OF_TOLERANCE'); +}); + +it('verifies GitHub and takes the event type from X-GitHub-Event', function () { + /** @var WebhooksCapstoneTestCase $this */ + $body = '{"ref":"refs/heads/main"}'; + + $this->postWebhook('/webhooks/github', $body, [ + 'X-Hub-Signature-256' => 'sha256='.hash_hmac('sha256', $body, 'gh-secret'), + 'X-GitHub-Event' => 'push', + 'X-GitHub-Delivery' => 'gh-delivery-1', + ])->assertStatus(202)->assertJsonPath('eventType', 'webhook.github.push')->assertJsonPath('idempotencyKey', 'gh-delivery-1'); +}); + +it('verifies a Twilio form callback against the URL it was sent to', function () { + /** @var WebhooksCapstoneTestCase $this */ + $form = ['MessageSid' => 'SM1', 'MessageStatus' => 'delivered', 'To' => '+15005550006']; + $url = 'http://localhost/webhooks/twilio'; + $signature = base64_encode(hash_hmac('sha1', $url.'MessageSidSM1MessageStatusdeliveredTo+15005550006', 'twilio-token', true)); + + $this->postWebhook('/webhooks/twilio', http_build_query($form), ['X-Twilio-Signature' => $signature], $form, 'application/x-www-form-urlencoded') + ->assertStatus(202)->assertJsonPath('eventType', 'webhook.twilio.delivered'); + + expect($this->events->published[0]['payload']['body'])->toBe($form); +}); + +it('accepts an unsigned delivery for a provider explicitly set to none', function () { + /** @var WebhooksCapstoneTestCase $this */ + $this->postWebhook('/webhooks/open', '{"type":"ping"}')->assertStatus(202)->assertJsonPath('eventType', 'webhook.open.ping'); +}); diff --git a/packages/webhooks/tests/Support/FailingPublisherWebhooksCapstoneTestCase.php b/packages/webhooks/tests/Support/FailingPublisherWebhooksCapstoneTestCase.php new file mode 100644 index 00000000..9c63e5ec --- /dev/null +++ b/packages/webhooks/tests/Support/FailingPublisherWebhooksCapstoneTestCase.php @@ -0,0 +1,29 @@ + */ protected function fireflyProviders(): array { - return [WebhooksServiceProvider::class, WebhooksWiringProvider::class]; + return [EdaServiceProvider::class, WebhooksServiceProvider::class, WebhooksWiringProvider::class]; } /** @return array */ diff --git a/packages/webhooks/tests/Support/WebhooksCapstoneTestCase.php b/packages/webhooks/tests/Support/WebhooksCapstoneTestCase.php new file mode 100644 index 00000000..14ce2351 --- /dev/null +++ b/packages/webhooks/tests/Support/WebhooksCapstoneTestCase.php @@ -0,0 +1,78 @@ + */ + protected function fireflyProviders(): array + { + return [ValidationServiceProvider::class, WebServiceProvider::class, WebhooksServiceProvider::class, WebhooksWiringProvider::class]; + } + + protected function webhooksEnabled(): bool + { + return true; + } + + /** @return array */ + protected function configOverrides(): array + { + return [ + 'firefly.webhooks.enabled' => $this->webhooksEnabled(), + 'firefly.webhooks.max-body-bytes' => 4096, + 'firefly.webhooks.idempotency.store' => 'memory', + 'firefly.webhooks.providers' => [ + 'acme' => ['secret' => 'acme-secret'], + 'stripe' => ['validator' => 'stripe', 'secret' => 'whsec_test'], + 'github' => ['validator' => 'github', 'secret' => 'gh-secret'], + 'twilio' => ['validator' => 'twilio', 'secret' => 'twilio-token', 'event-type-field' => 'MessageStatus', 'idempotency-field' => 'MessageSid'], + 'open' => ['validator' => 'none'], + ], + ]; + } + + protected function defineFireflyEnvironment(Application $app): void + { + $this->events = new RecordingEventPublisher; + $app->instance(EventPublisher::class, $this->bus()); + } + + protected function bus(): EventPublisher + { + return $this->events; + } + + /** + * POSTs a raw body with exact headers — what a signature covers is the bytes, so postJson() (which re-encodes) + * cannot be used. + * + * @param array $headers + * @param array $form + * @return TestResponse + */ + public function postWebhook(string $uri, string $body, array $headers = [], array $form = [], string $contentType = 'application/json'): TestResponse + { + return $this->call('POST', $uri, $form, [], [], $this->transformHeadersToServerVars(['Content-Type' => $contentType, ...$headers]), $body); + } +} diff --git a/packages/webhooks/tests/Support/WebhooksDatabaseTestCase.php b/packages/webhooks/tests/Support/WebhooksDatabaseTestCase.php index 13adb651..9477e782 100644 --- a/packages/webhooks/tests/Support/WebhooksDatabaseTestCase.php +++ b/packages/webhooks/tests/Support/WebhooksDatabaseTestCase.php @@ -4,17 +4,18 @@ namespace Firefly\Webhooks\Tests\Support; +use Firefly\Eda\EdaServiceProvider; use Firefly\Testing\FireflyDatabaseTestCase; use Firefly\Webhooks\WebhooksServiceProvider; use Firefly\Webhooks\WebhooksWiringProvider; -/** Webhooks enabled on the database idempotency store, over Testbench + sqlite :memory:. */ +/** Webhooks enabled on the database idempotency store, over Testbench + sqlite :memory: — with firefly/eda, whose EventPublisher the processor publishes to. */ abstract class WebhooksDatabaseTestCase extends FireflyDatabaseTestCase { /** @return list */ protected function fireflyProviders(): array { - return [WebhooksServiceProvider::class, WebhooksWiringProvider::class]; + return [EdaServiceProvider::class, WebhooksServiceProvider::class, WebhooksWiringProvider::class]; } /** @return array */ diff --git a/packages/webhooks/tests/Support/WebhooksDisabledCapstoneTestCase.php b/packages/webhooks/tests/Support/WebhooksDisabledCapstoneTestCase.php new file mode 100644 index 00000000..28a09e97 --- /dev/null +++ b/packages/webhooks/tests/Support/WebhooksDisabledCapstoneTestCase.php @@ -0,0 +1,14 @@ + ['webhooks' => [ + 'enabled' => true, + 'max-body-bytes' => 64, + 'providers' => ['open' => ['validator' => 'none', 'event-type-field' => 'event.kind', 'idempotency-field' => 'event.id']], + ]]]))); + $store = new InMemoryWebhookIdempotencyStore; + + return [new WebhookProcessor($settings, new SignatureValidatorFactory(static fn (): int => time()), $store, $publisher), $store]; +} + +it('publishes an accepted webhook with its type, destination and idempotency key', function () { + $events = new RecordingEventPublisher; + [$processor] = w4Processor($events); + + $result = $processor->process(WebhookRequests::make('open', '{"event":{"kind":"order.paid","id":"d-1"}}')); + + expect($result->status)->toBe('accepted') + ->and($result->httpStatus())->toBe(202) + ->and($result->eventType)->toBe('webhook.open.order.paid') + ->and($events->published[0]['destination'])->toBe('webhooks.open') + ->and($events->published[0]['eventType'])->toBe('webhook.open.order.paid') + ->and($events->published[0]['payload']['idempotencyKey'])->toBe('d-1') + ->and($events->published[0]['payload']['body'])->toBe(['event' => ['kind' => 'order.paid', 'id' => 'd-1']]) + ->and($events->published[0]['headers']['x-webhook-provider'])->toBe('open'); +}); + +it('answers a repeated delivery as a duplicate and publishes nothing more', function () { + $events = new RecordingEventPublisher; + [$processor] = w4Processor($events); + $processor->process(WebhookRequests::make('open', '{"event":{"kind":"a","id":"d-1"}}')); + + $again = $processor->process(WebhookRequests::make('open', '{"event":{"kind":"a","id":"d-1"}}')); + + expect($again->status)->toBe('duplicate')->and($again->httpStatus())->toBe(200)->and($events->published)->toHaveCount(1); +}); + +it('refuses an unknown provider, an oversized body and malformed JSON', function () { + [$processor] = w4Processor(new RecordingEventPublisher); + + foreach ([ + 'WEBHOOK_PROVIDER_NOT_FOUND' => WebhookRequests::make('nobody', '{}'), + 'WEBHOOK_PAYLOAD_TOO_LARGE' => WebhookRequests::make('open', str_repeat('x', 65)), + 'WEBHOOK_PAYLOAD_INVALID' => WebhookRequests::make('open', '{"broken'), + ] as $code => $request) { + $refused = null; + try { + $processor->process($request); + } catch (WebhookRejectedException $e) { + $refused = $e->errorCode(); + } + + expect($refused)->toBe($code); + } +}); + +it('releases the claim when the publish fails, so the provider retry is processed', function () { + $broken = new class implements EventPublisher + { + public int $calls = 0; + + public function subscribe(string $eventTypePattern, callable $handler): void {} + + public function publish(string $destination, string $eventType, array $payload, array $headers = []): void + { + $this->calls++; + + throw new RuntimeException('broker down'); + } + + public function start(): void {} + + public function stop(): void {} + }; + [$processor] = w4Processor($broken); + $request = WebhookRequests::make('open', '{"event":{"kind":"a","id":"d-9"}}'); + + expect(fn () => $processor->process($request))->toThrow(ServiceUnavailableException::class) + ->and(fn () => $processor->process($request))->toThrow(ServiceUnavailableException::class) + ->and($broken->calls)->toBe(2); +}); From b3357b06b8db398b8eeeb9e7f3d9a93cc7e75889 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 20:53:14 -0700 Subject: [PATCH 36/47] feat(callbacks): register the firefly/callbacks package with a subscription registry on memory or any database --- composer.json | 5 + deptrac.yaml | 18 ++ docs/modules.md | 4 +- packages/callbacks/.gitattributes | 2 + packages/callbacks/LICENSE | 204 ++++++++++++++++++ packages/callbacks/README.md | 12 ++ .../cache/firefly-callbacks-components.php | 49 +++++ .../cache/firefly-callbacks-context.php | 59 +++++ packages/callbacks/composer.json | 65 ++++++ ...000001_create_firefly_callbacks_tables.php | 31 +++ .../src/CallbacksAutoConfiguration.php | 56 +++++ .../src/CallbacksServiceProvider.php | 21 ++ packages/callbacks/src/CallbacksSettings.php | 144 +++++++++++++ .../callbacks/src/CallbacksWiringProvider.php | 40 ++++ .../callbacks/src/Schema/CallbackSchema.php | 88 ++++++++ .../src/Subscription/CallbackSubscription.php | 109 ++++++++++ .../DatabaseSubscriptionRegistry.php | 128 +++++++++++ .../InMemorySubscriptionRegistry.php | 57 +++++ .../src/Subscription/SubscriptionRegistry.php | 22 ++ .../callbacks/tests/CallbacksSettingsTest.php | 62 ++++++ .../tests/CompiledManifestFreshnessTest.php | 29 +++ packages/callbacks/tests/PackageBootsTest.php | 28 +++ ...aseSubscriptionRegistryIntegrationTest.php | 40 ++++ .../SubscriptionRegistriesTest.php | 71 ++++++ .../tests/Support/CallbackDatabases.php | 46 ++++ packages/firefly/composer.json | 1 + packages/installer/src/CapabilityCatalog.php | 1 + 27 files changed, 1390 insertions(+), 2 deletions(-) create mode 100644 packages/callbacks/.gitattributes create mode 100644 packages/callbacks/LICENSE create mode 100644 packages/callbacks/README.md create mode 100644 packages/callbacks/cache/firefly-callbacks-components.php create mode 100644 packages/callbacks/cache/firefly-callbacks-context.php create mode 100644 packages/callbacks/composer.json create mode 100644 packages/callbacks/database/migrations/2026_10_12_000001_create_firefly_callbacks_tables.php create mode 100644 packages/callbacks/src/CallbacksAutoConfiguration.php create mode 100644 packages/callbacks/src/CallbacksServiceProvider.php create mode 100644 packages/callbacks/src/CallbacksSettings.php create mode 100644 packages/callbacks/src/CallbacksWiringProvider.php create mode 100644 packages/callbacks/src/Schema/CallbackSchema.php create mode 100644 packages/callbacks/src/Subscription/CallbackSubscription.php create mode 100644 packages/callbacks/src/Subscription/DatabaseSubscriptionRegistry.php create mode 100644 packages/callbacks/src/Subscription/InMemorySubscriptionRegistry.php create mode 100644 packages/callbacks/src/Subscription/SubscriptionRegistry.php create mode 100644 packages/callbacks/tests/CallbacksSettingsTest.php create mode 100644 packages/callbacks/tests/CompiledManifestFreshnessTest.php create mode 100644 packages/callbacks/tests/PackageBootsTest.php create mode 100644 packages/callbacks/tests/Subscription/Integration/DatabaseSubscriptionRegistryIntegrationTest.php create mode 100644 packages/callbacks/tests/Subscription/SubscriptionRegistriesTest.php create mode 100644 packages/callbacks/tests/Support/CallbackDatabases.php diff --git a/composer.json b/composer.json index 0355f252..eb369ea4 100644 --- a/composer.json +++ b/composer.json @@ -58,6 +58,7 @@ "Firefly\\Actuator\\Tests\\": "packages/actuator/tests/", "Firefly\\Admin\\Tests\\": "packages/admin/tests/", "Firefly\\AutoConfigure\\Tests\\": "packages/autoconfigure/tests/", + "Firefly\\Callbacks\\Tests\\": "packages/callbacks/tests/", "Firefly\\Cli\\Tests\\": "packages/cli/tests/", "Firefly\\Config\\Tests\\": "packages/config/tests/", "Firefly\\Container\\Tests\\": "packages/container/tests/", @@ -124,6 +125,7 @@ "firefly/actuator": "self.version", "firefly/admin": "self.version", "firefly/autoconfigure": "self.version", + "firefly/callbacks": "self.version", "firefly/cli": "self.version", "firefly/config": "self.version", "firefly/container": "self.version", @@ -158,6 +160,7 @@ "Firefly\\Actuator\\": "packages/actuator/src/", "Firefly\\Admin\\": "packages/admin/src/", "Firefly\\AutoConfigure\\": "packages/autoconfigure/src/", + "Firefly\\Callbacks\\": "packages/callbacks/src/", "Firefly\\Cli\\": "packages/cli/src/", "Firefly\\Config\\": "packages/config/src/", "Firefly\\Container\\": "packages/container/src/", @@ -265,6 +268,8 @@ "Firefly\\FeatureFlags\\FeatureFlagsWiringProvider", "Firefly\\Webhooks\\WebhooksServiceProvider", "Firefly\\Webhooks\\WebhooksWiringProvider", + "Firefly\\Callbacks\\CallbacksServiceProvider", + "Firefly\\Callbacks\\CallbacksWiringProvider", "Firefly\\Cli\\Boot\\FireflyCacheServiceProvider" ] } diff --git a/deptrac.yaml b/deptrac.yaml index 874d82f0..1e376ba6 100644 --- a/deptrac.yaml +++ b/deptrac.yaml @@ -106,6 +106,10 @@ deptrac: collectors: - type: directory value: packages/webhooks/src/.* + - name: Callbacks + collectors: + - type: directory + value: packages/callbacks/src/.* - name: OpenApi collectors: - type: directory @@ -522,6 +526,20 @@ deptrac: - Eda - Resilience + # Callbacks is the outbound-webhook capability: a subscription registry and an execution log on memory or any + # database connection (illuminate, unlayered), deliveries over Laravel's HTTP client driven by Eda envelopes + # (=> Eda, through a BootPass subscription), and one circuit breaker per endpoint built from Resilience's + # CircuitBreaker over the bound ResilienceStore (=> Resilience). It reads Config, carries Container/Context + # attributes, extends AutoConfigure's base and reuses Kernel exceptions. NOTHING depends on it. + Callbacks: + - Kernel + - Container + - Config + - Context + - AutoConfigure + - Eda + - Resilience + # OpenApi is a top-of-stack capability (like Web/Cqrs/Security/Actuator): it generates an OpenAPI 3.1 # document from manifests that already exist, so it is nearly all READS. It reads Web's RouteManifest/ # RouteDescriptor (paths, verbs, statuses, binding plans) and Validation's ConstraintManifest + Rule diff --git a/docs/modules.md b/docs/modules.md index 7fdb64ee..83561bf4 100644 --- a/docs/modules.md +++ b/docs/modules.md @@ -1,12 +1,12 @@ # Modules -LaraFly publishes one `fireflyframework/larafly` library containing the **31 component directories** under +LaraFly publishes one `fireflyframework/larafly` library containing the **32 component directories** under `packages/*`, each with its own tests or metadata. The **33 guides** below explain their public surfaces. Several components carry more than one guide: `firefly/data` answers for *Data & Repositories*, *Relational Data* and *Transactions*. Laravel discovers the root library's providers, and configuration selects optional features. A bean you declare wins over the framework default. Two idioms deliver that outcome, and -which one you are looking at matters the moment you go reading the source. **19 of the 31 packages** carry a +which one you are looking at matters the moment you go reading the source. **20 of the 32 packages** carry a `#[Configuration]` class whose `#[Bean]` methods sit behind `#[ConditionalOnMissingBean]`, so a bean of yours makes the framework's back off silently — `packages/data/src/DataAutoConfiguration.php` is the model. The **other 12** have no such class to go looking for; `firefly/web`, the one nearly every reader installs, guards diff --git a/packages/callbacks/.gitattributes b/packages/callbacks/.gitattributes new file mode 100644 index 00000000..538b69a4 --- /dev/null +++ b/packages/callbacks/.gitattributes @@ -0,0 +1,2 @@ +/tests export-ignore +/.gitattributes export-ignore diff --git a/packages/callbacks/LICENSE b/packages/callbacks/LICENSE new file mode 100644 index 00000000..2240005d --- /dev/null +++ b/packages/callbacks/LICENSE @@ -0,0 +1,204 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +Copyright 2026 Firefly Software Solutions Inc. diff --git a/packages/callbacks/README.md b/packages/callbacks/README.md new file mode 100644 index 00000000..5f1c59f1 --- /dev/null +++ b/packages/callbacks/README.md @@ -0,0 +1,12 @@ +# firefly/callbacks + +LaraFly's outbound callbacks. Subscriptions — a URL, the event types it wants, a secret — live in memory or in a +portable table on any database connection. Every `firefly/eda` event whose type matches `firefly.callbacks.event-types` +is delivered to each matching subscription as a JSON POST signed with `X-Firefly-Timestamp` and +`X-Firefly-Signature: sha256=…` (HMAC-SHA256 over "."), retried with exponential back-off on +408, 429, 5xx and transport errors, through a circuit breaker per endpoint built on `firefly/resilience`, and +recorded in an execution log (memory or database). `php artisan firefly:callbacks:list` shows subscriptions and +deliveries; `php artisan firefly:callbacks:replay` sends logged deliveries again. Off until +`firefly.callbacks.enabled` is true. + +Apache-2.0 © Firefly Software Solutions Inc. diff --git a/packages/callbacks/cache/firefly-callbacks-components.php b/packages/callbacks/cache/firefly-callbacks-components.php new file mode 100644 index 00000000..2bee1e65 --- /dev/null +++ b/packages/callbacks/cache/firefly-callbacks-components.php @@ -0,0 +1,49 @@ + [ + 'class' => 'Firefly\\Callbacks\\CallbacksAutoConfiguration', + 'stereotype' => 'configuration', + 'name' => null, + 'scope' => 'Singleton', + 'primary' => false, + 'order' => 1000, + 'qualifier' => null, + 'interfaces' => [ + ], + 'beans' => [ + 0 => [ + 'method' => 'callbacksSettings', + 'returns' => 'Firefly\\Callbacks\\CallbacksSettings', + 'name' => null, + 'scope' => 'Singleton', + 'primary' => false, + 'order' => 0, + 'lazy' => false, + 'dependencies' => [ + 0 => 'Firefly\\Config\\Config', + ], + ], + 1 => [ + 'method' => 'subscriptionRegistry', + 'returns' => 'Firefly\\Callbacks\\Subscription\\SubscriptionRegistry', + 'name' => null, + 'scope' => 'Singleton', + 'primary' => false, + 'order' => 0, + 'lazy' => false, + 'dependencies' => [ + 0 => 'Firefly\\Callbacks\\CallbacksSettings', + 1 => 'Illuminate\\Container\\Container', + ], + ], + ], + 'lazy' => false, + 'dependencies' => [ + ], + ], +]; diff --git a/packages/callbacks/cache/firefly-callbacks-context.php b/packages/callbacks/cache/firefly-callbacks-context.php new file mode 100644 index 00000000..a8d2375d --- /dev/null +++ b/packages/callbacks/cache/firefly-callbacks-context.php @@ -0,0 +1,59 @@ + [ + 'class' => 'Firefly\\Callbacks\\CallbacksAutoConfiguration', + 'postConstruct' => [ + ], + 'preDestroy' => [ + ], + 'listeners' => [ + ], + 'conditions' => [ + ], + 'beanConditions' => [ + 0 => [ + 'method' => 'callbacksSettings', + 'conditions' => [ + 0 => [ + 'type' => 'Firefly\\Context\\Condition\\Attributes\\ConditionalOnProperty', + 'args' => [ + 0 => 'firefly.callbacks.enabled', + 1 => 'true', + 2 => false, + ], + ], + 1 => [ + 'type' => 'Firefly\\Context\\Condition\\Attributes\\ConditionalOnMissingBean', + 'args' => [ + 0 => 'Firefly\\Callbacks\\CallbacksSettings', + ], + ], + ], + ], + 1 => [ + 'method' => 'subscriptionRegistry', + 'conditions' => [ + 0 => [ + 'type' => 'Firefly\\Context\\Condition\\Attributes\\ConditionalOnProperty', + 'args' => [ + 0 => 'firefly.callbacks.enabled', + 1 => 'true', + 2 => false, + ], + ], + 1 => [ + 'type' => 'Firefly\\Context\\Condition\\Attributes\\ConditionalOnMissingBean', + 'args' => [ + 0 => 'Firefly\\Callbacks\\Subscription\\SubscriptionRegistry', + ], + ], + ], + ], + ], + ], +]; diff --git a/packages/callbacks/composer.json b/packages/callbacks/composer.json new file mode 100644 index 00000000..0e6fdbd0 --- /dev/null +++ b/packages/callbacks/composer.json @@ -0,0 +1,65 @@ +{ + "name": "firefly/callbacks", + "description": "LaraFly outbound callbacks: subscriptions on memory or any database connection, HMAC-SHA256-signed HTTP deliveries driven by firefly/eda events, retries with exponential back-off, a circuit breaker per endpoint through firefly/resilience, an execution log on memory or a database, and artisan commands to list and replay deliveries.", + "type": "library", + "license": "Apache-2.0", + "homepage": "https://github.com/fireflyframework/fireflyframework-php", + "authors": [ + { + "name": "Firefly Software Solutions Inc.", + "homepage": "https://github.com/fireflyframework" + } + ], + "keywords": [ + "firefly", + "laravel", + "callbacks", + "webhooks" + ], + "support": { + "issues": "https://github.com/fireflyframework/fireflyframework-php/issues", + "source": "https://github.com/fireflyframework/fireflyframework-php/tree/main/packages/callbacks" + }, + "require": { + "php": "^8.3", + "firefly/autoconfigure": "*@dev", + "firefly/config": "*@dev", + "firefly/container": "*@dev", + "firefly/context": "*@dev", + "firefly/eda": "*@dev", + "firefly/kernel": "*@dev", + "firefly/resilience": "*@dev", + "illuminate/console": "^13.0", + "illuminate/container": "^13.0", + "illuminate/contracts": "^13.0", + "illuminate/database": "^13.0", + "illuminate/http": "^13.0", + "illuminate/support": "^13.0", + "psr/log": "^3.0" + }, + "extra": { + "laravel": { + "providers": [ + "Firefly\\Callbacks\\CallbacksServiceProvider", + "Firefly\\Callbacks\\CallbacksWiringProvider" + ] + }, + "branch-alias": { + "dev-main": "26.x-dev" + } + }, + "autoload": { + "psr-4": { + "Firefly\\Callbacks\\": "src/" + } + }, + "autoload-dev": { + "psr-4": { + "Firefly\\Callbacks\\Tests\\": "tests/" + } + }, + "minimum-stability": "stable", + "config": { + "sort-packages": true + } +} diff --git a/packages/callbacks/database/migrations/2026_10_12_000001_create_firefly_callbacks_tables.php b/packages/callbacks/database/migrations/2026_10_12_000001_create_firefly_callbacks_tables.php new file mode 100644 index 00000000..9763586e --- /dev/null +++ b/packages/callbacks/database/migrations/2026_10_12_000001_create_firefly_callbacks_tables.php @@ -0,0 +1,31 @@ +getConnection())); + } + + public function down(): void + { + CallbackSchema::drop(Schema::connection($this->getConnection())); + } +}; diff --git a/packages/callbacks/src/CallbacksAutoConfiguration.php b/packages/callbacks/src/CallbacksAutoConfiguration.php new file mode 100644 index 00000000..2402c59b --- /dev/null +++ b/packages/callbacks/src/CallbacksAutoConfiguration.php @@ -0,0 +1,56 @@ +registryStore === 'database' + ? new DatabaseSubscriptionRegistry($this->connection($container, $settings->connection)) + : new InMemorySubscriptionRegistry($settings->subscriptions); + } + + private function connection(Container $container, ?string $name): ConnectionInterface + { + $databases = $container->bound('db') ? $container->make('db') : null; + if (! $databases instanceof ConnectionResolverInterface) { + throw new ConfigurationException('firefly.callbacks uses a database store but no database connection resolver (db) is bound.'); + } + + return $databases->connection($name); + } +} diff --git a/packages/callbacks/src/CallbacksServiceProvider.php b/packages/callbacks/src/CallbacksServiceProvider.php new file mode 100644 index 00000000..c2fef327 --- /dev/null +++ b/packages/callbacks/src/CallbacksServiceProvider.php @@ -0,0 +1,21 @@ + */ + public const array STORES = ['memory', 'database']; + + /** + * @param list $eventTypes + * @param list $subscriptions + * @param list $allowedHosts + */ + public function __construct( + public bool $enabled = false, + public array $eventTypes = [], + public string $registryStore = 'memory', + public string $logStore = 'memory', + public ?string $connection = null, + public array $subscriptions = [], + public int $maxAttempts = 3, + public float $initialDelay = 1.0, + public float $multiplier = 2.0, + public float $maxDelay = 30.0, + public float $timeout = 10.0, + public bool $requireHttps = true, + public array $allowedHosts = [], + public bool $breakerEnabled = true, + public int $failureThreshold = 5, + public float $waitDurationInOpen = 30.0, + public string $signatureHeader = 'X-Firefly-Signature', + public string $timestampHeader = 'X-Firefly-Timestamp', + ) {} + + public static function fromConfig(Config $config): self + { + $registry = self::store($config->string('firefly.callbacks.registry.store', 'memory'), 'firefly.callbacks.registry.store'); + $log = self::store($config->string('firefly.callbacks.log.store', 'memory'), 'firefly.callbacks.log.store'); + + $maxAttempts = $config->int('firefly.callbacks.retry.max-attempts', 3); + if ($maxAttempts < 1) { + throw new ConfigurationException("firefly.callbacks.retry.max-attempts must be at least 1; got {$maxAttempts}."); + } + + $threshold = $config->int('firefly.callbacks.circuit-breaker.failure-threshold', 5); + if ($threshold < 1) { + throw new ConfigurationException("firefly.callbacks.circuit-breaker.failure-threshold must be at least 1; got {$threshold}."); + } + + $multiplier = $config->get('firefly.callbacks.retry.multiplier'); + $connection = $config->get('firefly.callbacks.connection'); + + $subscriptions = []; + foreach ($config->array('firefly.callbacks.subscriptions', []) as $index => $raw) { + if (! is_array($raw)) { + throw new ConfigurationException("firefly.callbacks.subscriptions.{$index} must be an array."); + } + $subscriptions[] = CallbackSubscription::fromArray($raw, "firefly.callbacks.subscriptions.{$index}"); + } + + return new self( + enabled: $config->bool('firefly.callbacks.enabled', false), + eventTypes: self::strings($config->array('firefly.callbacks.event-types', []), 'firefly.callbacks.event-types'), + registryStore: $registry, + logStore: $log, + connection: is_string($connection) && $connection !== '' ? $connection : null, + subscriptions: $subscriptions, + maxAttempts: $maxAttempts, + initialDelay: self::seconds($config->get('firefly.callbacks.retry.initial-delay'), 1.0, 'firefly.callbacks.retry.initial-delay'), + multiplier: is_numeric($multiplier) && (float) $multiplier >= 1.0 ? (float) $multiplier : 2.0, + maxDelay: self::seconds($config->get('firefly.callbacks.retry.max-delay'), 30.0, 'firefly.callbacks.retry.max-delay'), + timeout: self::seconds($config->get('firefly.callbacks.http.timeout'), 10.0, 'firefly.callbacks.http.timeout'), + requireHttps: $config->bool('firefly.callbacks.http.require-https', true), + allowedHosts: array_map('strtolower', self::strings($config->array('firefly.callbacks.http.allowed-hosts', []), 'firefly.callbacks.http.allowed-hosts')), + breakerEnabled: $config->bool('firefly.callbacks.circuit-breaker.enabled', true), + failureThreshold: $threshold, + waitDurationInOpen: self::seconds($config->get('firefly.callbacks.circuit-breaker.wait-duration-in-open'), 30.0, 'firefly.callbacks.circuit-breaker.wait-duration-in-open'), + signatureHeader: $config->string('firefly.callbacks.signature.header', 'X-Firefly-Signature'), + timestampHeader: $config->string('firefly.callbacks.signature.timestamp-header', 'X-Firefly-Timestamp'), + ); + } + + private static function store(string $value, string $key): string + { + if (! in_array($value, self::STORES, true)) { + throw new ConfigurationException("{$key} must be memory or database; got '{$value}'."); + } + + return $value; + } + + /** + * @param array $values + * @return list + */ + private static function strings(array $values, string $key): array + { + $strings = []; + foreach ($values as $value) { + if (! is_string($value) || $value === '') { + throw new ConfigurationException("{$key} must be a list of non-empty strings."); + } + $strings[] = $value; + } + + return $strings; + } + + private static function seconds(mixed $value, float $default, string $key): float + { + if ($value === null) { + return $default; + } + + if ((is_int($value) || is_float($value)) && $value >= 0) { + return (float) $value; + } + + if (is_string($value)) { + try { + return Duration::parse($value); + } catch (Throwable $e) { + throw new ConfigurationException("{$key} must be a duration such as 250ms, 10s or 1m; got '{$value}'.", previous: $e); + } + } + + throw new ConfigurationException("{$key} must be a duration such as 250ms, 10s or 1m, or a number of seconds."); + } +} diff --git a/packages/callbacks/src/CallbacksWiringProvider.php b/packages/callbacks/src/CallbacksWiringProvider.php new file mode 100644 index 00000000..44ebe989 --- /dev/null +++ b/packages/callbacks/src/CallbacksWiringProvider.php @@ -0,0 +1,40 @@ + + */ + public function passes(): array + { + return []; + } + + public function boot(): void + { + $migrations = dirname(__DIR__).'/database/migrations'; + $this->publishes([$migrations.'/'.self::MIGRATION => $this->app->databasePath('migrations/'.self::MIGRATION)], 'firefly-callbacks-migrations'); + + $config = new Config($this->app->make('config')); + $database = $config->string('firefly.callbacks.registry.store', 'memory') === 'database' + || $config->string('firefly.callbacks.log.store', 'memory') === 'database'; + if ($config->bool('firefly.callbacks.enabled', false) && $database) { + $this->loadMigrationsFrom($migrations); + } + } +} diff --git a/packages/callbacks/src/Schema/CallbackSchema.php b/packages/callbacks/src/Schema/CallbackSchema.php new file mode 100644 index 00000000..b1d52e21 --- /dev/null +++ b/packages/callbacks/src/Schema/CallbackSchema.php @@ -0,0 +1,88 @@ +getConnection(); + + if (! $schema->hasTable(self::SUBSCRIPTIONS)) { + $schema->create(self::SUBSCRIPTIONS, static function (Blueprint $table) use ($connection): void { + self::binary($table->string('id', 36), $connection); + $table->string('name', 191); + $table->text('url'); + $table->text('event_types'); + $table->text('secret')->nullable(); + $table->text('headers'); + $table->boolean('active'); + $table->integer('max_attempts')->nullable(); + $table->bigInteger('created_at'); + $table->bigInteger('updated_at'); + $table->primary(['id'], 'firefly_callback_subscriptions_pk'); + }); + } + + if (! $schema->hasTable(self::EXECUTIONS)) { + $schema->create(self::EXECUTIONS, static function (Blueprint $table) use ($connection): void { + self::binary($table->string('id', 64), $connection); + self::binary($table->string('subscription_id', 36), $connection); + $table->string('event_id', 64); + $table->string('event_type', 191); + $table->text('url'); + $table->longText('body'); + $table->string('status', 16); + $table->integer('attempts'); + $table->integer('response_status')->nullable(); + $table->text('last_error')->nullable(); + $table->bigInteger('created_at'); + $table->bigInteger('updated_at'); + $table->primary(['id'], 'firefly_callback_executions_pk'); + $table->index(['subscription_id'], 'firefly_callback_executions_subscription'); + $table->index(['status', 'updated_at'], 'firefly_callback_executions_status'); + }); + } + } + + public static function drop(Builder $schema): void + { + $schema->dropIfExists(self::EXECUTIONS); + $schema->dropIfExists(self::SUBSCRIPTIONS); + } + + private static function binary(ColumnDefinition $column, Connection $connection): void + { + if ($connection instanceof MySqlConnection) { + $modern = version_compare($connection->getServerVersion(), $connection->isMaria() ? '10.2.2' : '8.0.1', '>='); + $column->charset('utf8mb4')->collation($modern ? ($connection->isMaria() ? 'utf8mb4_nopad_bin' : 'utf8mb4_0900_bin') : 'utf8mb4_bin'); + } elseif ($connection instanceof SqlServerConnection) { + $column->collation('Latin1_General_100_BIN2'); + } + } +} diff --git a/packages/callbacks/src/Subscription/CallbackSubscription.php b/packages/callbacks/src/Subscription/CallbackSubscription.php new file mode 100644 index 00000000..91909c90 --- /dev/null +++ b/packages/callbacks/src/Subscription/CallbackSubscription.php @@ -0,0 +1,109 @@ + $eventTypes + * @param array $headers + */ + public function __construct( + public string $id, + public string $name, + public string $url, + public array $eventTypes, + public ?string $secret = null, + public array $headers = [], + public bool $active = true, + public ?int $maxAttempts = null, + ) {} + + /** + * @param list $eventTypes + * @param array $headers + */ + public static function create(string $name, string $url, array $eventTypes, ?string $secret = null, array $headers = [], ?int $maxAttempts = null): self + { + $bytes = random_bytes(16); + $bytes[6] = chr((ord($bytes[6]) & 0x0F) | 0x40); + $bytes[8] = chr((ord($bytes[8]) & 0x3F) | 0x80); + + return new self(vsprintf('%s%s-%s-%s-%s-%s%s%s', str_split(bin2hex($bytes), 4)), $name, $url, $eventTypes, $secret, $headers, true, $maxAttempts); + } + + /** + * A subscription written in configuration (firefly.callbacks.subscriptions.N). A seed without an `id` gets one + * derived from its URL and event types, so it is the same subscription — with the same execution history — + * across deployments. + * + * @param array $raw + */ + public static function fromArray(array $raw, string $where): self + { + $url = $raw['url'] ?? null; + if (! is_string($url) || filter_var($url, FILTER_VALIDATE_URL) === false) { + throw new ConfigurationException("{$where}.url must be an absolute URL."); + } + + $types = $raw['event-types'] ?? null; + if (! is_array($types) || $types === []) { + throw new ConfigurationException("{$where}.event-types must list at least one event-type pattern."); + } + $eventTypes = []; + foreach ($types as $type) { + if (! is_string($type) || $type === '') { + throw new ConfigurationException("{$where}.event-types must contain only non-empty strings."); + } + $eventTypes[] = $type; + } + + $headers = []; + foreach (is_array($raw['headers'] ?? null) ? $raw['headers'] : [] as $name => $value) { + if (is_string($value)) { + $headers[(string) $name] = $value; + } + } + + $id = is_string($raw['id'] ?? null) && $raw['id'] !== '' ? $raw['id'] : substr(hash('sha256', $url."\0".implode(',', $eventTypes)), 0, 32); + $name = is_string($raw['name'] ?? null) && $raw['name'] !== '' ? $raw['name'] : (string) parse_url($url, PHP_URL_HOST); + $secret = is_string($raw['secret'] ?? null) && $raw['secret'] !== '' ? $raw['secret'] : null; + $maxAttempts = is_int($raw['max-attempts'] ?? null) && $raw['max-attempts'] > 0 ? $raw['max-attempts'] : null; + + return new self($id, $name, $url, $eventTypes, $secret, $headers, ($raw['active'] ?? true) !== false, $maxAttempts); + } + + public function matches(string $eventType): bool + { + if (! $this->active) { + return false; + } + + foreach ($this->eventTypes as $pattern) { + if (fnmatch($pattern, $eventType)) { + return true; + } + } + + return false; + } + + public function endpoint(): string + { + $scheme = strtolower((string) parse_url($this->url, PHP_URL_SCHEME)); + $host = strtolower((string) parse_url($this->url, PHP_URL_HOST)); + $port = parse_url($this->url, PHP_URL_PORT) ?? ($scheme === 'https' ? 443 : 80); + + return "{$scheme}://{$host}:{$port}"; + } +} diff --git a/packages/callbacks/src/Subscription/DatabaseSubscriptionRegistry.php b/packages/callbacks/src/Subscription/DatabaseSubscriptionRegistry.php new file mode 100644 index 00000000..40234a39 --- /dev/null +++ b/packages/callbacks/src/Subscription/DatabaseSubscriptionRegistry.php @@ -0,0 +1,128 @@ +clock = $clock ?? static fn (): int => time(); + } + + public function save(CallbackSubscription $subscription): void + { + $now = ($this->clock)(); + $row = [ + 'name' => $subscription->name, + 'url' => $subscription->url, + 'event_types' => json_encode($subscription->eventTypes, JSON_THROW_ON_ERROR), + 'secret' => $subscription->secret, + 'headers' => json_encode((object) $subscription->headers, JSON_THROW_ON_ERROR), + 'active' => $subscription->active, + 'max_attempts' => $subscription->maxAttempts, + 'updated_at' => $now, + ]; + + try { + $this->connection->transaction(fn (): bool => $this->connection->table(CallbackSchema::SUBSCRIPTIONS)->insert(['id' => $subscription->id, 'created_at' => $now] + $row)); + } catch (UniqueConstraintViolationException) { + $this->connection->table(CallbackSchema::SUBSCRIPTIONS)->where('id', $subscription->id)->update($row); + } + } + + public function find(string $id): ?CallbackSubscription + { + $row = $this->connection->table(CallbackSchema::SUBSCRIPTIONS)->where('id', $id)->first(); + + return is_object($row) ? self::hydrate($row) : null; + } + + public function delete(string $id): bool + { + return $this->connection->table(CallbackSchema::SUBSCRIPTIONS)->where('id', $id)->delete() > 0; + } + + public function all(): array + { + $subscriptions = []; + foreach ($this->connection->table(CallbackSchema::SUBSCRIPTIONS)->orderBy('created_at')->orderBy('id')->get() as $row) { + $subscriptions[] = self::hydrate($row); + } + + return $subscriptions; + } + + public function matching(string $eventType): array + { + return array_values(array_filter($this->all(), static fn (CallbackSubscription $s): bool => $s->matches($eventType))); + } + + private static function hydrate(object $row): CallbackSubscription + { + $values = get_object_vars($row); + + try { + $types = json_decode(self::text($values['event_types'] ?? '[]'), true, 16, JSON_THROW_ON_ERROR); + $headers = json_decode(self::text($values['headers'] ?? '{}'), true, 16, JSON_THROW_ON_ERROR); + } catch (JsonException) { + $types = []; + $headers = []; + } + + $eventTypes = []; + foreach (is_array($types) ? $types : [] as $type) { + if (is_string($type)) { + $eventTypes[] = $type; + } + } + + $stringHeaders = []; + foreach (is_array($headers) ? $headers : [] as $name => $value) { + if (is_string($value)) { + $stringHeaders[(string) $name] = $value; + } + } + + $maxAttempts = $values['max_attempts'] ?? null; + $secret = $values['secret'] ?? null; + + return new CallbackSubscription( + self::text($values['id'] ?? ''), + self::text($values['name'] ?? ''), + self::text($values['url'] ?? ''), + $eventTypes, + is_string($secret) ? $secret : null, + $stringHeaders, + (bool) ($values['active'] ?? false), + is_numeric($maxAttempts) ? (int) $maxAttempts : null, + ); + } + + /** A column read back as text: drivers return strings (or ints for numeric-looking ids), never anything else here. */ + private static function text(mixed $value): string + { + return is_scalar($value) ? (string) $value : ''; + } +} diff --git a/packages/callbacks/src/Subscription/InMemorySubscriptionRegistry.php b/packages/callbacks/src/Subscription/InMemorySubscriptionRegistry.php new file mode 100644 index 00000000..b0a397ba --- /dev/null +++ b/packages/callbacks/src/Subscription/InMemorySubscriptionRegistry.php @@ -0,0 +1,57 @@ + */ + private array $subscriptions = []; + + /** + * @param list $seed + */ + public function __construct(array $seed = []) + { + foreach ($seed as $subscription) { + $this->save($subscription); + } + } + + public function save(CallbackSubscription $subscription): void + { + $this->subscriptions[$subscription->id] = $subscription; + } + + public function find(string $id): ?CallbackSubscription + { + return $this->subscriptions[$id] ?? null; + } + + public function delete(string $id): bool + { + if (! isset($this->subscriptions[$id])) { + return false; + } + + unset($this->subscriptions[$id]); + + return true; + } + + public function all(): array + { + return array_values($this->subscriptions); + } + + public function matching(string $eventType): array + { + return array_values(array_filter($this->subscriptions, static fn (CallbackSubscription $s): bool => $s->matches($eventType))); + } +} diff --git a/packages/callbacks/src/Subscription/SubscriptionRegistry.php b/packages/callbacks/src/Subscription/SubscriptionRegistry.php new file mode 100644 index 00000000..7e673bb2 --- /dev/null +++ b/packages/callbacks/src/Subscription/SubscriptionRegistry.php @@ -0,0 +1,22 @@ + in the order they were first saved */ + public function all(): array; + + /** @return list the active subscriptions one of whose patterns matches $eventType */ + public function matching(string $eventType): array; +} diff --git a/packages/callbacks/tests/CallbacksSettingsTest.php b/packages/callbacks/tests/CallbacksSettingsTest.php new file mode 100644 index 00000000..0d1d35fc --- /dev/null +++ b/packages/callbacks/tests/CallbacksSettingsTest.php @@ -0,0 +1,62 @@ + $callbacks a firefly.callbacks subtree, as a config file or a dataset spells it + */ +function w4CallbacksSettings(array $callbacks): CallbacksSettings +{ + return CallbacksSettings::fromConfig(new Config(new Repository(['firefly' => ['callbacks' => $callbacks]]))); +} + +it('has safe defaults: off, nothing subscribed, https only, three attempts, a breaker per endpoint', function () { + $settings = w4CallbacksSettings([]); + + expect($settings)->toEqual(new CallbacksSettings) + ->and($settings->enabled)->toBeFalse() + ->and($settings->eventTypes)->toBe([]) + ->and($settings->requireHttps)->toBeTrue() + ->and($settings->maxAttempts)->toBe(3) + ->and($settings->breakerEnabled)->toBeTrue(); +}); + +it('reads every key, durations and seeded subscriptions included', function () { + $settings = w4CallbacksSettings([ + 'enabled' => true, + 'event-types' => ['order.*'], + 'registry' => ['store' => 'database'], + 'log' => ['store' => 'database'], + 'connection' => 'pgsql', + 'retry' => ['max-attempts' => 5, 'initial-delay' => '250ms', 'multiplier' => 3, 'max-delay' => '10s'], + 'http' => ['timeout' => '2s', 'require-https' => false, 'allowed-hosts' => ['Example.com']], + 'circuit-breaker' => ['enabled' => false, 'failure-threshold' => 2, 'wait-duration-in-open' => '1m'], + 'signature' => ['header' => 'X-Sig', 'timestamp-header' => 'X-Ts'], + 'subscriptions' => [['url' => 'https://partner.example.com/hooks', 'event-types' => ['order.created'], 'secret' => 's']], + ]); + + expect($settings->registryStore)->toBe('database') + ->and($settings->connection)->toBe('pgsql') + ->and([$settings->maxAttempts, $settings->initialDelay, $settings->multiplier, $settings->maxDelay])->toBe([5, 0.25, 3.0, 10.0]) + ->and([$settings->timeout, $settings->requireHttps, $settings->allowedHosts])->toBe([2.0, false, ['example.com']]) + ->and([$settings->breakerEnabled, $settings->failureThreshold, $settings->waitDurationInOpen])->toBe([false, 2, 60.0]) + ->and([$settings->signatureHeader, $settings->timestampHeader])->toBe(['X-Sig', 'X-Ts']) + ->and($settings->subscriptions[0]->eventTypes)->toBe(['order.created']) + ->and($settings->subscriptions[0]->endpoint())->toBe('https://partner.example.com:443'); +}); + +it('refuses at boot what could never deliver', function (array $callbacks, string $message) { + expect(fn () => w4CallbacksSettings($callbacks))->toThrow(ConfigurationException::class, $message); +})->with([ + 'unknown registry' => [['registry' => ['store' => 'redis']], 'firefly.callbacks.registry.store'], + 'unknown log' => [['log' => ['store' => 'file']], 'firefly.callbacks.log.store'], + 'no attempts' => [['retry' => ['max-attempts' => 0]], 'firefly.callbacks.retry.max-attempts'], + 'event types not strings' => [['event-types' => [1]], 'firefly.callbacks.event-types'], + 'subscription without url' => [['subscriptions' => [['event-types' => ['a']]]], 'firefly.callbacks.subscriptions.0.url'], + 'subscription without event types' => [['subscriptions' => [['url' => 'https://x.example.com']]], 'firefly.callbacks.subscriptions.0.event-types'], +]); diff --git a/packages/callbacks/tests/CompiledManifestFreshnessTest.php b/packages/callbacks/tests/CompiledManifestFreshnessTest.php new file mode 100644 index 00000000..d438f0c3 --- /dev/null +++ b/packages/callbacks/tests/CompiledManifestFreshnessTest.php @@ -0,0 +1,29 @@ + dirname(__DIR__).'/src']; + $cache = dirname(__DIR__).'/cache'; + + $freshComponents = sys_get_temp_dir().'/firefly-callbacks-fresh-c-'.bin2hex(random_bytes(6)).'.php'; + $freshContext = sys_get_temp_dir().'/firefly-callbacks-fresh-x-'.bin2hex(random_bytes(6)).'.php'; + + try { + (new AutoConfigManifestCompiler)->write($src, $freshComponents, $freshContext); + + expect(ComponentManifest::load($freshComponents))->toEqual(ComponentManifest::load($cache.'/firefly-callbacks-components.php')) + ->and(ContextManifest::load($freshContext))->toEqual(ContextManifest::load($cache.'/firefly-callbacks-context.php')); + } finally { + @unlink($freshComponents); + @unlink($freshContext); + } +}); diff --git a/packages/callbacks/tests/PackageBootsTest.php b/packages/callbacks/tests/PackageBootsTest.php new file mode 100644 index 00000000..5ab8cd38 --- /dev/null +++ b/packages/callbacks/tests/PackageBootsTest.php @@ -0,0 +1,28 @@ + []], providers: [CallbacksServiceProvider::class, CallbacksWiringProvider::class]); + + expect($app->make(ApplicationContext::class))->toBeInstanceOf(ApplicationContext::class) + ->and($app->bound(CallbacksSettings::class))->toBeFalse(); +}); + +it('binds the memory registry seeded from configuration when enabled', function () { + $app = fireflyApplication( + config: ['firefly' => ['callbacks' => ['enabled' => true, 'subscriptions' => [['id' => 's-1', 'url' => 'https://partner.example.com/hooks', 'event-types' => ['order.*']]]]]], + providers: [CallbacksServiceProvider::class, CallbacksWiringProvider::class], + ); + $registry = $app->make(SubscriptionRegistry::class); + + expect($registry)->toBeInstanceOf(InMemorySubscriptionRegistry::class) + ->and($registry->find('s-1')?->url)->toBe('https://partner.example.com/hooks'); +}); diff --git a/packages/callbacks/tests/Subscription/Integration/DatabaseSubscriptionRegistryIntegrationTest.php b/packages/callbacks/tests/Subscription/Integration/DatabaseSubscriptionRegistryIntegrationTest.php new file mode 100644 index 00000000..7731cd98 --- /dev/null +++ b/packages/callbacks/tests/Subscription/Integration/DatabaseSubscriptionRegistryIntegrationTest.php @@ -0,0 +1,40 @@ + ['pgsql', 'FIREFLY_PG_DSN'], + 'MySQL' => ['mysql', 'FIREFLY_MYSQL_DSN'], + 'MariaDB' => ['mysql', 'FIREFLY_MARIADB_DSN'], + 'SQL Server' => ['sqlsrv', 'FIREFLY_SQLSRV_DSN'], +] as $backend => [$driver, $variable]) { + it("round-trips subscriptions, re-saves unchanged ones and matches patterns on {$backend}", function () use ($driver, $variable): void { + $connection = CallbackDatabases::connection($driver, $variable); + CallbackSchema::drop($connection->getSchemaBuilder()); + CallbackSchema::create($connection->getSchemaBuilder()); + CallbackSchema::create($connection->getSchemaBuilder()); + $registry = new DatabaseSubscriptionRegistry($connection); + + try { + $orders = CallbackSubscription::create('órdenes', 'https://partner.example.com/hooks?x=1', ['order.*', 'refund.created'], 'sécret', ['X-Partner' => 'p1'], 4); + $registry->save($orders); + $registry->save($orders); // unchanged: MySQL reports 0 affected rows, which must not insert a duplicate + $registry->save(new CallbackSubscription('UPPER', 'case', 'https://a.example.com', ['a'], null, [], true, null)); + $registry->save(new CallbackSubscription('upper', 'case', 'https://b.example.com', ['a'], null, [], false, null)); + + expect($registry->find($orders->id))->toEqual($orders) + ->and($registry->all())->toHaveCount(3) + ->and($registry->find('UPPER')?->url)->toBe('https://a.example.com') + ->and($registry->find('upper')?->url)->toBe('https://b.example.com') + ->and(array_map(static fn (CallbackSubscription $s): string => $s->id, $registry->matching('refund.created')))->toBe([$orders->id]) + ->and($registry->matching('a'))->toHaveCount(1); + } finally { + CallbackSchema::drop($connection->getSchemaBuilder()); + } + })->group('integration')->skip(getenv($variable) === false, "Set {$variable} to an isolated database."); +} diff --git a/packages/callbacks/tests/Subscription/SubscriptionRegistriesTest.php b/packages/callbacks/tests/Subscription/SubscriptionRegistriesTest.php new file mode 100644 index 00000000..ea8fe70f --- /dev/null +++ b/packages/callbacks/tests/Subscription/SubscriptionRegistriesTest.php @@ -0,0 +1,71 @@ +addConnection(['driver' => 'sqlite', 'database' => ':memory:', 'prefix' => '']); + $connection = $capsule->getConnection(); + CallbackSchema::create($connection->getSchemaBuilder()); + + return $connection; +} + +dataset('w4 subscription registries', [ + 'memory' => [static fn (): SubscriptionRegistry => new InMemorySubscriptionRegistry], + // A ticking clock: every save is one second later, so created_at orders all() exactly as the memory registry + // does (two saves inside one wall-clock second would otherwise tie and fall back to the random ids). + 'database (sqlite)' => [static function (): SubscriptionRegistry { + $tick = 0; + + return new DatabaseSubscriptionRegistry(w4CallbacksSqlite(), static function () use (&$tick): int { + return ++$tick; + }); + }], +]); + +it('saves, finds, updates, lists and deletes subscriptions', function (Closure $make) { + /** @var SubscriptionRegistry $registry */ + $registry = $make(); + $orders = CallbackSubscription::create('orders', 'https://partner.example.com/orders', ['order.*'], 'secret', ['X-Partner' => 'p1'], 4); + $billing = CallbackSubscription::create('billing', 'https://billing.example.com/hook', ['invoice.paid']); + + $registry->save($orders); + $registry->save($billing); + $registry->save(new CallbackSubscription($orders->id, 'orders v2', $orders->url, ['order.created'], 'rotated', [], false, null)); + + expect($registry->find($orders->id))->toEqual(new CallbackSubscription($orders->id, 'orders v2', $orders->url, ['order.created'], 'rotated', [], false, null)) + ->and($registry->find($billing->id))->toEqual($billing) + ->and(array_map(static fn (CallbackSubscription $s): string => $s->name, $registry->all()))->toBe(['orders v2', 'billing']) + ->and($registry->delete($billing->id))->toBeTrue() + ->and($registry->delete($billing->id))->toBeFalse() + ->and($registry->find('nope'))->toBeNull(); +})->with('w4 subscription registries'); + +it('matches active subscriptions by event-type pattern only', function (Closure $make) { + /** @var SubscriptionRegistry $registry */ + $registry = $make(); + $registry->save($all = CallbackSubscription::create('all orders', 'https://a.example.com', ['order.*'])); + $registry->save(CallbackSubscription::create('paid', 'https://b.example.com', ['invoice.paid'])); + $registry->save(new CallbackSubscription('off', 'paused', 'https://c.example.com', ['order.*'], null, [], false, null)); + + expect(array_map(static fn (CallbackSubscription $s): string => $s->id, $registry->matching('order.created')))->toBe([$all->id]) + ->and($registry->matching('user.created'))->toBe([]); +})->with('w4 subscription registries'); + +it('seeds the memory registry and derives a stable id for a seed without one', function () { + $seed = CallbackSubscription::fromArray(['url' => 'https://a.example.com/x', 'event-types' => ['a']], 'firefly.callbacks.subscriptions.0'); + + expect($seed->id)->toBe(CallbackSubscription::fromArray(['url' => 'https://a.example.com/x', 'event-types' => ['a']], 'k')->id) + ->and($seed->name)->toBe('a.example.com') + ->and((new InMemorySubscriptionRegistry([$seed]))->all())->toEqual([$seed]); +}); diff --git a/packages/callbacks/tests/Support/CallbackDatabases.php b/packages/callbacks/tests/Support/CallbackDatabases.php new file mode 100644 index 00000000..714383df --- /dev/null +++ b/packages/callbacks/tests/Support/CallbackDatabases.php @@ -0,0 +1,46 @@ +addConnection([ + 'driver' => $driver, + 'host' => $parts['host'] ?? '127.0.0.1', + 'port' => (int) ($parts['port'] ?? 0), + 'database' => $parts['dbname'] ?? '', + 'username' => $parts['user'] ?? '', + 'password' => $parts['password'] ?? '', + 'charset' => match ($driver) { + 'pgsql', 'sqlsrv' => 'utf8', + default => 'utf8mb4', + }, + 'prefix' => '', + 'schema' => 'public', + 'trust_server_certificate' => true, + ]); + + return $capsule->getConnection(); + } +} diff --git a/packages/firefly/composer.json b/packages/firefly/composer.json index 3c13d468..f14ae10b 100644 --- a/packages/firefly/composer.json +++ b/packages/firefly/composer.json @@ -25,6 +25,7 @@ "firefly/actuator": "*@dev", "firefly/admin": "*@dev", "firefly/autoconfigure": "*@dev", + "firefly/callbacks": "*@dev", "firefly/cli": "*@dev", "firefly/config": "*@dev", "firefly/container": "*@dev", diff --git a/packages/installer/src/CapabilityCatalog.php b/packages/installer/src/CapabilityCatalog.php index 05ff46b8..d1e1915a 100644 --- a/packages/installer/src/CapabilityCatalog.php +++ b/packages/installer/src/CapabilityCatalog.php @@ -45,6 +45,7 @@ public static function all(): array new Capability('openapi', 'firefly/openapi', 'OpenAPI 3.1 document generated from the route and constraint manifests, plus a viewer'), new Capability('feature-flags', 'firefly/feature-flags', 'Feature flags on OpenFeature: flagd documents, targeting and percentage rollouts, a writable store, admin and actuator controls'), new Capability('webhooks', 'firefly/webhooks', 'Inbound webhooks: an endpoint per provider, HMAC/Stripe/GitHub/Twilio signatures, a replay window and idempotency, published to the event bus', requires: ['eda']), + new Capability('callbacks', 'firefly/callbacks', 'Outbound callbacks: subscriptions, HMAC-signed deliveries driven by events, retries with back-off, a circuit breaker per endpoint and a replayable execution log', requires: ['eda', 'resilience']), new Capability('eda-kafka', 'firefly/eda-kafka', 'Kafka publisher/consumer over ext-rdkafka', requires: ['eda'], adapter: true), new Capability('eda-rabbitmq', 'firefly/eda-rabbitmq', 'RabbitMQ publisher/consumer over php-amqplib', requires: ['eda'], adapter: true), From 478327add9a1c3e671ab4a61f4e3dcd04bb95394 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 20:56:57 -0700 Subject: [PATCH 37/47] fix(callbacks): refuse at boot a seeded subscription id wider than the 36-character id columns --- packages/callbacks/src/Schema/CallbackSchema.php | 5 +++-- .../src/Subscription/CallbackSubscription.php | 15 +++++++++++++-- .../callbacks/tests/CallbacksSettingsTest.php | 11 +++++++++++ 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/packages/callbacks/src/Schema/CallbackSchema.php b/packages/callbacks/src/Schema/CallbackSchema.php index b1d52e21..6306c6e9 100644 --- a/packages/callbacks/src/Schema/CallbackSchema.php +++ b/packages/callbacks/src/Schema/CallbackSchema.php @@ -4,6 +4,7 @@ namespace Firefly\Callbacks\Schema; +use Firefly\Callbacks\Subscription\CallbackSubscription; use Illuminate\Database\Connection; use Illuminate\Database\MySqlConnection; use Illuminate\Database\Schema\Blueprint; @@ -35,7 +36,7 @@ public static function create(Builder $schema): void if (! $schema->hasTable(self::SUBSCRIPTIONS)) { $schema->create(self::SUBSCRIPTIONS, static function (Blueprint $table) use ($connection): void { - self::binary($table->string('id', 36), $connection); + self::binary($table->string('id', CallbackSubscription::MAX_ID_LENGTH), $connection); $table->string('name', 191); $table->text('url'); $table->text('event_types'); @@ -52,7 +53,7 @@ public static function create(Builder $schema): void if (! $schema->hasTable(self::EXECUTIONS)) { $schema->create(self::EXECUTIONS, static function (Blueprint $table) use ($connection): void { self::binary($table->string('id', 64), $connection); - self::binary($table->string('subscription_id', 36), $connection); + self::binary($table->string('subscription_id', CallbackSubscription::MAX_ID_LENGTH), $connection); $table->string('event_id', 64); $table->string('event_type', 191); $table->text('url'); diff --git a/packages/callbacks/src/Subscription/CallbackSubscription.php b/packages/callbacks/src/Subscription/CallbackSubscription.php index 91909c90..ed753ca1 100644 --- a/packages/callbacks/src/Subscription/CallbackSubscription.php +++ b/packages/callbacks/src/Subscription/CallbackSubscription.php @@ -14,6 +14,9 @@ */ final readonly class CallbackSubscription { + /** The width of the id columns in CallbackSchema — a UUID fits exactly. */ + public const MAX_ID_LENGTH = 36; + /** * @param list $eventTypes * @param array $headers @@ -45,7 +48,8 @@ public static function create(string $name, string $url, array $eventTypes, ?str /** * A subscription written in configuration (firefly.callbacks.subscriptions.N). A seed without an `id` gets one * derived from its URL and event types, so it is the same subscription — with the same execution history — - * across deployments. + * across deployments. An explicit `id` is at most MAX_ID_LENGTH characters of `[A-Za-z0-9._-]`, the width the + * subscription and execution tables give it. * * @param array $raw */ @@ -75,7 +79,14 @@ public static function fromArray(array $raw, string $where): self } } - $id = is_string($raw['id'] ?? null) && $raw['id'] !== '' ? $raw['id'] : substr(hash('sha256', $url."\0".implode(',', $eventTypes)), 0, 32); + $id = $raw['id'] ?? null; + if ($id === null || $id === '') { + $id = substr(hash('sha256', $url."\0".implode(',', $eventTypes)), 0, 32); + } elseif (! is_string($id) || preg_match('/^[A-Za-z0-9._-]{1,'.self::MAX_ID_LENGTH.'}$/', $id) !== 1) { + // The registry and execution tables key on a 36-character column, so a longer seed would boot and + // then fail every execution insert — refuse it here instead. + throw new ConfigurationException("{$where}.id must be at most ".self::MAX_ID_LENGTH.' characters of letters, digits, dots, dashes or underscores.'); + } $name = is_string($raw['name'] ?? null) && $raw['name'] !== '' ? $raw['name'] : (string) parse_url($url, PHP_URL_HOST); $secret = is_string($raw['secret'] ?? null) && $raw['secret'] !== '' ? $raw['secret'] : null; $maxAttempts = is_int($raw['max-attempts'] ?? null) && $raw['max-attempts'] > 0 ? $raw['max-attempts'] : null; diff --git a/packages/callbacks/tests/CallbacksSettingsTest.php b/packages/callbacks/tests/CallbacksSettingsTest.php index 0d1d35fc..c1a3af73 100644 --- a/packages/callbacks/tests/CallbacksSettingsTest.php +++ b/packages/callbacks/tests/CallbacksSettingsTest.php @@ -3,6 +3,7 @@ declare(strict_types=1); use Firefly\Callbacks\CallbacksSettings; +use Firefly\Callbacks\Subscription\CallbackSubscription; use Firefly\Config\Config; use Firefly\Kernel\Exception\Framework\ConfigurationException; use Illuminate\Config\Repository; @@ -59,4 +60,14 @@ function w4CallbacksSettings(array $callbacks): CallbacksSettings 'event types not strings' => [['event-types' => [1]], 'firefly.callbacks.event-types'], 'subscription without url' => [['subscriptions' => [['event-types' => ['a']]]], 'firefly.callbacks.subscriptions.0.url'], 'subscription without event types' => [['subscriptions' => [['url' => 'https://x.example.com']]], 'firefly.callbacks.subscriptions.0.event-types'], + 'subscription id longer than its column' => [['subscriptions' => [['id' => 'partner-orders-production-eu-west-1a-2', 'url' => 'https://x.example.com', 'event-types' => ['a']]]], 'firefly.callbacks.subscriptions.0.id must be at most 36 characters'], + 'subscription id with unsafe characters' => [['subscriptions' => [['id' => 'orders/eu west', 'url' => 'https://x.example.com', 'event-types' => ['a']]]], 'firefly.callbacks.subscriptions.0.id'], + 'subscription id not a string' => [['subscriptions' => [['id' => 42, 'url' => 'https://x.example.com', 'event-types' => ['a']]]], 'firefly.callbacks.subscriptions.0.id'], ]); + +it('accepts a seeded id exactly as wide as the id columns', function () { + $id = str_repeat('a', CallbackSubscription::MAX_ID_LENGTH); + $settings = w4CallbacksSettings(['subscriptions' => [['id' => $id, 'url' => 'https://x.example.com', 'event-types' => ['a']]]]); + + expect($settings->subscriptions[0]->id)->toBe($id); +}); From 2064bb246b4dabe51df0a6310015df8765d64969 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 21:02:24 -0700 Subject: [PATCH 38/47] feat(callbacks): an execution log of every delivery on memory or any database, one row per subscription and event --- .../cache/firefly-callbacks-components.php | 13 ++ .../cache/firefly-callbacks-context.php | 19 +++ .../src/CallbacksAutoConfiguration.php | 13 ++ .../src/Execution/CallbackExecution.php | 77 +++++++++++ .../src/Execution/CallbackStatus.php | 13 ++ .../src/Execution/DatabaseExecutionLog.php | 107 +++++++++++++++ .../callbacks/src/Execution/ExecutionLog.php | 23 ++++ .../src/Execution/InMemoryExecutionLog.php | 62 +++++++++ .../DatabaseExecutionLogBootTest.php | 34 +++++ .../tests/Execution/ExecutionLogsTest.php | 129 ++++++++++++++++++ .../DatabaseExecutionLogIntegrationTest.php | 74 ++++++++++ .../Support/CallbacksDatabaseLogTestCase.php | 29 ++++ 12 files changed, 593 insertions(+) create mode 100644 packages/callbacks/src/Execution/CallbackExecution.php create mode 100644 packages/callbacks/src/Execution/CallbackStatus.php create mode 100644 packages/callbacks/src/Execution/DatabaseExecutionLog.php create mode 100644 packages/callbacks/src/Execution/ExecutionLog.php create mode 100644 packages/callbacks/src/Execution/InMemoryExecutionLog.php create mode 100644 packages/callbacks/tests/Execution/DatabaseExecutionLogBootTest.php create mode 100644 packages/callbacks/tests/Execution/ExecutionLogsTest.php create mode 100644 packages/callbacks/tests/Execution/Integration/DatabaseExecutionLogIntegrationTest.php create mode 100644 packages/callbacks/tests/Support/CallbacksDatabaseLogTestCase.php diff --git a/packages/callbacks/cache/firefly-callbacks-components.php b/packages/callbacks/cache/firefly-callbacks-components.php index 2bee1e65..e9cd6210 100644 --- a/packages/callbacks/cache/firefly-callbacks-components.php +++ b/packages/callbacks/cache/firefly-callbacks-components.php @@ -41,6 +41,19 @@ 1 => 'Illuminate\\Container\\Container', ], ], + 2 => [ + 'method' => 'executionLog', + 'returns' => 'Firefly\\Callbacks\\Execution\\ExecutionLog', + 'name' => null, + 'scope' => 'Singleton', + 'primary' => false, + 'order' => 0, + 'lazy' => false, + 'dependencies' => [ + 0 => 'Firefly\\Callbacks\\CallbacksSettings', + 1 => 'Illuminate\\Container\\Container', + ], + ], ], 'lazy' => false, 'dependencies' => [ diff --git a/packages/callbacks/cache/firefly-callbacks-context.php b/packages/callbacks/cache/firefly-callbacks-context.php index a8d2375d..d8aac450 100644 --- a/packages/callbacks/cache/firefly-callbacks-context.php +++ b/packages/callbacks/cache/firefly-callbacks-context.php @@ -54,6 +54,25 @@ ], ], ], + 2 => [ + 'method' => 'executionLog', + 'conditions' => [ + 0 => [ + 'type' => 'Firefly\\Context\\Condition\\Attributes\\ConditionalOnProperty', + 'args' => [ + 0 => 'firefly.callbacks.enabled', + 1 => 'true', + 2 => false, + ], + ], + 1 => [ + 'type' => 'Firefly\\Context\\Condition\\Attributes\\ConditionalOnMissingBean', + 'args' => [ + 0 => 'Firefly\\Callbacks\\Execution\\ExecutionLog', + ], + ], + ], + ], ], ], ]; diff --git a/packages/callbacks/src/CallbacksAutoConfiguration.php b/packages/callbacks/src/CallbacksAutoConfiguration.php index 2402c59b..c87f816d 100644 --- a/packages/callbacks/src/CallbacksAutoConfiguration.php +++ b/packages/callbacks/src/CallbacksAutoConfiguration.php @@ -4,6 +4,9 @@ namespace Firefly\Callbacks; +use Firefly\Callbacks\Execution\DatabaseExecutionLog; +use Firefly\Callbacks\Execution\ExecutionLog; +use Firefly\Callbacks\Execution\InMemoryExecutionLog; use Firefly\Callbacks\Subscription\DatabaseSubscriptionRegistry; use Firefly\Callbacks\Subscription\InMemorySubscriptionRegistry; use Firefly\Callbacks\Subscription\SubscriptionRegistry; @@ -44,6 +47,16 @@ public function subscriptionRegistry(CallbacksSettings $settings, Container $con : new InMemorySubscriptionRegistry($settings->subscriptions); } + #[Bean] + #[ConditionalOnProperty(name: 'firefly.callbacks.enabled', havingValue: 'true')] + #[ConditionalOnMissingBean(ExecutionLog::class)] + public function executionLog(CallbacksSettings $settings, Container $container): ExecutionLog + { + return $settings->logStore === 'database' + ? new DatabaseExecutionLog($this->connection($container, $settings->connection)) + : new InMemoryExecutionLog; + } + private function connection(Container $container, ?string $name): ConnectionInterface { $databases = $container->bound('db') ? $container->make('db') : null; diff --git a/packages/callbacks/src/Execution/CallbackExecution.php b/packages/callbacks/src/Execution/CallbackExecution.php new file mode 100644 index 00000000..a7f74bca --- /dev/null +++ b/packages/callbacks/src/Execution/CallbackExecution.php @@ -0,0 +1,77 @@ +, "eventId": …, "eventType": …, "occurredAt": , "data": } + * + * Times are unix seconds, like every other firefly table. lastError is cut to MAX_ERROR_LENGTH bytes on a UTF-8 + * character boundary, so a long transport error or response excerpt always fits the column. + */ +final readonly class CallbackExecution +{ + public const int MAX_ERROR_LENGTH = 2000; + + public function __construct( + public string $id, + public string $subscriptionId, + public string $eventId, + public string $eventType, + public string $url, + public string $body, + public CallbackStatus $status = CallbackStatus::Pending, + public int $attempts = 0, + public ?int $responseStatus = null, + public ?string $lastError = null, + public int $createdAt = 0, + public int $updatedAt = 0, + ) {} + + /** The deterministic execution id: 64 hex characters, the NUL separator keeping ("a", "bc") and ("ab", "c") apart. */ + public static function idFor(string $subscriptionId, string $eventId): string + { + return hash('sha256', $subscriptionId."\0".$eventId); + } + + public static function pending(CallbackSubscription $subscription, EventEnvelope $envelope, int $now): self + { + $id = self::idFor($subscription->id, $envelope->eventId); + $body = json_encode([ + 'id' => $id, + 'eventId' => $envelope->eventId, + 'eventType' => $envelope->eventType, + 'occurredAt' => $envelope->timestamp->format(DATE_ATOM), + 'data' => $envelope->payload, + ], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE | JSON_PRESERVE_ZERO_FRACTION); + + return new self($id, $subscription->id, $envelope->eventId, $envelope->eventType, $subscription->url, $body, CallbackStatus::Pending, 0, null, null, $now, $now); + } + + public function withOutcome(CallbackStatus $status, int $attempts, ?int $responseStatus, ?string $lastError, int $now): self + { + return new self( + $this->id, + $this->subscriptionId, + $this->eventId, + $this->eventType, + $this->url, + $this->body, + $status, + $attempts, + $responseStatus, + $lastError === null ? null : mb_strcut($lastError, 0, self::MAX_ERROR_LENGTH, 'UTF-8'), + $this->createdAt, + $now, + ); + } +} diff --git a/packages/callbacks/src/Execution/CallbackStatus.php b/packages/callbacks/src/Execution/CallbackStatus.php new file mode 100644 index 00000000..338dea83 --- /dev/null +++ b/packages/callbacks/src/Execution/CallbackStatus.php @@ -0,0 +1,13 @@ + $execution->subscriptionId, + 'event_id' => $execution->eventId, + 'event_type' => $execution->eventType, + 'url' => $execution->url, + 'body' => $execution->body, + 'status' => $execution->status->value, + 'attempts' => $execution->attempts, + 'response_status' => $execution->responseStatus, + 'last_error' => $execution->lastError, + 'updated_at' => $execution->updatedAt, + ]; + + try { + $this->connection->transaction(fn (): bool => $this->connection->table(CallbackSchema::EXECUTIONS)->insert(['id' => $execution->id, 'created_at' => $execution->createdAt] + $row)); + } catch (UniqueConstraintViolationException) { + $this->connection->table(CallbackSchema::EXECUTIONS)->where('id', $execution->id)->update($row); + } + } + + public function find(string $id): ?CallbackExecution + { + $row = $this->connection->table(CallbackSchema::EXECUTIONS)->where('id', $id)->first(); + + return is_object($row) ? self::hydrate($row) : null; + } + + public function recent(int $limit = 20, ?CallbackStatus $status = null, ?string $subscriptionId = null): array + { + if ($limit < 1) { + return []; + } + + $query = $this->connection->table(CallbackSchema::EXECUTIONS); + if ($status !== null) { + $query->where('status', $status->value); + } + if ($subscriptionId !== null) { + $query->where('subscription_id', $subscriptionId); + } + + $executions = []; + foreach ($query->orderByDesc('updated_at')->orderByDesc('id')->limit($limit)->get() as $row) { + $executions[] = self::hydrate($row); + } + + return $executions; + } + + private static function hydrate(object $row): CallbackExecution + { + $values = get_object_vars($row); + $responseStatus = $values['response_status'] ?? null; + $lastError = $values['last_error'] ?? null; + + return new CallbackExecution( + self::text($values['id'] ?? ''), + self::text($values['subscription_id'] ?? ''), + self::text($values['event_id'] ?? ''), + self::text($values['event_type'] ?? ''), + self::text($values['url'] ?? ''), + self::text($values['body'] ?? ''), + CallbackStatus::tryFrom(self::text($values['status'] ?? '')) ?? CallbackStatus::Pending, + self::integer($values['attempts'] ?? 0), + is_numeric($responseStatus) ? (int) $responseStatus : null, + is_string($lastError) ? $lastError : null, + self::integer($values['created_at'] ?? 0), + self::integer($values['updated_at'] ?? 0), + ); + } + + /** A column read back as text: drivers return strings (or ints for numeric-looking ids), never anything else here. */ + private static function text(mixed $value): string + { + return is_scalar($value) ? (string) $value : ''; + } + + /** An integer column: PDO returns ints on most drivers and numeric strings on some (MySQL without native types). */ + private static function integer(mixed $value): int + { + return is_numeric($value) ? (int) $value : 0; + } +} diff --git a/packages/callbacks/src/Execution/ExecutionLog.php b/packages/callbacks/src/Execution/ExecutionLog.php new file mode 100644 index 00000000..91705888 --- /dev/null +++ b/packages/callbacks/src/Execution/ExecutionLog.php @@ -0,0 +1,23 @@ + newest first (by updatedAt); empty when $limit is below 1 + */ + public function recent(int $limit = 20, ?CallbackStatus $status = null, ?string $subscriptionId = null): array; +} diff --git a/packages/callbacks/src/Execution/InMemoryExecutionLog.php b/packages/callbacks/src/Execution/InMemoryExecutionLog.php new file mode 100644 index 00000000..95bccdd0 --- /dev/null +++ b/packages/callbacks/src/Execution/InMemoryExecutionLog.php @@ -0,0 +1,62 @@ + */ + private array $executions = []; + + /** @var array record sequence, the tie-breaker for equal updatedAt (latest record first) */ + private array $sequence = []; + + private int $next = 0; + + public function record(CallbackExecution $execution): void + { + $existing = $this->executions[$execution->id] ?? null; + if ($existing !== null && $existing->createdAt !== $execution->createdAt) { + $execution = new CallbackExecution( + $execution->id, + $execution->subscriptionId, + $execution->eventId, + $execution->eventType, + $execution->url, + $execution->body, + $execution->status, + $execution->attempts, + $execution->responseStatus, + $execution->lastError, + $existing->createdAt, + $execution->updatedAt, + ); + } + + $this->executions[$execution->id] = $execution; + $this->sequence[$execution->id] = ++$this->next; + } + + public function find(string $id): ?CallbackExecution + { + return $this->executions[$id] ?? null; + } + + public function recent(int $limit = 20, ?CallbackStatus $status = null, ?string $subscriptionId = null): array + { + if ($limit < 1) { + return []; + } + + $matching = array_filter( + $this->executions, + static fn (CallbackExecution $e): bool => ($status === null || $e->status === $status) && ($subscriptionId === null || $e->subscriptionId === $subscriptionId), + ); + + uasort($matching, fn (CallbackExecution $a, CallbackExecution $b): int => [$b->updatedAt, $this->sequence[$b->id] ?? 0] <=> [$a->updatedAt, $this->sequence[$a->id] ?? 0]); + + return array_slice(array_values($matching), 0, $limit); + } +} diff --git a/packages/callbacks/tests/Execution/DatabaseExecutionLogBootTest.php b/packages/callbacks/tests/Execution/DatabaseExecutionLogBootTest.php new file mode 100644 index 00000000..9c084138 --- /dev/null +++ b/packages/callbacks/tests/Execution/DatabaseExecutionLogBootTest.php @@ -0,0 +1,34 @@ +toBe(0); + + $log = $this->app()->make(ExecutionLog::class); + $subscription = new CallbackSubscription('sub-1', 'orders', 'https://partner.example.com/hooks', ['order.*']); + $execution = CallbackExecution::pending($subscription, new EventEnvelope('order.created', 'orders', ['id' => 1], [], 'e-1'), 10); + $log->record($execution); + $log->record($execution->withOutcome(CallbackStatus::Delivered, 1, 200, null, 11)); + + expect($log)->toBeInstanceOf(DatabaseExecutionLog::class) + ->and($this->app()->make(SubscriptionRegistry::class))->toBeInstanceOf(InMemorySubscriptionRegistry::class) + ->and(DB::table(CallbackSchema::EXECUTIONS)->where('id', $execution->id)->value('status'))->toBe('delivered') + ->and($log->find($execution->id)?->createdAt)->toBe(10); +}); diff --git a/packages/callbacks/tests/Execution/ExecutionLogsTest.php b/packages/callbacks/tests/Execution/ExecutionLogsTest.php new file mode 100644 index 00000000..5916452f --- /dev/null +++ b/packages/callbacks/tests/Execution/ExecutionLogsTest.php @@ -0,0 +1,129 @@ +addConnection(['driver' => 'sqlite', 'database' => ':memory:', 'prefix' => '']); + $connection = $capsule->getConnection(); + CallbackSchema::create($connection->getSchemaBuilder()); + + return new DatabaseExecutionLog($connection); +} + +dataset('w4 execution logs', [ + 'memory' => [static fn (): ExecutionLog => new InMemoryExecutionLog], + 'database (sqlite)' => [static fn (): ExecutionLog => w4ExecutionsSqlite()], +]); + +function w4Subscription(string $id = 'sub-1'): CallbackSubscription +{ + return new CallbackSubscription($id, 'orders', 'https://partner.example.com/hooks', ['order.*'], 's'); +} + +it('derives one execution per subscription and event, carrying the exact body to send', function () { + $envelope = new EventEnvelope('order.created', 'orders', ['id' => 7, 'total' => 9.5], [], 'evt-1', new DateTimeImmutable('2026-10-12T10:00:00+00:00')); + $execution = CallbackExecution::pending(w4Subscription(), $envelope, 100); + + expect($execution->id)->toBe(CallbackExecution::idFor('sub-1', 'evt-1')) + ->and($execution->id)->toMatch('/^[0-9a-f]{64}$/') + ->and($execution->status)->toBe(CallbackStatus::Pending) + ->and($execution->attempts)->toBe(0) + ->and($execution->createdAt)->toBe(100) + ->and($execution->updatedAt)->toBe(100) + ->and($execution->url)->toBe('https://partner.example.com/hooks') + ->and(json_decode($execution->body, true))->toBe([ + 'id' => $execution->id, + 'eventId' => 'evt-1', + 'eventType' => 'order.created', + 'occurredAt' => '2026-10-12T10:00:00+00:00', + 'data' => ['id' => 7, 'total' => 9.5], + ]); +}); + +it('gives a redelivered envelope the same execution id and keeps distinct pairs apart', function () { + $redelivered = EventEnvelope::fromArray((new EventEnvelope('order.created', 'orders', [], [], 'evt-9'))->toArray()); + + expect(CallbackExecution::pending(w4Subscription(), $redelivered, 5)->id)->toBe(CallbackExecution::idFor('sub-1', 'evt-9')) + ->and(CallbackExecution::idFor('sub-1', 'evt-9'))->not->toBe(CallbackExecution::idFor('sub-2', 'evt-9')) + // The separator keeps (a, bc) and (ab, c) apart. + ->and(CallbackExecution::idFor('a', 'bc'))->not->toBe(CallbackExecution::idFor('ab', 'c')); +}); + +it('records, updates in place, finds and lists executions newest first with filters', function (Closure $make) { + /** @var ExecutionLog $log */ + $log = $make(); + $first = CallbackExecution::pending(w4Subscription(), new EventEnvelope('order.created', 'orders', [], [], 'e-1'), 100); + $second = CallbackExecution::pending(w4Subscription('sub-2'), new EventEnvelope('order.created', 'orders', [], [], 'e-2'), 101); + + $log->record($first); + $log->record($second); + $log->record($first->withOutcome(CallbackStatus::Failed, 3, 503, 'HTTP 503', 102)); + + expect($log->find($first->id)?->status)->toBe(CallbackStatus::Failed) + ->and($log->find($first->id)?->attempts)->toBe(3) + ->and($log->find($first->id)?->responseStatus)->toBe(503) + ->and($log->find($first->id)?->lastError)->toBe('HTTP 503') + ->and($log->find($first->id)?->createdAt)->toBe(100) + ->and($log->find($first->id)?->updatedAt)->toBe(102) + ->and($log->find($first->id)?->body)->toBe($first->body) + ->and($log->find($second->id)?->responseStatus)->toBeNull() + ->and($log->find($second->id)?->lastError)->toBeNull() + ->and(array_map(static fn (CallbackExecution $e): string => $e->eventId, $log->recent()))->toBe(['e-1', 'e-2']) + ->and(array_map(static fn (CallbackExecution $e): string => $e->eventId, $log->recent(10, CallbackStatus::Pending)))->toBe(['e-2']) + ->and(array_map(static fn (CallbackExecution $e): string => $e->eventId, $log->recent(10, null, 'sub-1')))->toBe(['e-1']) + ->and($log->recent(10, CallbackStatus::Delivered))->toBe([]) + ->and($log->recent(1))->toHaveCount(1) + ->and($log->recent(0))->toBe([]) + ->and($log->find('missing'))->toBeNull(); +})->with('w4 execution logs'); + +it('keeps the createdAt of the first record when an execution is recorded again', function (Closure $make) { + /** @var ExecutionLog $log */ + $log = $make(); + $execution = CallbackExecution::pending(w4Subscription(), new EventEnvelope('order.created', 'orders', [], [], 'e-1'), 100); + $log->record($execution); + + // A replay rebuilt from a fresh pending() carries a later createdAt; the log keeps the original. + $log->record(CallbackExecution::pending(w4Subscription(), new EventEnvelope('order.created', 'orders', [], [], 'e-1'), 200)); + + expect($log->find($execution->id)?->createdAt)->toBe(100) + ->and($log->find($execution->id)?->updatedAt)->toBe(200) + ->and($log->recent())->toHaveCount(1); +})->with('w4 execution logs'); + +it('keeps the last error short enough to store', function () { + $execution = CallbackExecution::pending(w4Subscription(), new EventEnvelope('a', 'd'), 1) + ->withOutcome(CallbackStatus::Failed, 1, null, str_repeat('x', 5000), 2); + $multibyte = CallbackExecution::pending(w4Subscription(), new EventEnvelope('a', 'd'), 1) + ->withOutcome(CallbackStatus::Failed, 1, null, str_repeat('é', 5000), 2); + + expect(strlen((string) $execution->lastError))->toBe(2000) + ->and(strlen((string) $multibyte->lastError))->toBeLessThanOrEqual(2000) + ->and(mb_check_encoding((string) $multibyte->lastError, 'UTF-8'))->toBeTrue(); +}); + +it('binds the memory log by default through the boot pipeline', function () { + $app = fireflyApplication(config: ['firefly' => ['callbacks' => ['enabled' => true]]], providers: [CallbacksServiceProvider::class, CallbacksWiringProvider::class]); + + expect($app->make(ExecutionLog::class))->toBeInstanceOf(InMemoryExecutionLog::class); +}); + +it('binds no execution log while firefly.callbacks.enabled is off', function () { + $app = fireflyApplication(config: ['firefly' => []], providers: [CallbacksServiceProvider::class, CallbacksWiringProvider::class]); + + expect($app->bound(ExecutionLog::class))->toBeFalse(); +}); diff --git a/packages/callbacks/tests/Execution/Integration/DatabaseExecutionLogIntegrationTest.php b/packages/callbacks/tests/Execution/Integration/DatabaseExecutionLogIntegrationTest.php new file mode 100644 index 00000000..6d533ee9 --- /dev/null +++ b/packages/callbacks/tests/Execution/Integration/DatabaseExecutionLogIntegrationTest.php @@ -0,0 +1,74 @@ + ['pgsql', 'FIREFLY_PG_DSN'], + 'MySQL' => ['mysql', 'FIREFLY_MYSQL_DSN'], + 'MariaDB' => ['mysql', 'FIREFLY_MARIADB_DSN'], + 'SQL Server' => ['sqlsrv', 'FIREFLY_SQLSRV_DSN'], +] as $backend => [$driver, $variable]) { + it("records a large unicode body, updates it in place and filters recent deliveries on {$backend}", function () use ($driver, $variable): void { + $connection = CallbackDatabases::connection($driver, $variable); + CallbackSchema::drop($connection->getSchemaBuilder()); + CallbackSchema::create($connection->getSchemaBuilder()); + $log = new DatabaseExecutionLog($connection); + $subscription = new CallbackSubscription('sub-1', 'orders', 'https://partner.example.com/hooks', ['order.*']); + + try { + $big = CallbackExecution::pending($subscription, new EventEnvelope('order.created', 'orders', ['note' => str_repeat('é', 40000)], [], 'e-1'), 100); + $small = CallbackExecution::pending($subscription, new EventEnvelope('order.created', 'orders', [], [], 'e-2'), 101); + $log->record($big); + $log->record($small); + $log->record($big->withOutcome(CallbackStatus::Delivered, 2, 204, null, 105)); + $log->record($big->withOutcome(CallbackStatus::Delivered, 2, 204, null, 105)); // unchanged re-record + + expect($log->find($big->id)?->body)->toBe($big->body) + ->and($log->find($big->id)?->status)->toBe(CallbackStatus::Delivered) + ->and($log->find($big->id)?->attempts)->toBe(2) + ->and($log->find($big->id)?->responseStatus)->toBe(204) + ->and($log->find($big->id)?->createdAt)->toBe(100) + ->and($log->find($big->id)?->updatedAt)->toBe(105) + ->and(array_map(static fn (CallbackExecution $e): string => $e->eventId, $log->recent()))->toBe(['e-1', 'e-2']) + ->and($log->recent(10, CallbackStatus::Pending))->toHaveCount(1) + ->and($log->recent(10, null, 'SUB-1'))->toBe([]) // ids are case-sensitive on every driver + ->and($connection->table(CallbackSchema::EXECUTIONS)->count())->toBe(2); + } finally { + CallbackSchema::drop($connection->getSchemaBuilder()); + } + })->group('integration')->skip(getenv($variable) === false, "Set {$variable} to an isolated database."); + + it("re-records an execution inside a caller's transaction and leaves that transaction usable on {$backend}", function () use ($driver, $variable): void { + $connection = CallbackDatabases::connection($driver, $variable); + CallbackSchema::drop($connection->getSchemaBuilder()); + CallbackSchema::create($connection->getSchemaBuilder()); + $log = new DatabaseExecutionLog($connection); + $subscription = new CallbackSubscription('sub-1', 'orders', 'https://partner.example.com/hooks', ['order.*']); + $execution = CallbackExecution::pending($subscription, new EventEnvelope('order.created', 'orders', [], [], 'e-1'), 100); + + try { + $log->record($execution); + $connection->transaction(function () use ($log, $execution): void { + // The INSERT hits the primary key; on PostgreSQL that would abort the caller's transaction + // unless it runs under its own savepoint. + $log->record($execution->withOutcome(CallbackStatus::Failed, 3, 503, 'HTTP 503', 110)); + $log->record(CallbackExecution::pending(new CallbackSubscription('sub-2', 'orders', 'https://partner.example.com/hooks', ['order.*']), new EventEnvelope('order.created', 'orders', [], [], 'e-1'), 111)); + }); + + expect($log->find($execution->id)?->status)->toBe(CallbackStatus::Failed) + ->and($log->find($execution->id)?->lastError)->toBe('HTTP 503') + ->and($log->find($execution->id)?->createdAt)->toBe(100) + ->and($connection->table(CallbackSchema::EXECUTIONS)->count())->toBe(2); + } finally { + CallbackSchema::drop($connection->getSchemaBuilder()); + } + })->group('integration')->skip(getenv($variable) === false, "Set {$variable} to an isolated database."); +} diff --git a/packages/callbacks/tests/Support/CallbacksDatabaseLogTestCase.php b/packages/callbacks/tests/Support/CallbacksDatabaseLogTestCase.php new file mode 100644 index 00000000..455f76b6 --- /dev/null +++ b/packages/callbacks/tests/Support/CallbacksDatabaseLogTestCase.php @@ -0,0 +1,29 @@ + */ + protected function fireflyProviders(): array + { + return [EdaServiceProvider::class, CallbacksServiceProvider::class, CallbacksWiringProvider::class]; + } + + /** @return array */ + protected function configOverrides(): array + { + return [ + 'firefly.callbacks.enabled' => true, + 'firefly.callbacks.log.store' => 'database', + ]; + } +} From 162afe305e4e21dc1385a13cf5ecf3e088aebab8 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 21:07:04 -0700 Subject: [PATCH 39/47] feat(callbacks): deliver events to subscriptions as signed POSTs with retries, back-off and a circuit breaker per endpoint --- .../cache/firefly-callbacks-components.php | 53 +++++++ .../cache/firefly-callbacks-context.php | 76 +++++++++ .../src/Boot/CallbacksWiringPass.php | 56 +++++++ .../src/CallbacksAutoConfiguration.php | 45 ++++++ .../callbacks/src/CallbacksWiringProvider.php | 3 +- .../src/Delivery/CallbackDeliverer.php | 149 ++++++++++++++++++ .../src/Delivery/CallbackDispatcher.php | 77 +++++++++ .../callbacks/src/Delivery/CallbackSender.php | 14 ++ .../callbacks/src/Delivery/CallbackSigner.php | 34 ++++ .../src/Delivery/EndpointCircuitBreakers.php | 37 +++++ .../Delivery/LaravelHttpCallbackSender.php | 32 ++++ .../src/Delivery/RetryableDeliveryFailure.php | 20 +++ .../callbacks/tests/CapstoneCallbacksTest.php | 68 ++++++++ .../tests/Delivery/CallbackDelivererTest.php | 135 ++++++++++++++++ .../tests/Delivery/CallbackDispatcherTest.php | 72 +++++++++ .../tests/Delivery/CallbackSignerTest.php | 25 +++ .../tests/Execution/ExecutionLogsTest.php | 7 +- .../tests/Fixtures/ScriptedCallbackSender.php | 31 ++++ packages/callbacks/tests/PackageBootsTest.php | 12 +- .../Support/CallbacksCapstoneTestCase.php | 56 +++++++ .../Support/CallbacksDatabaseLogTestCase.php | 9 +- 21 files changed, 1005 insertions(+), 6 deletions(-) create mode 100644 packages/callbacks/src/Boot/CallbacksWiringPass.php create mode 100644 packages/callbacks/src/Delivery/CallbackDeliverer.php create mode 100644 packages/callbacks/src/Delivery/CallbackDispatcher.php create mode 100644 packages/callbacks/src/Delivery/CallbackSender.php create mode 100644 packages/callbacks/src/Delivery/CallbackSigner.php create mode 100644 packages/callbacks/src/Delivery/EndpointCircuitBreakers.php create mode 100644 packages/callbacks/src/Delivery/LaravelHttpCallbackSender.php create mode 100644 packages/callbacks/src/Delivery/RetryableDeliveryFailure.php create mode 100644 packages/callbacks/tests/CapstoneCallbacksTest.php create mode 100644 packages/callbacks/tests/Delivery/CallbackDelivererTest.php create mode 100644 packages/callbacks/tests/Delivery/CallbackDispatcherTest.php create mode 100644 packages/callbacks/tests/Delivery/CallbackSignerTest.php create mode 100644 packages/callbacks/tests/Fixtures/ScriptedCallbackSender.php create mode 100644 packages/callbacks/tests/Support/CallbacksCapstoneTestCase.php diff --git a/packages/callbacks/cache/firefly-callbacks-components.php b/packages/callbacks/cache/firefly-callbacks-components.php index e9cd6210..073f5cc9 100644 --- a/packages/callbacks/cache/firefly-callbacks-components.php +++ b/packages/callbacks/cache/firefly-callbacks-components.php @@ -54,6 +54,59 @@ 1 => 'Illuminate\\Container\\Container', ], ], + 3 => [ + 'method' => 'callbackSender', + 'returns' => 'Firefly\\Callbacks\\Delivery\\CallbackSender', + 'name' => null, + 'scope' => 'Singleton', + 'primary' => false, + 'order' => 0, + 'lazy' => false, + 'dependencies' => [ + 0 => 'Illuminate\\Container\\Container', + ], + ], + 4 => [ + 'method' => 'callbackSigner', + 'returns' => 'Firefly\\Callbacks\\Delivery\\CallbackSigner', + 'name' => null, + 'scope' => 'Singleton', + 'primary' => false, + 'order' => 0, + 'lazy' => false, + 'dependencies' => [ + 0 => 'Firefly\\Callbacks\\CallbacksSettings', + ], + ], + 5 => [ + 'method' => 'callbackDeliverer', + 'returns' => 'Firefly\\Callbacks\\Delivery\\CallbackDeliverer', + 'name' => null, + 'scope' => 'Singleton', + 'primary' => false, + 'order' => 0, + 'lazy' => false, + 'dependencies' => [ + 0 => 'Firefly\\Callbacks\\CallbacksSettings', + 1 => 'Firefly\\Callbacks\\Delivery\\CallbackSender', + 2 => 'Firefly\\Callbacks\\Delivery\\CallbackSigner', + 3 => 'Firefly\\Resilience\\Store\\ResilienceStore', + ], + ], + 6 => [ + 'method' => 'callbackDispatcher', + 'returns' => 'Firefly\\Callbacks\\Delivery\\CallbackDispatcher', + 'name' => null, + 'scope' => 'Singleton', + 'primary' => false, + 'order' => 0, + 'lazy' => false, + 'dependencies' => [ + 0 => 'Firefly\\Callbacks\\Subscription\\SubscriptionRegistry', + 1 => 'Firefly\\Callbacks\\Execution\\ExecutionLog', + 2 => 'Firefly\\Callbacks\\Delivery\\CallbackDeliverer', + ], + ], ], 'lazy' => false, 'dependencies' => [ diff --git a/packages/callbacks/cache/firefly-callbacks-context.php b/packages/callbacks/cache/firefly-callbacks-context.php index d8aac450..ee0fd91b 100644 --- a/packages/callbacks/cache/firefly-callbacks-context.php +++ b/packages/callbacks/cache/firefly-callbacks-context.php @@ -73,6 +73,82 @@ ], ], ], + 3 => [ + 'method' => 'callbackSender', + 'conditions' => [ + 0 => [ + 'type' => 'Firefly\\Context\\Condition\\Attributes\\ConditionalOnProperty', + 'args' => [ + 0 => 'firefly.callbacks.enabled', + 1 => 'true', + 2 => false, + ], + ], + 1 => [ + 'type' => 'Firefly\\Context\\Condition\\Attributes\\ConditionalOnMissingBean', + 'args' => [ + 0 => 'Firefly\\Callbacks\\Delivery\\CallbackSender', + ], + ], + ], + ], + 4 => [ + 'method' => 'callbackSigner', + 'conditions' => [ + 0 => [ + 'type' => 'Firefly\\Context\\Condition\\Attributes\\ConditionalOnProperty', + 'args' => [ + 0 => 'firefly.callbacks.enabled', + 1 => 'true', + 2 => false, + ], + ], + 1 => [ + 'type' => 'Firefly\\Context\\Condition\\Attributes\\ConditionalOnMissingBean', + 'args' => [ + 0 => 'Firefly\\Callbacks\\Delivery\\CallbackSigner', + ], + ], + ], + ], + 5 => [ + 'method' => 'callbackDeliverer', + 'conditions' => [ + 0 => [ + 'type' => 'Firefly\\Context\\Condition\\Attributes\\ConditionalOnProperty', + 'args' => [ + 0 => 'firefly.callbacks.enabled', + 1 => 'true', + 2 => false, + ], + ], + 1 => [ + 'type' => 'Firefly\\Context\\Condition\\Attributes\\ConditionalOnMissingBean', + 'args' => [ + 0 => 'Firefly\\Callbacks\\Delivery\\CallbackDeliverer', + ], + ], + ], + ], + 6 => [ + 'method' => 'callbackDispatcher', + 'conditions' => [ + 0 => [ + 'type' => 'Firefly\\Context\\Condition\\Attributes\\ConditionalOnProperty', + 'args' => [ + 0 => 'firefly.callbacks.enabled', + 1 => 'true', + 2 => false, + ], + ], + 1 => [ + 'type' => 'Firefly\\Context\\Condition\\Attributes\\ConditionalOnMissingBean', + 'args' => [ + 0 => 'Firefly\\Callbacks\\Delivery\\CallbackDispatcher', + ], + ], + ], + ], ], ], ]; diff --git a/packages/callbacks/src/Boot/CallbacksWiringPass.php b/packages/callbacks/src/Boot/CallbacksWiringPass.php new file mode 100644 index 00000000..875af5cc --- /dev/null +++ b/packages/callbacks/src/Boot/CallbacksWiringPass.php @@ -0,0 +1,56 @@ +container; + if (! $container->bound(CallbacksSettings::class) || ! $container->bound(EventPublisher::class)) { + return; + } + + /** @var CallbacksSettings $settings */ + $settings = $container->make(CallbacksSettings::class); + if (! $settings->enabled || $settings->eventTypes === []) { + return; + } + + /** @var EventPublisher $publisher */ + $publisher = $container->make(EventPublisher::class); + foreach ($settings->eventTypes as $pattern) { + $publisher->subscribe($pattern, static function (EventEnvelope $envelope) use ($container): void { + $container->make(CallbackDispatcher::class)->dispatch($envelope); + }); + } + } +} diff --git a/packages/callbacks/src/CallbacksAutoConfiguration.php b/packages/callbacks/src/CallbacksAutoConfiguration.php index c87f816d..a908fca1 100644 --- a/packages/callbacks/src/CallbacksAutoConfiguration.php +++ b/packages/callbacks/src/CallbacksAutoConfiguration.php @@ -4,6 +4,12 @@ namespace Firefly\Callbacks; +use Firefly\Callbacks\Delivery\CallbackDeliverer; +use Firefly\Callbacks\Delivery\CallbackDispatcher; +use Firefly\Callbacks\Delivery\CallbackSender; +use Firefly\Callbacks\Delivery\CallbackSigner; +use Firefly\Callbacks\Delivery\EndpointCircuitBreakers; +use Firefly\Callbacks\Delivery\LaravelHttpCallbackSender; use Firefly\Callbacks\Execution\DatabaseExecutionLog; use Firefly\Callbacks\Execution\ExecutionLog; use Firefly\Callbacks\Execution\InMemoryExecutionLog; @@ -17,6 +23,7 @@ use Firefly\Context\Condition\Attributes\ConditionalOnMissingBean; use Firefly\Context\Condition\Attributes\ConditionalOnProperty; use Firefly\Kernel\Exception\Framework\ConfigurationException; +use Firefly\Resilience\Store\ResilienceStore; use Illuminate\Container\Container; use Illuminate\Database\ConnectionInterface; use Illuminate\Database\ConnectionResolverInterface; @@ -57,6 +64,44 @@ public function executionLog(CallbacksSettings $settings, Container $container): : new InMemoryExecutionLog; } + #[Bean] + #[ConditionalOnProperty(name: 'firefly.callbacks.enabled', havingValue: 'true')] + #[ConditionalOnMissingBean(CallbackSender::class)] + public function callbackSender(Container $container): CallbackSender + { + return new LaravelHttpCallbackSender($container); + } + + #[Bean] + #[ConditionalOnProperty(name: 'firefly.callbacks.enabled', havingValue: 'true')] + #[ConditionalOnMissingBean(CallbackSigner::class)] + public function callbackSigner(CallbacksSettings $settings): CallbackSigner + { + return new CallbackSigner($settings->signatureHeader, $settings->timestampHeader); + } + + /** + * The breaker per endpoint rides on the application's ResilienceStore — the same store, with the same + * lockless refusal, every attribute breaker uses — unless firefly.callbacks.circuit-breaker.enabled is false. + */ + #[Bean] + #[ConditionalOnProperty(name: 'firefly.callbacks.enabled', havingValue: 'true')] + #[ConditionalOnMissingBean(CallbackDeliverer::class)] + public function callbackDeliverer(CallbacksSettings $settings, CallbackSender $sender, CallbackSigner $signer, ResilienceStore $store): CallbackDeliverer + { + $breakers = $settings->breakerEnabled ? new EndpointCircuitBreakers($store, $settings->failureThreshold, $settings->waitDurationInOpen) : null; + + return new CallbackDeliverer($settings, $sender, $signer, $breakers); + } + + #[Bean] + #[ConditionalOnProperty(name: 'firefly.callbacks.enabled', havingValue: 'true')] + #[ConditionalOnMissingBean(CallbackDispatcher::class)] + public function callbackDispatcher(SubscriptionRegistry $subscriptions, ExecutionLog $log, CallbackDeliverer $deliverer): CallbackDispatcher + { + return new CallbackDispatcher($subscriptions, $log, $deliverer); + } + private function connection(Container $container, ?string $name): ConnectionInterface { $databases = $container->bound('db') ? $container->make('db') : null; diff --git a/packages/callbacks/src/CallbacksWiringProvider.php b/packages/callbacks/src/CallbacksWiringProvider.php index 44ebe989..b7f39dcd 100644 --- a/packages/callbacks/src/CallbacksWiringProvider.php +++ b/packages/callbacks/src/CallbacksWiringProvider.php @@ -4,6 +4,7 @@ namespace Firefly\Callbacks; +use Firefly\Callbacks\Boot\CallbacksWiringPass; use Firefly\Config\Config; use Firefly\Context\Boot\BootPass; use Firefly\Context\Boot\FireflyServiceProvider; @@ -22,7 +23,7 @@ final class CallbacksWiringProvider extends FireflyServiceProvider */ public function passes(): array { - return []; + return [new CallbacksWiringPass]; } public function boot(): void diff --git a/packages/callbacks/src/Delivery/CallbackDeliverer.php b/packages/callbacks/src/Delivery/CallbackDeliverer.php new file mode 100644 index 00000000..a99daedf --- /dev/null +++ b/packages/callbacks/src/Delivery/CallbackDeliverer.php @@ -0,0 +1,149 @@ +sleep = $sleep ?? static function (float $seconds): void { + if ($seconds > 0.0) { + usleep((int) round($seconds * 1_000_000)); + } + }; + $this->clock = $clock ?? static fn (): int => time(); + } + + public function deliver(CallbackSubscription $subscription, CallbackExecution $execution): CallbackExecution + { + $refusal = $this->refusal($execution->url); + if ($refusal !== null) { + return $execution->withOutcome(CallbackStatus::Failed, $execution->attempts, null, $refusal, ($this->clock)()); + } + + $maxAttempts = max(1, $subscription->maxAttempts ?? $this->settings->maxAttempts); + $attempts = $execution->attempts; + $status = null; + $error = null; + + for ($try = 1; $try <= $maxAttempts; $try++) { + $attempts++; + + try { + $status = $this->attempt($subscription, $execution); + if ($status >= 200 && $status < 300) { + return $execution->withOutcome(CallbackStatus::Delivered, $attempts, $status, null, ($this->clock)()); + } + + return $execution->withOutcome(CallbackStatus::Failed, $attempts, $status, "HTTP {$status} is not retryable", ($this->clock)()); + } catch (CircuitBreakerOpenException) { + return $execution->withOutcome(CallbackStatus::Failed, $attempts - 1, $status, 'circuit open for '.$subscription->endpoint(), ($this->clock)()); + } catch (RetryableDeliveryFailure $e) { + $status = $e->status; + $error = $e->getMessage(); + } catch (Throwable $e) { + $error = $e::class.': '.$e->getMessage(); + } + + if ($try < $maxAttempts) { + ($this->sleep)($this->delayAfter($try)); + } + } + + return $execution->withOutcome(CallbackStatus::Failed, $attempts, $status, $error, ($this->clock)()); + } + + public function delayAfter(int $failures): float + { + return max(0.0, min($this->settings->maxDelay, $this->settings->initialDelay * ($this->settings->multiplier ** max(0, $failures - 1)))); + } + + private function attempt(CallbackSubscription $subscription, CallbackExecution $execution): int + { + $headers = array_merge( + $subscription->headers, + [ + 'Content-Type' => 'application/json', + 'User-Agent' => 'LaraFly-Callbacks', + 'X-Firefly-Event' => $execution->eventType, + 'X-Firefly-Event-Id' => $execution->eventId, + 'X-Firefly-Delivery' => $execution->id, + ], + $this->signer->sign($execution->body, $subscription->secret, ($this->clock)()), + ); + + $send = function () use ($execution, $headers): int { + $status = $this->sender->send($execution->url, $execution->body, $headers, $this->settings->timeout); + if ($status === 408 || $status === 429 || $status >= 500) { + throw new RetryableDeliveryFailure($status); + } + + return $status; + }; + + return $this->breakers === null ? $send() : $this->breakers->for($subscription->endpoint())->call($send); + } + + private function refusal(string $url): ?string + { + $scheme = strtolower((string) parse_url($url, PHP_URL_SCHEME)); + $host = strtolower((string) parse_url($url, PHP_URL_HOST)); + + if (! in_array($scheme, ['http', 'https'], true) || $host === '') { + return "[{$url}] is not an http(s) URL"; + } + + if ($scheme === 'http' && $this->settings->requireHttps) { + return "[{$url}] is not https and firefly.callbacks.http.require-https is on"; + } + + if ($this->settings->allowedHosts !== []) { + foreach ($this->settings->allowedHosts as $allowed) { + if ($host === $allowed || str_ends_with($host, '.'.$allowed)) { + return null; + } + } + + return "host [{$host}] is not in firefly.callbacks.http.allowed-hosts"; + } + + return null; + } +} diff --git a/packages/callbacks/src/Delivery/CallbackDispatcher.php b/packages/callbacks/src/Delivery/CallbackDispatcher.php new file mode 100644 index 00000000..a78a0f0b --- /dev/null +++ b/packages/callbacks/src/Delivery/CallbackDispatcher.php @@ -0,0 +1,77 @@ +clock = $clock ?? static fn (): int => time(); + } + + /** + * @return list + */ + public function dispatch(EventEnvelope $envelope): array + { + $results = []; + + foreach ($this->subscriptions->matching($envelope->eventType) as $subscription) { + $existing = $this->log->find(CallbackExecution::idFor($subscription->id, $envelope->eventId)); + if ($existing !== null && $existing->status === CallbackStatus::Delivered) { + $results[] = $existing; + + continue; + } + + $execution = $existing ?? CallbackExecution::pending($subscription, $envelope, ($this->clock)()); + $this->log->record($execution); + + $execution = $this->deliverer->deliver($subscription, $execution); + $this->log->record($execution); + $results[] = $execution; + } + + return $results; + } + + /** Sends a logged execution again (`firefly:callbacks:replay`); null when its subscription no longer exists. */ + public function replay(CallbackExecution $execution): ?CallbackExecution + { + $subscription = $this->subscriptions->find($execution->subscriptionId); + if ($subscription === null) { + return null; + } + + $result = $this->deliverer->deliver($subscription, $execution); + $this->log->record($result); + + return $result; + } +} diff --git a/packages/callbacks/src/Delivery/CallbackSender.php b/packages/callbacks/src/Delivery/CallbackSender.php new file mode 100644 index 00000000..7db0f623 --- /dev/null +++ b/packages/callbacks/src/Delivery/CallbackSender.php @@ -0,0 +1,14 @@ + $headers + */ + public function send(string $url, string $body, array $headers, float $timeoutSeconds): int; +} diff --git a/packages/callbacks/src/Delivery/CallbackSigner.php b/packages/callbacks/src/Delivery/CallbackSigner.php new file mode 100644 index 00000000..037c9b99 --- /dev/null +++ b/packages/callbacks/src/Delivery/CallbackSigner.php @@ -0,0 +1,34 @@ +` and `X-Firefly-Signature: sha256=.")>`. The timestamp is inside the signature so a captured delivery cannot be replayed later + * (PyFly signs the body alone; spec Decision 12), and it is exactly what firefly/webhooks' `hmac` validator checks + * with `timestamp-header: X-Firefly-Timestamp`. Header names are configurable for receivers that expect others. + */ +final class CallbackSigner +{ + public function __construct( + private readonly string $signatureHeader = 'X-Firefly-Signature', + private readonly string $timestampHeader = 'X-Firefly-Timestamp', + ) {} + + /** + * @return array + */ + public function sign(string $body, ?string $secret, int $timestamp): array + { + if ($secret === null || $secret === '') { + return []; + } + + return [ + $this->timestampHeader => (string) $timestamp, + $this->signatureHeader => 'sha256='.hash_hmac('sha256', $timestamp.'.'.$body, $secret), + ]; + } +} diff --git a/packages/callbacks/src/Delivery/EndpointCircuitBreakers.php b/packages/callbacks/src/Delivery/EndpointCircuitBreakers.php new file mode 100644 index 00000000..7147b9d5 --- /dev/null +++ b/packages/callbacks/src/Delivery/EndpointCircuitBreakers.php @@ -0,0 +1,37 @@ +`. + */ +final class EndpointCircuitBreakers +{ + /** @var array */ + private array $breakers = []; + + public function __construct( + private readonly ResilienceStore $store, + private readonly int $failureThreshold = 5, + private readonly float $waitDurationInOpen = 30.0, + ) {} + + public function for(string $endpoint): CircuitBreaker + { + return $this->breakers[$endpoint] ??= new CircuitBreaker( + key: 'firefly:callbacks:circuit-breaker:'.sha1($endpoint), + store: $this->store, + failureThreshold: $this->failureThreshold, + windowSize: max(10, $this->failureThreshold), + waitDurationInOpen: $this->waitDurationInOpen, + ); + } +} diff --git a/packages/callbacks/src/Delivery/LaravelHttpCallbackSender.php b/packages/callbacks/src/Delivery/LaravelHttpCallbackSender.php new file mode 100644 index 00000000..af6d25fe --- /dev/null +++ b/packages/callbacks/src/Delivery/LaravelHttpCallbackSender.php @@ -0,0 +1,32 @@ +container->make(Factory::class); + + return $http->withHeaders($headers) + ->timeout($timeoutSeconds) + ->withoutRedirecting() + ->withBody($body, 'application/json') + ->post($url) + ->status(); + } +} diff --git a/packages/callbacks/src/Delivery/RetryableDeliveryFailure.php b/packages/callbacks/src/Delivery/RetryableDeliveryFailure.php new file mode 100644 index 00000000..626e73de --- /dev/null +++ b/packages/callbacks/src/Delivery/RetryableDeliveryFailure.php @@ -0,0 +1,20 @@ + Http::response('', 204)]); + + $this->events->publish('orders', 'order.created', ['id' => 7]); + + Http::assertSentCount(1); + Http::assertSent(function (Request $request): bool { + $headers = []; + foreach ($request->headers() as $name => $values) { + $first = is_array($values) ? ($values[0] ?? '') : $values; + $headers[strtolower($name)] = is_string($first) ? $first : ''; + } + + (new HmacSignatureValidator('shared-secret', 'X-Firefly-Signature', 'sha256=', 'sha256', 'hex', 'X-Firefly-Timestamp', 300.0)) + ->verify(new WebhookRequest('partner', 'POST', $request->url(), $headers, $request->body(), [], 'application/json', time())); + + return $request->url() === 'https://partner.example.com/hooks/orders' + && $request->header('X-Firefly-Event') === ['order.created'] + && w4CapstoneDeliveredData($request->body()) === ['id' => 7]; + }); + + $logged = $this->app()->make(ExecutionLog::class)->recent(); + expect($logged)->toHaveCount(1)->and($logged[0]->status)->toBe(CallbackStatus::Delivered); +}); + +it('sends nothing for an event type no subscription wants', function () { + /** @var CallbacksCapstoneTestCase $this */ + Http::fake(); + + $this->events->publish('orders', 'order.shipped', ['id' => 7]); + $this->events->publish('users', 'user.created', ['id' => 1]); + + Http::assertNothingSent(); +}); + +it('retries a failing endpoint and records the failure without disturbing the publisher', function () { + /** @var CallbacksCapstoneTestCase $this */ + Http::fake(['partner.example.com/*' => Http::sequence()->push('', 503)->push('', 503)->push('', 503)]); + + $this->events->publish('orders', 'order.created', ['id' => 8]); + + Http::assertSentCount(3); + $failed = $this->app()->make(ExecutionLog::class)->recent(10, CallbackStatus::Failed); + expect($failed)->toHaveCount(1)->and($failed[0]->attempts)->toBe(3)->and($failed[0]->responseStatus)->toBe(503); +}); diff --git a/packages/callbacks/tests/Delivery/CallbackDelivererTest.php b/packages/callbacks/tests/Delivery/CallbackDelivererTest.php new file mode 100644 index 00000000..242be1c9 --- /dev/null +++ b/packages/callbacks/tests/Delivery/CallbackDelivererTest.php @@ -0,0 +1,135 @@ +} + */ +function w4Deliverer(ScriptedCallbackSender $sender, CallbacksSettings $settings = new CallbacksSettings, ?EndpointCircuitBreakers $breakers = null): array +{ + /** @var ArrayObject $slept */ + $slept = new ArrayObject; + $deliverer = new CallbackDeliverer( + $settings, + $sender, + new CallbackSigner, + $breakers, + static function (float $seconds) use ($slept): void { + $slept[] = $seconds; + }, + static fn (): int => 1_760_000_000, + ); + + return [$deliverer, $slept]; +} + +function w4Execution(CallbackSubscription $subscription, string $eventId = 'evt-1'): CallbackExecution +{ + return CallbackExecution::pending($subscription, new EventEnvelope('order.created', 'orders', ['id' => 7], [], $eventId), 1_760_000_000); +} + +function w4Partner(?int $maxAttempts = null, string $url = 'https://partner.example.com/hooks'): CallbackSubscription +{ + return new CallbackSubscription('sub-1', 'partner', $url, ['order.*'], 'shared', ['X-Partner' => 'p1'], true, $maxAttempts); +} + +it('delivers on a 2xx with the signature, the delivery id and the subscription headers', function () { + $sender = new ScriptedCallbackSender([204]); + [$deliverer] = w4Deliverer($sender); + $execution = w4Execution(w4Partner()); + + $result = $deliverer->deliver(w4Partner(), $execution); + + expect($result->status)->toBe(CallbackStatus::Delivered) + ->and($result->attempts)->toBe(1) + ->and($result->responseStatus)->toBe(204) + ->and($sender->sent[0]['body'])->toBe($execution->body) + ->and($sender->sent[0]['headers'])->toMatchArray([ + 'X-Partner' => 'p1', + 'Content-Type' => 'application/json', + 'X-Firefly-Event' => 'order.created', + 'X-Firefly-Event-Id' => 'evt-1', + 'X-Firefly-Delivery' => $execution->id, + 'X-Firefly-Timestamp' => '1760000000', + 'X-Firefly-Signature' => 'sha256='.hash_hmac('sha256', '1760000000.'.$execution->body, 'shared'), + ]); +}); + +it('retries 5xx, 429 and transport errors with exponential back-off, then delivers', function () { + $sender = new ScriptedCallbackSender([503, 429, new RuntimeException('connection reset'), 200]); + [$deliverer, $slept] = w4Deliverer($sender, new CallbacksSettings(maxAttempts: 4)); + + $result = $deliverer->deliver(w4Partner(), w4Execution(w4Partner())); + + expect($result->status)->toBe(CallbackStatus::Delivered) + ->and($result->attempts)->toBe(4) + ->and($slept->getArrayCopy())->toBe([1.0, 2.0, 4.0]); +}); + +it('fails for good on a 4xx other than 408 and 429, without retrying', function () { + $sender = new ScriptedCallbackSender([410]); + [$deliverer] = w4Deliverer($sender); + + $result = $deliverer->deliver(w4Partner(), w4Execution(w4Partner())); + + expect($result->status)->toBe(CallbackStatus::Failed) + ->and($result->attempts)->toBe(1) + ->and($result->responseStatus)->toBe(410) + ->and($result->lastError)->toContain('not retryable') + ->and($sender->sent)->toHaveCount(1); +}); + +it('gives up after the attempts and records the last error', function () { + $sender = new ScriptedCallbackSender([500, 500, 500]); + [$deliverer] = w4Deliverer($sender); + + $result = $deliverer->deliver(w4Partner(), w4Execution(w4Partner())); + + expect($result->status)->toBe(CallbackStatus::Failed)->and($result->attempts)->toBe(3)->and($result->lastError)->toBe('HTTP 500'); +}); + +it("honours the subscription's own attempt budget", function () { + $sender = new ScriptedCallbackSender([500, 500]); + [$deliverer] = w4Deliverer($sender); + + expect($deliverer->deliver(w4Partner(1), w4Execution(w4Partner(1)))->attempts)->toBe(1); +}); + +it('refuses plain http unless allowed, and hosts outside the allow-list', function () { + $sender = new ScriptedCallbackSender; + [$httpsOnly] = w4Deliverer($sender); + [$allowList] = w4Deliverer($sender, new CallbacksSettings(allowedHosts: ['example.com'])); + + $http = w4Partner(url: 'http://partner.example.com/hooks'); + $elsewhere = w4Partner(url: 'https://evil.test/hooks'); + $sub = w4Partner(url: 'https://hooks.example.com/x'); + + expect($httpsOnly->deliver($http, w4Execution($http))->lastError)->toContain('require-https') + ->and($allowList->deliver($elsewhere, w4Execution($elsewhere))->lastError)->toContain('allowed-hosts') + ->and($allowList->deliver($sub, w4Execution($sub))->status)->toBe(CallbackStatus::Delivered) + ->and($sender->sent)->toHaveCount(1); +}); + +it('opens the circuit for an endpoint after its failures and stops calling it', function () { + $sender = new ScriptedCallbackSender([500, 500, 500]); + [$deliverer] = w4Deliverer($sender, new CallbacksSettings(maxAttempts: 1), new EndpointCircuitBreakers(new InMemoryResilienceStore, 2, 60.0)); + + $deliverer->deliver(w4Partner(), w4Execution(w4Partner(), 'e-1')); + $deliverer->deliver(w4Partner(), w4Execution(w4Partner(), 'e-2')); + $third = $deliverer->deliver(w4Partner(), w4Execution(w4Partner(), 'e-3')); + + expect($third->status)->toBe(CallbackStatus::Failed) + ->and($third->lastError)->toContain('circuit open for https://partner.example.com:443') + ->and($sender->sent)->toHaveCount(2); +}); diff --git a/packages/callbacks/tests/Delivery/CallbackDispatcherTest.php b/packages/callbacks/tests/Delivery/CallbackDispatcherTest.php new file mode 100644 index 00000000..9a024408 --- /dev/null +++ b/packages/callbacks/tests/Delivery/CallbackDispatcherTest.php @@ -0,0 +1,72 @@ +dispatch(new EventEnvelope('order.created', 'orders', ['id' => 1], [], 'evt-1')); + + expect($executions)->toHaveCount(1) + ->and($executions[0]->subscriptionId)->toBe('orders') + ->and($sender->sent[0]['url'])->toBe('https://a.example.com/hooks') + ->and($log->find($executions[0]->id)?->status)->toBe(CallbackStatus::Delivered); +}); + +it('never sends a redelivered event twice once it was delivered, but retries one that failed', function () { + $sender = new ScriptedCallbackSender([204, 500, 204]); + [$dispatcher, $log] = w4Dispatcher($sender); + $delivered = new EventEnvelope('order.created', 'orders', [], [], 'evt-1'); + $failing = new EventEnvelope('order.created', 'orders', [], [], 'evt-2'); + + $dispatcher->dispatch($delivered); + $dispatcher->dispatch($delivered); + $dispatcher->dispatch($failing); + $retried = $dispatcher->dispatch($failing); + + expect($sender->sent)->toHaveCount(3) + ->and($retried[0]->status)->toBe(CallbackStatus::Delivered) + ->and($retried[0]->attempts)->toBe(2); +}); + +it('replays a logged execution through its subscription, and declines when the subscription is gone', function () { + $sender = new ScriptedCallbackSender([500, 204]); + [$dispatcher, $log] = w4Dispatcher($sender); + $failed = $dispatcher->dispatch(new EventEnvelope('order.created', 'orders', [], [], 'evt-3'))[0]; + + $replayed = $dispatcher->replay($failed); + $orphan = new CallbackExecution('x', 'deleted', 'e', 't', 'https://a.example.com', '{}'); + + expect($failed->status)->toBe(CallbackStatus::Failed) + ->and($replayed?->status)->toBe(CallbackStatus::Delivered) + ->and($log->find($failed->id)?->status)->toBe(CallbackStatus::Delivered) + ->and($dispatcher->replay($orphan))->toBeNull(); +}); diff --git a/packages/callbacks/tests/Delivery/CallbackSignerTest.php b/packages/callbacks/tests/Delivery/CallbackSignerTest.php new file mode 100644 index 00000000..5fc1588a --- /dev/null +++ b/packages/callbacks/tests/Delivery/CallbackSignerTest.php @@ -0,0 +1,25 @@ +." with HMAC-SHA256, the scheme the webhooks hmac validator verifies', function () { + $body = '{"eventType":"order.created"}'; + $headers = (new CallbackSigner)->sign($body, 'shared', 1_760_000_000); + + expect($headers)->toBe([ + 'X-Firefly-Timestamp' => '1760000000', + 'X-Firefly-Signature' => 'sha256='.hash_hmac('sha256', '1760000000.'.$body, 'shared'), + ]); + + $receiver = new HmacSignatureValidator('shared', 'X-Firefly-Signature', 'sha256=', 'sha256', 'hex', 'X-Firefly-Timestamp', 300.0, static fn (): int => 1_760_000_100); + $receiver->verify(new WebhookRequest('partner', 'POST', 'https://x', array_change_key_case($headers), $body, [], 'application/json', 1_760_000_100)); +}); + +it('sends unsigned when the subscription has no secret', function () { + expect((new CallbackSigner)->sign('{}', null, 1))->toBe([]) + ->and((new CallbackSigner)->sign('{}', '', 1))->toBe([]); +}); diff --git a/packages/callbacks/tests/Execution/ExecutionLogsTest.php b/packages/callbacks/tests/Execution/ExecutionLogsTest.php index 5916452f..090ab56b 100644 --- a/packages/callbacks/tests/Execution/ExecutionLogsTest.php +++ b/packages/callbacks/tests/Execution/ExecutionLogsTest.php @@ -12,6 +12,7 @@ use Firefly\Callbacks\Schema\CallbackSchema; use Firefly\Callbacks\Subscription\CallbackSubscription; use Firefly\Eda\EventEnvelope; +use Firefly\Resilience\ResilienceServiceProvider; use Illuminate\Database\Capsule\Manager as Capsule; function w4ExecutionsSqlite(): DatabaseExecutionLog @@ -117,7 +118,11 @@ function w4Subscription(string $id = 'sub-1'): CallbackSubscription }); it('binds the memory log by default through the boot pipeline', function () { - $app = fireflyApplication(config: ['firefly' => ['callbacks' => ['enabled' => true]]], providers: [CallbacksServiceProvider::class, CallbacksWiringProvider::class]); + $app = fireflyApplication( + config: ['firefly' => ['callbacks' => ['enabled' => true], 'resilience' => ['method' => ['enabled' => false]]]], + providers: [ResilienceServiceProvider::class, CallbacksServiceProvider::class, CallbacksWiringProvider::class], + needs: ['cache'], + ); expect($app->make(ExecutionLog::class))->toBeInstanceOf(InMemoryExecutionLog::class); }); diff --git a/packages/callbacks/tests/Fixtures/ScriptedCallbackSender.php b/packages/callbacks/tests/Fixtures/ScriptedCallbackSender.php new file mode 100644 index 00000000..eb5e1359 --- /dev/null +++ b/packages/callbacks/tests/Fixtures/ScriptedCallbackSender.php @@ -0,0 +1,31 @@ +}> */ + public array $sent = []; + + /** + * @param list $answers + */ + public function __construct(private array $answers = []) {} + + public function send(string $url, string $body, array $headers, float $timeoutSeconds): int + { + $this->sent[] = ['url' => $url, 'body' => $body, 'headers' => $headers]; + $answer = array_shift($this->answers) ?? 204; + if ($answer instanceof Throwable) { + throw $answer; + } + + return $answer; + } +} diff --git a/packages/callbacks/tests/PackageBootsTest.php b/packages/callbacks/tests/PackageBootsTest.php index 5ab8cd38..5bc27109 100644 --- a/packages/callbacks/tests/PackageBootsTest.php +++ b/packages/callbacks/tests/PackageBootsTest.php @@ -8,6 +8,10 @@ use Firefly\Callbacks\Subscription\InMemorySubscriptionRegistry; use Firefly\Callbacks\Subscription\SubscriptionRegistry; use Firefly\Context\Boot\ApplicationContext; +use Firefly\Resilience\ResilienceServiceProvider; +use Illuminate\Cache\ArrayStore; +use Illuminate\Cache\Repository as CacheRepository; +use Illuminate\Contracts\Cache\Repository; it('boots inert when firefly.callbacks.enabled is off', function () { $app = fireflyApplication(config: ['firefly' => []], providers: [CallbacksServiceProvider::class, CallbacksWiringProvider::class]); @@ -18,8 +22,12 @@ it('binds the memory registry seeded from configuration when enabled', function () { $app = fireflyApplication( - config: ['firefly' => ['callbacks' => ['enabled' => true, 'subscriptions' => [['id' => 's-1', 'url' => 'https://partner.example.com/hooks', 'event-types' => ['order.*']]]]]], - providers: [CallbacksServiceProvider::class, CallbacksWiringProvider::class], + config: ['firefly' => [ + 'callbacks' => ['enabled' => true, 'subscriptions' => [['id' => 's-1', 'url' => 'https://partner.example.com/hooks', 'event-types' => ['order.*']]]], + 'resilience' => ['method' => ['enabled' => false]], + ]], + providers: [ResilienceServiceProvider::class, CallbacksServiceProvider::class, CallbacksWiringProvider::class], + bindings: [Repository::class => new CacheRepository(new ArrayStore)], ); $registry = $app->make(SubscriptionRegistry::class); diff --git a/packages/callbacks/tests/Support/CallbacksCapstoneTestCase.php b/packages/callbacks/tests/Support/CallbacksCapstoneTestCase.php new file mode 100644 index 00000000..01296312 --- /dev/null +++ b/packages/callbacks/tests/Support/CallbacksCapstoneTestCase.php @@ -0,0 +1,56 @@ + */ + protected function fireflyProviders(): array + { + return [ResilienceServiceProvider::class, CallbacksServiceProvider::class, CallbacksWiringProvider::class]; + } + + /** @return array */ + protected function configOverrides(): array + { + return [ + 'cache.default' => 'array', + 'firefly.resilience.method.enabled' => false, + 'firefly.callbacks.enabled' => true, + 'firefly.callbacks.event-types' => ['order.*'], + 'firefly.callbacks.retry.initial-delay' => 0, + 'firefly.callbacks.subscriptions' => [[ + 'id' => 'sub-orders', + 'name' => 'partner orders', + 'url' => 'https://partner.example.com/hooks/orders', + 'event-types' => ['order.created'], + 'secret' => 'shared-secret', + ]], + ]; + } + + protected function defineFireflyEnvironment(Application $app): void + { + $this->events = new InMemoryEventBus(new SubscriberRegistry); + $app->instance(EventPublisher::class, $this->events); + } +} diff --git a/packages/callbacks/tests/Support/CallbacksDatabaseLogTestCase.php b/packages/callbacks/tests/Support/CallbacksDatabaseLogTestCase.php index 455f76b6..01522691 100644 --- a/packages/callbacks/tests/Support/CallbacksDatabaseLogTestCase.php +++ b/packages/callbacks/tests/Support/CallbacksDatabaseLogTestCase.php @@ -7,21 +7,26 @@ use Firefly\Callbacks\CallbacksServiceProvider; use Firefly\Callbacks\CallbacksWiringProvider; use Firefly\Eda\EdaServiceProvider; +use Firefly\Resilience\ResilienceServiceProvider; use Firefly\Testing\FireflyDatabaseTestCase; -/** Callbacks enabled with the database execution log (and the memory registry), over Testbench + sqlite :memory:. */ +/** + * Callbacks enabled with the database execution log (and the memory registry), over Testbench + sqlite :memory:. + * firefly/resilience is registered because the deliverer's per-endpoint circuit breakers ride on its ResilienceStore. + */ abstract class CallbacksDatabaseLogTestCase extends FireflyDatabaseTestCase { /** @return list */ protected function fireflyProviders(): array { - return [EdaServiceProvider::class, CallbacksServiceProvider::class, CallbacksWiringProvider::class]; + return [EdaServiceProvider::class, ResilienceServiceProvider::class, CallbacksServiceProvider::class, CallbacksWiringProvider::class]; } /** @return array */ protected function configOverrides(): array { return [ + 'firefly.resilience.method.enabled' => false, 'firefly.callbacks.enabled' => true, 'firefly.callbacks.log.store' => 'database', ]; From eff3e9ae8b768991a076d340da8d17d00ad0842e Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 21:10:04 -0700 Subject: [PATCH 40/47] fix(callbacks): dispatch each event once when firefly.callbacks.event-types patterns overlap --- .../src/Boot/CallbacksWiringPass.php | 31 +++++++++++++++++-- .../tests/CapstoneOverlappingPatternsTest.php | 31 +++++++++++++++++++ .../OverlappingPatternsCapstoneTestCase.php | 20 ++++++++++++ 3 files changed, 80 insertions(+), 2 deletions(-) create mode 100644 packages/callbacks/tests/CapstoneOverlappingPatternsTest.php create mode 100644 packages/callbacks/tests/Support/OverlappingPatternsCapstoneTestCase.php diff --git a/packages/callbacks/src/Boot/CallbacksWiringPass.php b/packages/callbacks/src/Boot/CallbacksWiringPass.php index 875af5cc..f91a4a51 100644 --- a/packages/callbacks/src/Boot/CallbacksWiringPass.php +++ b/packages/callbacks/src/Boot/CallbacksWiringPass.php @@ -19,6 +19,12 @@ * resolved per envelope, never captured, so a test or an application can rebind it. Nothing is subscribed while * callbacks are off or the list is empty: a process that delivers no callbacks is not a consumer of the bus. On a * broker provider, the worker binds the destinations those events are published to through firefly.eda.destinations. + * + * The bus calls EVERY handler whose pattern matches, so overlapping patterns (`order.*` beside `order.created`, or + * `*` beside anything) would hand one envelope to the dispatcher once per match — and a failing delivery would run + * its whole retry cycle, back-off sleeps and breaker failures again each time. The patterns are therefore + * deduplicated, and each handler dispatches only when its pattern is the FIRST in the list that matches the + * envelope, so every event reaches the dispatcher exactly once however the list overlaps. */ final class CallbacksWiringPass implements BootPass { @@ -47,10 +53,31 @@ public function run(BootContext $context): void /** @var EventPublisher $publisher */ $publisher = $container->make(EventPublisher::class); - foreach ($settings->eventTypes as $pattern) { - $publisher->subscribe($pattern, static function (EventEnvelope $envelope) use ($container): void { + $patterns = array_values(array_unique($settings->eventTypes)); + foreach ($patterns as $pattern) { + $publisher->subscribe($pattern, static function (EventEnvelope $envelope) use ($container, $patterns, $pattern): void { + if (self::firstMatch($patterns, $envelope->eventType) !== $pattern) { + return; + } + $container->make(CallbackDispatcher::class)->dispatch($envelope); }); } } + + /** + * The first pattern that matches the event type, with the bus's own fnmatch() semantics, or null when none does. + * + * @param list $patterns + */ + private static function firstMatch(array $patterns, string $eventType): ?string + { + foreach ($patterns as $pattern) { + if (fnmatch($pattern, $eventType)) { + return $pattern; + } + } + + return null; + } } diff --git a/packages/callbacks/tests/CapstoneOverlappingPatternsTest.php b/packages/callbacks/tests/CapstoneOverlappingPatternsTest.php new file mode 100644 index 00000000..97e49eb8 --- /dev/null +++ b/packages/callbacks/tests/CapstoneOverlappingPatternsTest.php @@ -0,0 +1,31 @@ + Http::response('', 204)]); + + $this->events->publish('orders', 'order.created', ['id' => 7]); + + Http::assertSentCount(1); + expect($this->app()->make(ExecutionLog::class)->recent())->toHaveCount(1); +}); + +it('runs one retry cycle, not one per matching pattern, against a failing endpoint', function () { + /** @var OverlappingPatternsCapstoneTestCase $this */ + Http::fake(['partner.example.com/*' => Http::response('', 503)]); + + $this->events->publish('orders', 'order.created', ['id' => 8]); + + Http::assertSentCount(3); + $failed = $this->app()->make(ExecutionLog::class)->recent(10, CallbackStatus::Failed); + expect($failed)->toHaveCount(1)->and($failed[0]->attempts)->toBe(3); +}); diff --git a/packages/callbacks/tests/Support/OverlappingPatternsCapstoneTestCase.php b/packages/callbacks/tests/Support/OverlappingPatternsCapstoneTestCase.php new file mode 100644 index 00000000..f74c5054 --- /dev/null +++ b/packages/callbacks/tests/Support/OverlappingPatternsCapstoneTestCase.php @@ -0,0 +1,20 @@ + */ + protected function configOverrides(): array + { + return array_replace(parent::configOverrides(), [ + 'firefly.callbacks.event-types' => ['order.*', 'order.created', '*', 'order.*'], + ]); + } +} From d8c89014993d613543b4e9b728e650542509fdb4 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 21:13:58 -0700 Subject: [PATCH 41/47] feat(callbacks): firefly:callbacks:list and firefly:callbacks:replay over the execution log --- .../callbacks/src/CallbacksWiringProvider.php | 6 ++ .../src/Console/ListCallbacksCommand.php | 91 ++++++++++++++++++ .../src/Console/ReplayCallbacksCommand.php | 93 +++++++++++++++++++ .../CapstoneCallbacksCommandsDisabledTest.php | 14 +++ .../tests/CapstoneCallbacksCommandsTest.php | 65 +++++++++++++ .../tests/Support/CallbacksArtisan.php | 26 ++++++ .../CallbacksDisabledCapstoneTestCase.php | 15 +++ 7 files changed, 310 insertions(+) create mode 100644 packages/callbacks/src/Console/ListCallbacksCommand.php create mode 100644 packages/callbacks/src/Console/ReplayCallbacksCommand.php create mode 100644 packages/callbacks/tests/CapstoneCallbacksCommandsDisabledTest.php create mode 100644 packages/callbacks/tests/CapstoneCallbacksCommandsTest.php create mode 100644 packages/callbacks/tests/Support/CallbacksArtisan.php create mode 100644 packages/callbacks/tests/Support/CallbacksDisabledCapstoneTestCase.php diff --git a/packages/callbacks/src/CallbacksWiringProvider.php b/packages/callbacks/src/CallbacksWiringProvider.php index b7f39dcd..4655e6e2 100644 --- a/packages/callbacks/src/CallbacksWiringProvider.php +++ b/packages/callbacks/src/CallbacksWiringProvider.php @@ -5,6 +5,8 @@ namespace Firefly\Callbacks; use Firefly\Callbacks\Boot\CallbacksWiringPass; +use Firefly\Callbacks\Console\ListCallbacksCommand; +use Firefly\Callbacks\Console\ReplayCallbacksCommand; use Firefly\Config\Config; use Firefly\Context\Boot\BootPass; use Firefly\Context\Boot\FireflyServiceProvider; @@ -28,6 +30,10 @@ public function passes(): array public function boot(): void { + if ($this->app->runningInConsole()) { + $this->commands([ListCallbacksCommand::class, ReplayCallbacksCommand::class]); + } + $migrations = dirname(__DIR__).'/database/migrations'; $this->publishes([$migrations.'/'.self::MIGRATION => $this->app->databasePath('migrations/'.self::MIGRATION)], 'firefly-callbacks-migrations'); diff --git a/packages/callbacks/src/Console/ListCallbacksCommand.php b/packages/callbacks/src/Console/ListCallbacksCommand.php new file mode 100644 index 00000000..35222776 --- /dev/null +++ b/packages/callbacks/src/Console/ListCallbacksCommand.php @@ -0,0 +1,91 @@ +bound(SubscriptionRegistry::class) || ! $container->bound(ExecutionLog::class)) { + $this->error('firefly/callbacks is not enabled (firefly.callbacks.enabled is false).'); + + return self::FAILURE; + } + + if ((bool) $this->option('executions')) { + return $this->executions($container->make(ExecutionLog::class)); + } + + $rows = []; + foreach ($container->make(SubscriptionRegistry::class)->all() as $subscription) { + $rows[] = [$subscription->id, $subscription->name, $subscription->url, implode(', ', $subscription->eventTypes), $subscription->active ? 'yes' : 'no', $subscription->secret !== null ? 'yes' : 'no']; + } + + if ($rows === []) { + $this->info('No callback subscriptions.'); + + return self::SUCCESS; + } + + $this->table(['ID', 'Name', 'URL', 'Event types', 'Active', 'Signed'], $rows); + + return self::SUCCESS; + } + + private function executions(ExecutionLog $log): int + { + $statusOption = $this->option('status'); + $status = null; + if (is_string($statusOption) && $statusOption !== '') { + $status = CallbackStatus::tryFrom($statusOption); + if ($status === null) { + $this->error("--status must be pending, delivered or failed; got '{$statusOption}'."); + + return self::FAILURE; + } + } + + $subscription = $this->option('subscription'); + $rows = []; + foreach ($log->recent(max(1, (int) $this->option('limit')), $status, is_string($subscription) && $subscription !== '' ? $subscription : null) as $execution) { + $rows[] = [ + $execution->id, + $execution->subscriptionId, + $execution->eventType, + $execution->status->value, + (string) $execution->attempts, + $execution->responseStatus === null ? '-' : (string) $execution->responseStatus, + $execution->lastError ?? '', + gmdate('Y-m-d H:i:s', $execution->updatedAt), + ]; + } + + if ($rows === []) { + $this->info('No callback deliveries.'); + + return self::SUCCESS; + } + + $this->table(['ID', 'Subscription', 'Event type', 'Status', 'Attempts', 'HTTP', 'Last error', 'Updated (UTC)'], $rows); + + return self::SUCCESS; + } +} diff --git a/packages/callbacks/src/Console/ReplayCallbacksCommand.php b/packages/callbacks/src/Console/ReplayCallbacksCommand.php new file mode 100644 index 00000000..3b970a57 --- /dev/null +++ b/packages/callbacks/src/Console/ReplayCallbacksCommand.php @@ -0,0 +1,93 @@ +bound(CallbackDispatcher::class) || ! $container->bound(ExecutionLog::class)) { + $this->error('firefly/callbacks is not enabled (firefly.callbacks.enabled is false).'); + + return self::FAILURE; + } + + $log = $container->make(ExecutionLog::class); + $dispatcher = $container->make(CallbackDispatcher::class); + + $ids = array_values($this->argument('ids')); + $failed = (bool) $this->option('failed'); + + if ($ids === [] && ! $failed) { + $this->error('Name execution ids, or pass --failed.'); + + return self::FAILURE; + } + + /** @var array $executions */ + $executions = []; + foreach ($ids as $id) { + $execution = $log->find($id); + if ($execution === null) { + $this->warn("No execution {$id}."); + + continue; + } + $executions[$execution->id] = $execution; + } + + if ($failed) { + foreach ($log->recent(max(1, (int) $this->option('limit')), CallbackStatus::Failed) as $execution) { + $executions[$execution->id] = $execution; + } + } + + if ($executions === []) { + $this->info('Nothing to replay.'); + + return self::SUCCESS; + } + + $allDelivered = true; + foreach ($executions as $execution) { + $result = $dispatcher->replay($execution); + if ($result === null) { + $this->warn("{$execution->id}: subscription {$execution->subscriptionId} no longer exists; skipped."); + $allDelivered = false; + + continue; + } + + $line = "{$result->id}: {$result->status->value} after {$result->attempts} attempt(s)".($result->lastError !== null ? " — {$result->lastError}" : ''); + if ($result->status === CallbackStatus::Delivered) { + $this->info($line); + } else { + $this->warn($line); + $allDelivered = false; + } + } + + return $allDelivered ? self::SUCCESS : self::FAILURE; + } +} diff --git a/packages/callbacks/tests/CapstoneCallbacksCommandsDisabledTest.php b/packages/callbacks/tests/CapstoneCallbacksCommandsDisabledTest.php new file mode 100644 index 00000000..b37b50a9 --- /dev/null +++ b/packages/callbacks/tests/CapstoneCallbacksCommandsDisabledTest.php @@ -0,0 +1,14 @@ +artisan('firefly:callbacks:list'))->expectsOutputToContain('firefly.callbacks.enabled is false')->assertExitCode(1); + CallbacksArtisan::pending($this->artisan('firefly:callbacks:replay', ['--failed' => true]))->expectsOutputToContain('firefly.callbacks.enabled is false')->assertExitCode(1); +}); diff --git a/packages/callbacks/tests/CapstoneCallbacksCommandsTest.php b/packages/callbacks/tests/CapstoneCallbacksCommandsTest.php new file mode 100644 index 00000000..b368020f --- /dev/null +++ b/packages/callbacks/tests/CapstoneCallbacksCommandsTest.php @@ -0,0 +1,65 @@ +artisan('firefly:callbacks:list')) + ->expectsTable(['ID', 'Name', 'URL', 'Event types', 'Active', 'Signed'], [ + ['sub-orders', 'partner orders', 'https://partner.example.com/hooks/orders', 'order.created', 'yes', 'yes'], + ]) + ->assertExitCode(0); +}); + +it('replays the failed deliveries and lists them as delivered afterwards', function () { + /** @var CallbacksCapstoneTestCase $this */ + Http::fake(['partner.example.com/*' => Http::sequence()->push('', 503)->push('', 503)->push('', 503)->push('', 204)]); + $this->events->publish('orders', 'order.created', ['id' => 9]); + $failed = $this->app()->make(ExecutionLog::class)->recent(1, CallbackStatus::Failed)[0]; + + CallbacksArtisan::pending($this->artisan('firefly:callbacks:list', ['--executions' => true, '--status' => 'failed'])) + ->expectsOutputToContain($failed->id) + ->assertExitCode(0); + + CallbacksArtisan::pending($this->artisan('firefly:callbacks:replay', ['--failed' => true])) + ->expectsOutputToContain("{$failed->id}: delivered after 4 attempt(s)") + ->assertExitCode(0); + + expect($this->app()->make(ExecutionLog::class)->find($failed->id)?->status)->toBe(CallbackStatus::Delivered); + Http::assertSentCount(4); +}); + +it('replays named executions, reporting the ones it cannot find', function () { + /** @var CallbacksCapstoneTestCase $this */ + CallbacksArtisan::pending($this->artisan('firefly:callbacks:replay', ['ids' => ['no-such-execution']])) + ->expectsOutputToContain('No execution no-such-execution') + ->expectsOutputToContain('Nothing to replay') + ->assertExitCode(0); +}); + +it('refuses a replay with neither ids nor --failed, and an unknown status filter', function () { + /** @var CallbacksCapstoneTestCase $this */ + CallbacksArtisan::pending($this->artisan('firefly:callbacks:replay'))->expectsOutputToContain('Name execution ids, or pass --failed')->assertExitCode(1); + CallbacksArtisan::pending($this->artisan('firefly:callbacks:list', ['--executions' => true, '--status' => 'bogus']))->expectsOutputToContain('--status must be pending, delivered or failed')->assertExitCode(1); +}); + +it('exits 1 when a replayed delivery fails again', function () { + /** @var CallbacksCapstoneTestCase $this */ + Http::fake(['partner.example.com/*' => Http::response('', 500)]); + $this->events->publish('orders', 'order.created', ['id' => 10]); + + // Three attempts at publish, then the replay's: the fifth failure opens the endpoint's breaker (threshold 5), + // so the replay ends on "circuit open" without a sixth request reaching the partner. + CallbacksArtisan::pending($this->artisan('firefly:callbacks:replay', ['--failed' => true])) + ->expectsOutputToContain('failed after 5 attempt(s) — circuit open for https://partner.example.com:443') + ->assertExitCode(1); + Http::assertSentCount(5); +}); diff --git a/packages/callbacks/tests/Support/CallbacksArtisan.php b/packages/callbacks/tests/Support/CallbacksArtisan.php new file mode 100644 index 00000000..52d1082a --- /dev/null +++ b/packages/callbacks/tests/Support/CallbacksArtisan.php @@ -0,0 +1,26 @@ + */ + protected function configOverrides(): array + { + return ['firefly.callbacks.enabled' => false] + parent::configOverrides(); + } +} From 759cea4a0e9f7f27e9f4a88f8f93846367d573de Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 21:23:20 -0700 Subject: [PATCH 42/47] =?UTF-8?q?docs:=20document=20the=20W4=20ports=20?= =?UTF-8?q?=E2=80=94=20query=20cache,=20messaging=20brokers,=20resilience?= =?UTF-8?q?=20store,=20health=20indicators,=20webhooks=20and=20callbacks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 31 ++++++ README.md | 6 +- docs/README.md | 6 +- docs/index.md | 4 +- docs/modules.md | 6 +- docs/modules/actuator.md | 3 +- docs/modules/callbacks.md | 141 +++++++++++++++++++++++++++ docs/modules/cqrs.md | 68 +++++++++++-- docs/modules/eda-brokers.md | 10 ++ docs/modules/messaging.md | 43 +++++++-- docs/modules/webhooks.md | 165 ++++++++++++++++++++++++++++++++ mkdocs.yml | 2 + packages/actuator/README.md | 2 + packages/callbacks/README.md | 2 + packages/cqrs/README.md | 2 + packages/eda-kafka/README.md | 3 + packages/eda-rabbitmq/README.md | 3 + packages/messaging/README.md | 5 +- packages/resilience/README.md | 3 + packages/webhooks/README.md | 2 + skeleton/config/firefly.php | 86 +++++++++++++++++ tests/Support/DocsCodeAudit.php | 6 +- 22 files changed, 572 insertions(+), 27 deletions(-) create mode 100644 docs/modules/callbacks.md create mode 100644 docs/modules/webhooks.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f15508c..44b60bb9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,37 @@ All notable changes to LaraFly are documented here. This project uses CalVer (`Y ## [Unreleased] +### Added + +- **CQRS query cache (`firefly.cqrs.query.cache.provider=cache`).** `CacheQueryCache` on any Laravel cache store: + `#[CachedQuery(ttl:, group:)]` on a query class caches its results under a key derived from the query value + object; `#[InvalidatesQuery]` on an event class evicts one entry or a whole group when the event commits in + process or arrives over firefly/eda (`firefly.cqrs.query.cache.invalidation` covers events with no local class). + Groups are generations, so eviction works on file and database stores too. The default stays `none`. +- **Kafka and RabbitMQ for `firefly/messaging` (`firefly.messaging.provider=kafka|rabbitmq`).** Listener + containers in `firefly/eda-kafka` and `firefly/eda-rabbitmq`, built on their client factories, with retries and + exponential back-off (`firefly.messaging.listener.retry.*`), dead-letter topics / a dead-letter exchange with + diagnostic headers (`firefly.messaging.listener.dead-letter.enabled`), at-least-once commits, real consumer + groups, and `php artisan firefly:messaging:consume`. +- **Health indicators for cache stores, queue connections and Redis** (`firefly.management.endpoint.health. + {cache,queue,redis}.enabled`, all off by default). +- **`firefly/webhooks`.** `POST /webhooks/{provider}` per configured provider: HMAC, Stripe, GitHub, Twilio or + no signature, a replay window, atomic idempotency on memory, the cache or a portable `firefly_webhook_receipts` + table, publishing to firefly/eda as `webhook.{provider}.{type}`, problem+json refusals, `firefly:webhooks:prune`. +- **`firefly/callbacks`.** Subscriptions on memory or `firefly_callback_subscriptions`; deliveries of matching + firefly/eda events as HMAC-SHA256-signed POSTs (`X-Firefly-Timestamp`, `X-Firefly-Signature`), retries with + back-off, a circuit breaker per endpoint through firefly/resilience, an execution log on memory or + `firefly_callback_executions`, `firefly:callbacks:list` and `firefly:callbacks:replay`. +- **`firefly.resilience.store.provider`** — `cache` (default) or `memory`. + +### Changed + +- **A resilience store without atomic locks now refuses to boot** (`ConfigurationException` + `RESILIENCE_STORE_LOCKLESS`) instead of silently running breaker, rate-limiter and bulkhead transitions without + a critical section. Set `firefly.resilience.store.allow-lockless=true` to accept the race explicitly. A + `CacheResilienceStore` constructed by hand over such a store throws from `withLock()` unless built with + `allowLockless: true`. + ## [26.10.1] - 2026-10-06 ### Added diff --git a/README.md b/README.md index 5abf114c..b83cdde6 100644 --- a/README.md +++ b/README.md @@ -1066,7 +1066,7 @@ Full flag reference and generated-file contents: [CLI](docs/cli.md). One published library, `fireflyframework/larafly`, contains the components under `packages/*` and the bundled application skeleton. Its `replace` entries satisfy component requirements at the framework's version; -component names remain useful module boundaries, with their own test suites. The 33 +component names remain useful module boundaries, with their own test suites. The 35 [module guides](docs/modules/) below group them by concern: | Group | Module | Package(s) | @@ -1088,7 +1088,9 @@ component names remain useful module boundaries, with their own test suites. The | Data & Domain | [Transactions](docs/modules/transactional.md) — `#[Transactional]`, propagation, isolation, timeouts, `#[TransactionalEventListener]` | `firefly/data` | | Eventing & Messaging | [Event-Driven Architecture](docs/modules/eda.md) — `EventPublisher`, `#[EventListener]`, retry/DLQ | `firefly/eda` | | Eventing & Messaging | [EDA Brokers](docs/modules/eda-brokers.md) — RabbitMQ/Postgres/Kafka adapters, the same-tx outbox | `firefly/eda-rabbitmq`, `firefly/eda-postgres`, `firefly/eda-kafka` | -| Eventing & Messaging | [Messaging](docs/modules/messaging.md) — the lower-level raw-bytes broker layer | `firefly/messaging` | +| Eventing & Messaging | [Messaging](docs/modules/messaging.md) — the lower-level raw-bytes broker layer, in memory, on the Laravel queue, on Kafka or on RabbitMQ | `firefly/messaging` | +| Eventing & Messaging | [Webhooks](docs/modules/webhooks.md) — inbound webhooks: an endpoint per provider, HMAC/Stripe/GitHub/Twilio signatures, a replay window, idempotency, published to EDA | `firefly/webhooks` | +| Eventing & Messaging | [Callbacks](docs/modules/callbacks.md) — outbound callbacks: subscriptions, signed deliveries driven by events, retries, a circuit breaker per endpoint, a replayable execution log | `firefly/callbacks` | | CQRS | [CQRS](docs/modules/cqrs.md) — `CommandBus`/`QueryBus`, the domain→integration-event bridge | `firefly/cqrs` | | Security | [Security](docs/modules/security.md) — principal model, `HttpSecurity`, `#[PreAuthorize]`, JWT/OAuth2 | `firefly/security` | | Security | [OAuth2 Client](docs/modules/security-oauth2-client.md) — OpenID Connect login with presets and discovery, PKCE, id-token validation, `OidcUser`, RP-initiated logout, client credentials, `Http::oauth2Client()` | `firefly/security-oauth2-client` | diff --git a/docs/README.md b/docs/README.md index 48a43e2d..0eb09e36 100644 --- a/docs/README.md +++ b/docs/README.md @@ -28,7 +28,7 @@ ## Module Guides Every module guide lives under [`modules/`](modules/), in the ten groups `mkdocs.yml`'s navigation and the -site's own [Modules](modules.md) landing page use. All 33 are listed here, and neither the list nor the +site's own [Modules](modules.md) landing page use. All 35 are listed here, and neither the list nor the number is maintained on trust: `tests/ModuleDocumentationTest.php` fails the build if a guide is missing from the navigation, from the table below, from the root `README.md`'s table, from `docs/index.md` or from the landing page, and `tests/SiteNavigationTest.php` reads that number off `docs/modules/*.md` itself, so a @@ -76,6 +76,8 @@ thirty-fourth guide turns this page red rather than quietly making it wrong. | [Event-Driven Architecture](modules/eda.md) | `firefly/eda` — broker-backed `EventPublisher`, envelopes, retry/DLQ, in-memory + queue adapters | | [EDA Brokers](modules/eda-brokers.md) | Real broker adapters behind the `EventPublisher` port — RabbitMQ, Postgres LISTEN/NOTIFY, Kafka | | [Messaging](modules/messaging.md) | `firefly/messaging` — the lower-level raw-bytes `MessageBrokerPort`, `#[MessageListener]` | +| [Webhooks](modules/webhooks.md) | `firefly/webhooks` — an endpoint per provider, HMAC/Stripe/GitHub/Twilio signatures, replay window, idempotency, published to the event bus | +| [Callbacks](modules/callbacks.md) | `firefly/callbacks` — subscriptions, signed deliveries driven by events, retries, a circuit breaker per endpoint, a replayable log | ### CQRS @@ -123,7 +125,7 @@ thirty-fourth guide turns this page red rather than quietly making it wrong. | Document | Description | |----------|-------------| -| [Modules](modules.md) | The grouped index of all 33 module guides, and how a package wires itself | +| [Modules](modules.md) | The grouped index of all 35 module guides, and how a package wires itself | | [CLI Reference](cli.md) | Every `firefly:*` command and `make:firefly-*` generator, including the four contributed by capability packages | | [Laravel Comparison](laravel-comparison.md) | Side-by-side concept mapping for developers coming from plain Laravel | | [Versioning](versioning.md) | CalVer (`YY.MM.Patch`), no `version` field, how Packagist derives releases from tags | diff --git a/docs/index.md b/docs/index.md index 7a6ed7ba..1eb63daf 100644 --- a/docs/index.md +++ b/docs/index.md @@ -103,7 +103,7 @@ See [Installation](installation.md) for requirements and manual setup, and ## The modules -Every capability above ships in the root library; configuration selects optional features. The [module index](modules.md) lays all 33 +Every capability above ships in the root library; configuration selects optional features. The [module index](modules.md) lays all 35 guides out by concern, with a line on each saying what it is for, and the same grouping is the site's **Modules** tab. @@ -113,7 +113,7 @@ guides out by concern, with a line on each saying what it is for, and the same g | **Web & API** | [Web Layer](modules/web.md) · [Web Filters](modules/web-filters.md) · [OpenAPI](modules/openapi.md) | | **Resilience & Scheduling** | [Resilience](modules/resilience.md) · [Scheduling](modules/scheduling.md) | | **Data & Domain** | [Domain (DDD)](modules/domain.md) · [Data & Repositories](modules/data.md) · [Relational Data](modules/data-relational.md) · [Transactions](modules/transactional.md) | -| **Eventing & Messaging** | [EDA](modules/eda.md) · [EDA Brokers](modules/eda-brokers.md) · [Messaging](modules/messaging.md) | +| **Eventing & Messaging** | [EDA](modules/eda.md) · [EDA Brokers](modules/eda-brokers.md) · [Messaging](modules/messaging.md) · [Webhooks](modules/webhooks.md) · [Callbacks](modules/callbacks.md) | | **CQRS** | [Command/Query](modules/cqrs.md) | | **Security** | [Security](modules/security.md) · [OAuth2 Client](modules/security-oauth2-client.md) · [OAuth2 Authorization Server](modules/security-oauth2-server.md) | | **Operations** | [Actuator](modules/actuator.md) · [Observability](modules/observability.md) · [Tracing](modules/tracing.md) · [Logging](modules/logging.md) · [Admin Dashboard](modules/admin.md) · [Bean Graph](modules/bean-graph.md) · [Data Browser](modules/data-browser.md) · [Feature Flags](modules/feature-flags.md) | diff --git a/docs/modules.md b/docs/modules.md index 83561bf4..dbede1b7 100644 --- a/docs/modules.md +++ b/docs/modules.md @@ -1,7 +1,7 @@ # Modules LaraFly publishes one `fireflyframework/larafly` library containing the **32 component directories** under -`packages/*`, each with its own tests or metadata. The **33 guides** below explain their public surfaces. +`packages/*`, each with its own tests or metadata. The **35 guides** below explain their public surfaces. Several components carry more than one guide: `firefly/data` answers for *Data & Repositories*, *Relational Data* and *Transactions*. Laravel discovers the root library's providers, and configuration selects optional features. A bean you declare wins over the framework default. @@ -43,8 +43,8 @@ each binding in `WebServiceProvider::register()` with a `bound()` check instead,
Eventing & Messaging -[EDA](modules/eda.md) · [EDA Brokers](modules/eda-brokers.md) · [Messaging](modules/messaging.md) -One `EventPublisher` port with in-memory, queue, RabbitMQ, Postgres and Kafka adapters, a genuine same-transaction outbox, and the lower-level raw-bytes broker layer beneath it. +[EDA](modules/eda.md) · [EDA Brokers](modules/eda-brokers.md) · [Messaging](modules/messaging.md) · [Webhooks](modules/webhooks.md) · [Callbacks](modules/callbacks.md) +One `EventPublisher` port with in-memory, queue, RabbitMQ, Postgres and Kafka adapters, a genuine same-transaction outbox, and the lower-level raw-bytes broker layer beneath it. Signed inbound webhooks and outbound callbacks ride on the same bus.
diff --git a/docs/modules/actuator.md b/docs/modules/actuator.md index 6a2ad281..a6201ffc 100644 --- a/docs/modules/actuator.md +++ b/docs/modules/actuator.md @@ -37,7 +37,8 @@ always-on, and secured entirely by M11 config with zero code edge to `firefly/se `HealthIndicator { health(): Health }`; `Status` = UP/DOWN/OUT_OF_SERVICE/UNKNOWN (severity DOWN>OUT_OF_SERVICE>UP>UNKNOWN); a most-severe `StatusAggregator` (empty→UP); a bean-scan `HealthContributorRegistrar`; built-in `Ping`/`DiskSpace`/`Db` -indicators. A throwing indicator degrades to DOWN — never a 500. +indicators, and the opt-in `cache` (write/read/forget a sentinel per store), `queue` (`size()` per connection) and +`redis` (`PING` per connection) indicators. A throwing indicator degrades to DOWN — never a 500. ### Who may read the component details diff --git a/docs/modules/callbacks.md b/docs/modules/callbacks.md new file mode 100644 index 00000000..b5220e55 --- /dev/null +++ b/docs/modules/callbacks.md @@ -0,0 +1,141 @@ +# Callbacks + +`firefly/callbacks` sends webhooks. A subscription is a URL, the event types it wants and the secret its +deliveries are signed with; every [event-bus](eda.md) event whose type matches `firefly.callbacks.event-types` is +POSTed to each matching subscription, signed, retried with back-off, guarded by a circuit breaker per endpoint and +recorded in an execution log you can list and replay. Off until `firefly.callbacks.enabled` is true. + +## Turning it on + + + +```php +// config/firefly.php +return [ + 'callbacks' => [ + 'enabled' => env('FIREFLY_CALLBACKS_ENABLED', true), + 'event-types' => ['order.*'], + 'subscriptions' => [ + ['id' => 'partner-orders', 'url' => 'https://partner.example.com/hooks', 'event-types' => ['order.created', 'order.shipped'], 'secret' => env('PARTNER_CALLBACK_SECRET')], + ], + ], +]; +``` + +`event-types` is what the dispatcher subscribes to on the bus — one subscription per pattern, nothing at all while +the list is empty; each subscription then narrows it with its own patterns. On a broker provider, the worker binds +the destinations those events are published to with `firefly.eda.destinations`, as for any listener. + +## Subscriptions + +With `registry.store=memory` (the default) the subscriptions are the configured ones. With `database` they are +rows of `firefly_callback_subscriptions`, managed through the `SubscriptionRegistry` port: + + + +```php +subscriptions->save($subscription); + + return $subscription->id; + } +} +``` + +The secret is stored as given; encrypt it before `save()` if your threat model needs that. + +## What a delivery looks like + +A JSON `POST` with this body — fixed when the execution is created, so every attempt and every replay sends the +same bytes: + + + +```json +{"id": "9f2c…", "eventId": "4b1e…", "eventType": "order.created", "occurredAt": "2026-10-12T10:00:00+00:00", "data": {"id": 7}} +``` + +and these headers: `Content-Type: application/json`, `User-Agent: LaraFly-Callbacks`, `X-Firefly-Event`, +`X-Firefly-Event-Id`, `X-Firefly-Delivery` (the execution id — the receiver's idempotency key), the +subscription's own headers, and, when it has a secret, `X-Firefly-Timestamp: ` and +`X-Firefly-Signature: sha256=.")>`. The timestamp is inside the +signature, so a captured delivery cannot be replayed later; a receiver on [firefly/webhooks](webhooks.md) verifies +it with the `hmac` validator and `timestamp-header: X-Firefly-Timestamp`. + +## Retries, back-off and the circuit breaker + +- 2xx is delivered. 408, 429, 5xx and transport errors are attempted again after `retry.initial-delay` × + `retry.multiplier`^(n−1), capped at `retry.max-delay`, up to `retry.max-attempts` (a subscription's own + `max-attempts` wins). Any other status is the receiver's final answer: failed, not retried. Redirects are not + followed. +- Each endpoint (`scheme://host:port`) has a [circuit breaker](resilience.md#circuitbreaker) on the application's + resilience store: after `circuit-breaker.failure-threshold` failures it opens for + `circuit-breaker.wait-duration-in-open`, and deliveries to that endpoint fail at once with `circuit open` + instead of waiting on a partner that is down. +- URLs must be https unless `http.require-https` is false, and on a host in `http.allowed-hosts` (or a subdomain + of one) when that list is not empty. +- Attempts run where the event is consumed. On `firefly.eda.provider=memory` that is the request that published + the event — choose `queue` or a broker so deliveries run in workers. +- An event the broker redelivers maps to the same execution: one already delivered is not sent again, one that + failed is attempted again. + +## The execution log and the commands + +`log.store=memory` keeps the deliveries of the process; `database` keeps them in `firefly_callback_executions`. + + + +```bash +php artisan firefly:callbacks:list +php artisan firefly:callbacks:list --executions --status=failed --limit=50 +php artisan firefly:callbacks:replay --failed +php artisan firefly:callbacks:replay 9f2c4e… +``` + +`replay` re-sends the logged body re-signed with a fresh timestamp; its attempts keep counting, and it exits 1 if +any replay did not end delivered — schedule it and the scheduler's output says so. + + + +```bash +php artisan vendor:publish --tag=firefly-callbacks-migrations +php artisan migrate +``` + +## Configuration (`firefly.callbacks.*`) + +| Key | Default | Meaning | +|---|---|---| +| `firefly.callbacks.enabled` | `false` | Master gate. | +| `firefly.callbacks.event-types` | `[]` | Event-type patterns the dispatcher subscribes to. | +| `firefly.callbacks.registry.store` | `memory` | `memory` or `database`. | +| `firefly.callbacks.log.store` | `memory` | `memory` or `database`. | +| `firefly.callbacks.connection` | `null` | Connection for the database stores. | +| `firefly.callbacks.subscriptions` | `[]` | Subscriptions of the memory registry (`id`, `name`, `url`, `event-types`, `secret`, `headers`, `active`, `max-attempts`). | +| `firefly.callbacks.retry.max-attempts` | `3` | Attempts per delivery. | +| `firefly.callbacks.retry.initial-delay` | `1s` | First back-off. | +| `firefly.callbacks.retry.multiplier` | `2.0` | Back-off multiplier. | +| `firefly.callbacks.retry.max-delay` | `30s` | Back-off cap. | +| `firefly.callbacks.http.timeout` | `10s` | Per request. | +| `firefly.callbacks.http.require-https` | `true` | Refuse plain http URLs. | +| `firefly.callbacks.http.allowed-hosts` | `[]` | Host allow-list (suffix match); empty = any. | +| `firefly.callbacks.circuit-breaker.enabled` | `true` | One breaker per endpoint. | +| `firefly.callbacks.circuit-breaker.failure-threshold` | `5` | Failures that open it. | +| `firefly.callbacks.circuit-breaker.wait-duration-in-open` | `30s` | How long it stays open. | +| `firefly.callbacks.signature.header` | `X-Firefly-Signature` | Signature header name. | +| `firefly.callbacks.signature.timestamp-header` | `X-Firefly-Timestamp` | Timestamp header name. | diff --git a/docs/modules/cqrs.md b/docs/modules/cqrs.md index e93da570..aae8c714 100644 --- a/docs/modules/cqrs.md +++ b/docs/modules/cqrs.md @@ -79,8 +79,8 @@ All default to no-ops so the pipeline is a series of skippable collaborators: - **Tracing** — `CqrsTracing` seam (`NoOpCqrsTracing` default) wrapping validate → authorize → resolve → invoke; observability's `TracerCqrsTracing` puts an INTERNAL span around every message (see [Tracing](tracing.md)). -- **Query cache** — `QueryCache` seam (`NoOpQueryCache` = always-miss) consulted for a `Cacheable` query; - a real adapter lands with `firefly/cache`. +- **Query cache** — `QueryCache` seam, `NoOpQueryCache` (always-miss) by default; `firefly.cqrs.query.cache.provider=cache` + binds `CacheQueryCache` on the Laravel cache — see [Query cache](#query-cache). ## The domain → integration-event bridge @@ -126,13 +126,70 @@ silently), so cqrs installed without eda still boots and simply emits no integra edge): the bridge listens for `DomainEvent`s on the in-process `Context` dispatcher, never on `firefly/data`'s `AggregateTracker`/`DomainEventDispatcher` directly. -## Configuration (`firefly.cqrs.*`, snake_case) +## Query cache + +With `firefly.cqrs.query.cache.provider=cache`, a query class marked `#[CachedQuery]` is cached under a key derived +from its public properties — two equal value objects share an entry, `3` and `'3'` do not — for its `ttl:` +(falling back to `firefly.cqrs.query.cache_ttl`), in its `group:` (default: the query class). A `Cacheable` +query keeps its own `cacheKey()`. A `null` result is never cached, and a cache failure is logged and runs the +handler: the cache can make a query faster, never make it fail. A query holding private or protected state, or a +generator, cannot be keyed and simply runs uncached. + +`#[InvalidatesQuery]` on an event class says which cached results the event makes stale: the whole group, one entry +by mapping the query's properties to the event's (`with:`), or one `Cacheable` key (`key: 'statement:{account}'`). +Eviction happens when the event is dispatched after its transaction commits — a rolled-back write evicts nothing — +and when it arrives over the event bus from another process, matched by its event type (`eventType:`, default the +short class name). An envelope whose payload cannot rebuild the typed key (JSON delivers an enum as its backing +value, a date as an array) evicts the whole group instead, with a warning: serving stale data is the failure worth +preventing. + + + +```php + 'account'])] +final readonly class EntryPosted extends DomainEvent +{ + public function __construct(public string $account, public int $amount) + { + parent::__construct(); + } +} +``` + +Groups are evicted by GENERATIONS, not cache tags, so they work on every store (file and database included): +an evicted group's entries become unreachable at once and expire on their TTL — set one. The key is the value +object, so a query whose answer depends on the tenant or the user must carry them as properties. + +## Configuration (`firefly.cqrs.*`) + +The original keys are snake_case and stay so; the query-cache keys are kebab-case. | Key | Default | Meaning | |---|---|---| | `firefly.cqrs.default_destination` | `cqrs.events` | Fallback integration-event destination. | | `firefly.cqrs.event_failure_strategy` | `log` | `log` (swallow) or `raise` (re-throw) on a post-commit publish failure. | -| `firefly.cqrs.query.cache_ttl` | _(unset)_ | Default TTL in seconds passed to `QueryCache::put()` for a `Cacheable` query result. **Unset means "no TTL"**, not zero — so leave the key out unless you want one. The shipped `QueryCache` is `NoOpQueryCache`; a real store arrives with `firefly/cache`. | +| `firefly.cqrs.query.cache_ttl` | _(unset)_ | Default TTL in seconds for a cached query whose `#[CachedQuery]` names none (and for a `Cacheable` query). **Unset means "no TTL"**, not zero. | +| `firefly.cqrs.query.cache.provider` | `none` | `none` (NoOp) or `cache` (`CacheQueryCache`). Anything else refuses the boot. | +| `firefly.cqrs.query.cache.store` | `''` | Laravel cache store (`''` = default). | +| `firefly.cqrs.query.cache.prefix` | `firefly:cqrs:` | Key prefix. | +| `firefly.cqrs.query.cache.invalidation` | `[]` | Event type ⇒ list of query classes whose groups it evicts (for events with no local class). | ## Laravel comparison @@ -145,8 +202,7 @@ edge): the bridge listens for `DomainEvent`s on the in-process `Context` dispatc ## Known-latent -Read-model / projection scaffolding (→ `firefly/eventsourcing`), real authorization (→ M11), real query -cache (→ firefly/cache), CQRS metrics + health (→ M12), attribute-driven command validation, the fluent +Read-model / projection scaffolding (→ `firefly/eventsourcing`), real authorization (→ M11), CQRS metrics + health (→ M12), attribute-driven command validation, the fluent builder / distributed-tracing ergonomics, and exactly-once outbox durability (→ SP-4) are deferred and documented honestly. The `HandlerManifest` needs no hand-wiring: like every other compiled manifest it resolves to the `firefly:cache` artifact if present, otherwise an in-process scan of diff --git a/docs/modules/eda-brokers.md b/docs/modules/eda-brokers.md index 418266b6..fa1ccf7e 100644 --- a/docs/modules/eda-brokers.md +++ b/docs/modules/eda-brokers.md @@ -38,6 +38,16 @@ adapters (`EdaAutoConfiguration`, `#[Order(1000)]`); `rabbitmq`, `postgres`, and `#[ConditionalOnProperty(name: 'firefly.eda.provider', havingValue: '')]`-gated, so with any other provider selected the whole package resolves nothing and requires nothing. +## The same packages carry the messaging adapters + +`firefly/eda-kafka` and `firefly/eda-rabbitmq` also bind `firefly/messaging`'s raw-bytes `MessageBrokerPort` when +`firefly.messaging.provider` is `kafka` or `rabbitmq` — `KafkaMessageBroker` on this package's +`KafkaConsumerFactory`/`KafkaProducerFactory`, `RabbitMqMessageBroker` on its `RabbitMqConnectionFactory` — so the +two transports share one client stack: messaging's Kafka brokers default to `firefly.eda.kafka.brokers`, and its +RabbitMQ connection is `firefly.eda.rabbitmq.*`. The two provider keys are +independent: an application can publish events to Kafka and messages to RabbitMQ. See +[Messaging](messaging.md#kafka-and-rabbitmq). + ## Config keys ### RabbitMQ (`firefly/eda-rabbitmq`) diff --git a/docs/modules/messaging.md b/docs/modules/messaging.md index c5711352..aee7527b 100644 --- a/docs/modules/messaging.md +++ b/docs/modules/messaging.md @@ -2,7 +2,8 @@ `firefly/messaging` is LaraFly's lower-level raw-bytes broker layer (a pyfly `messaging` analog): a `MessageBrokerPort` moving opaque byte payloads by topic, with consumer-group semantics, an in-memory -default adapter plus a Laravel-queue async adapter, and a broker-agnostic `#[MessageListener]` attribute +default adapter plus a Laravel-queue async adapter (and [Kafka and RabbitMQ](#kafka-and-rabbitmq) listener +containers from the eda broker packages), and a broker-agnostic `#[MessageListener]` attribute carrying its **own** per-listener retry/DLQ policy. There is no envelope, no event-type pattern, and no serializer here — see [Event-Driven Architecture](eda.md) for that higher-level surface. @@ -89,8 +90,9 @@ Same enqueue-then-worker-deliver shape as eda's `QueueEventBus`, over `illuminat `QueueMessageBroker::subscribe()` accepts a `$group` argument (for interface parity) but **ignores it** — every subscriber on a topic is invoked on every delivered message, unconditionally. Genuine competing-consumer semantics (one message, one group member) require a real broker's own group - coordination, which only lands with a broker adapter at **SP-4**. Consumer-group round-robin is an - in-memory-only nicety until then — do not rely on it once `firefly.messaging.provider=queue`. + coordination, which the Kafka and RabbitMQ providers have (see [Kafka and RabbitMQ](#kafka-and-rabbitmq)). + Consumer-group round-robin is an in-memory-only nicety on the other adapters — do not rely on it once + `firefly.messaging.provider=queue`. ## `#[MessageListener]` → scanner → manifest → wiring pass @@ -167,17 +169,24 @@ rather than an `EventEnvelope`, faithful to the two ports' different payload sha | Key | Type | Default | Meaning | |---|---|---|---| -| `firefly.messaging.provider` | `memory`\|`queue` | `memory` | Selects `InMemoryMessageBroker` or `QueueMessageBroker` (`MessagingAutoConfiguration`). | +| `firefly.messaging.provider` | `memory`\|`queue`\|`kafka`\|`rabbitmq` | `memory` | Selects the broker: `InMemoryMessageBroker` or `QueueMessageBroker` (`MessagingAutoConfiguration`); `kafka`/`rabbitmq` are bound by the eda adapters (see [Kafka and RabbitMQ](#kafka-and-rabbitmq)). | | `firefly.messaging.queue.connection` | string\|null | `null` (default connection) | Queue connection `QueueMessageBroker`/`DispatchMessageJob` dispatch onto, when `provider=queue`. | | `firefly.messaging.queue.name` | string\|null | `null` (default queue) | Queue name, when `provider=queue`. | | `firefly.messaging.listener.retry.max-attempts` | int | `5` | Attempts per delivery (the first included) on a listener container broker. | | `firefly.messaging.listener.retry.initial-delay` | duration | `1s` | Exponential back-off after the first failure (seconds, or `250ms`/`30s`/`5m`/`1h`). | | `firefly.messaging.listener.retry.multiplier` | float >= 1 | `2.0` | Back-off multiplier after each further failure. | | `firefly.messaging.listener.retry.max-delay` | duration | `30s` | Back-off cap. | +| `firefly.messaging.listener.dead-letter.enabled` | bool | `true` | Dead-letter after the last attempt; false logs and skips. | +| `firefly.messaging.kafka.brokers` | string | `firefly.eda.kafka.brokers`, else `127.0.0.1:9092` | Kafka bootstrap list. | +| `firefly.messaging.kafka.group-id` | string | `firefly` | Group of a listener that names none. | +| `firefly.messaging.kafka.dlt-suffix` | string | `.DLT` | Dead-letter topic suffix. | +| `firefly.messaging.rabbitmq.exchange` | string | `firefly.messages` | Durable direct exchange. | +| `firefly.messaging.rabbitmq.dead-letter-exchange` | string | `''` (= `.dlx`) | Dead-letter exchange. | +| `firefly.messaging.rabbitmq.prefetch` | int | `10` | `basic.qos` prefetch. | There is no `firefly.messaging.retries`/`retry_delay` config key — on the `memory` and `queue` adapters retry/DLQ policy lives entirely on each `#[MessageListener]` attribute (see above), not in configuration. The -`listener.retry.*` keys apply only to a listener container broker, described next. +`listener.*` keys apply only to a listener container broker — the Kafka and RabbitMQ providers — described next. ## Listener container brokers and `firefly:messaging:consume` @@ -208,6 +217,26 @@ falling back to the in-memory broker. is always-on — installing `firefly/messaging` requires no configuration to boot with working (in-memory) defaults. +## Kafka and RabbitMQ + +`firefly.messaging.provider=kafka` (from `firefly/eda-kafka`, needs ext-rdkafka) and `=rabbitmq` (from +`firefly/eda-rabbitmq`) bind a `ListenerContainerBroker`: the same `#[MessageListener]`s, consumed by +`php artisan firefly:messaging:consume` (`--max-messages`, `--time-limit`, `--poll-timeout`, `--sleep`). + +- **Retries belong to the container.** A delivery is attempted `firefly.messaging.listener.retry.max-attempts` + times (default 5) with an exponential back-off (1s, ×2, capped at 30s); a listener with `retries: N` gets N + 1 + attempts with its `retryDelay` as a linear back-off instead. On a container, `retries: 0` means "the default". +- **Then a dead letter.** Kafka produces the record to the listener's `deadLetterTopic` or `.DLT`; RabbitMQ + republishes it to the listener's `deadLetterTopic`, or rejects it so the queue's dead-letter exchange + (`.dlx`) moves it to `.dlq`. Headers: `x-original-topic`, `x-exception` (short class), + `x-exception-message`, `x-delivery-attempts` (Kafka adds `x-dlt-reason`, `x-dlt-source-topic`, + `x-dlt-source-offset`). `firefly.messaging.listener.dead-letter.enabled=false` logs and skips instead. +- **At least once.** A record is committed / a message acknowledged only after its listeners handled it or it was + dead-lettered; a dead letter that cannot be written leaves it uncommitted and stops the worker. +- **Groups are real.** Kafka runs one consumer per group (a listener without one joins + `firefly.messaging.kafka.group-id`); RabbitMQ gives each group its own durable queue (`firefly.` without + one). The RabbitMQ connection is the event bus's — the `firefly.eda.rabbitmq.*` host, port, user, password and vhost. + ## Example The package's own consumer fixture, which the wiring test subscribes and delivers to: @@ -271,11 +300,9 @@ an oversight: These are carried-forward, documented limitations of the M9 shipment — not bugs: -- **Real brokers are OUT.** Only the in-memory and Laravel-queue adapters ship. Kafka/RabbitMQ land as their - own `firefly/messaging-` packages at **SP-4**, mirroring eda's broker-package plan. - **The queue path drops consumer-group round-robin** (see the warning above) — `QueueMessageBroker` ignores `$group` entirely and broadcasts to every subscriber on a topic. Genuine competing-consumer semantics need - a real broker's own group coordination, arriving with the SP-4 broker adapters. + a real broker's own group coordination, which the Kafka and RabbitMQ providers have. - **"Async" is queue-backed, not coroutine-based** — under the `sync` queue driver (or with no worker running), `QueueMessageBroker` delivers synchronously (minus the group-drop above), same async→sync contract as the rest of LaraFly. diff --git a/docs/modules/webhooks.md b/docs/modules/webhooks.md new file mode 100644 index 00000000..e4336bfb --- /dev/null +++ b/docs/modules/webhooks.md @@ -0,0 +1,165 @@ +# Webhooks + +`firefly/webhooks` receives webhooks. Every provider you configure gets an endpoint, `POST /webhooks/{provider}`, +that refuses an oversized body, verifies the provider's signature, holds a signed timestamp to a replay window, +answers a redelivery as a duplicate, and publishes everything it accepts to the [event bus](eda.md) — so the code +that reacts to a Stripe payment or a GitHub push is an ordinary `#[EventListener]`, with the bus's retries, dead +letters and brokers behind it. Off until `firefly.webhooks.enabled` is true. + +## Turning it on + + + +```php +// config/firefly.php +return [ + 'webhooks' => [ + 'enabled' => env('FIREFLY_WEBHOOKS_ENABLED', true), + 'idempotency' => ['store' => 'database'], + 'providers' => [ + 'stripe' => ['validator' => 'stripe', 'secret' => env('STRIPE_WEBHOOK_SECRET')], + 'github' => ['validator' => 'github', 'secret' => env('GITHUB_WEBHOOK_SECRET')], + 'twilio' => ['validator' => 'twilio', 'secret' => env('TWILIO_AUTH_TOKEN'), 'public-url' => 'https://api.example.com/webhooks/twilio'], + ], + ], +]; +``` + +The array key is the provider id and the last segment of its URL: the Stripe endpoint above is +`https://your-app/webhooks/stripe`. A signing provider without a secret refuses the boot, naming the key. + +## Providers + +| `validator` | Signature | Replay protection | Idempotency key | Event type | +|---|---|---|---|---| +| `hmac` (default) | `X-Signature: sha256=` — header, prefix, algorithm and hex/base64 encoding configurable; with `timestamp-header` the signed payload is `"."` | the timestamp, when `timestamp-header` is set | `X-Idempotency-Key` | body field `type` | +| `stripe` | `Stripe-Signature: t=…,v1=…` (any `v1` may match while a secret is rolled) | `t`, inside `tolerance` | body field `id` | body field `type` | +| `github` | `X-Hub-Signature-256: sha256=…` | none signed — the idempotency key | `X-GitHub-Delivery` | `X-GitHub-Event` | +| `twilio` | `X-Twilio-Signature`, base64 HMAC-SHA1 over the URL and the sorted form parameters; a JSON callback is checked through its `bodySHA256` parameter | none signed — the idempotency key | `I-Twilio-Idempotency-Token` | form field `EventType` | +| `none` | not verified — for a sender on a network you trust | — | `X-Idempotency-Key` | body field `type` | + +Every column is a default: `signature-header`, `signature-prefix`, `algorithm`, `encoding`, `timestamp-header`, +`tolerance`, `idempotency-header`, `idempotency-field`, `event-type-header`, `event-type-field`, `destination` and +`public-url` override it per provider. Fields are dotted paths into the decoded body (`data.object.id`). + +## What the endpoint answers + +| Status | When | Body | +|---|---|---| +| 202 | accepted and published | `{"status": "accepted", "id", "provider", "eventType", "destination", "idempotencyKey"}` | +| 200 | a delivery already accepted (same idempotency key within `idempotency.ttl`) | `{"status": "duplicate", "provider", "idempotencyKey"}` | +| 400 `WEBHOOK_PAYLOAD_INVALID` | the body says it is JSON and is not a JSON object | problem+json | +| 401 `WEBHOOK_SIGNATURE_INVALID` | missing or wrong signature | problem+json | +| 401 `WEBHOOK_TIMESTAMP_OUT_OF_TOLERANCE` | a signed timestamp outside the replay window | problem+json | +| 404 `WEBHOOK_PROVIDER_NOT_FOUND` | no such provider | problem+json | +| 413 `WEBHOOK_PAYLOAD_TOO_LARGE` | above `max-body-bytes` | problem+json | +| 503 `WEBHOOK_PUBLISH_FAILED` | verified, but the bus refused the publish — the claim is released, so the provider's retry is processed | problem+json | + +Refusals are rendered by the web layer's problem renderer whatever the sender's `Accept` header says; every 4xx +refusal carries the provider id as a `provider` member, and a 503 is also reported to the exception handler so it +reaches the logs. The route is mounted without a middleware group — no session, no CSRF +token a provider could never send. With firefly/security's `HttpSecurity` on, permit `webhooks/**`: the +signature is the authentication. + +## What reaches the event bus + +An accepted webhook is published as `webhook.{provider}.{type}` (`firefly.webhooks.event-type`) on +`webhooks.{provider}` (`firefly.webhooks.destination`, or the provider's own `destination`), with the payload +`id`, `provider`, `type`, `receivedAt`, `idempotencyKey`, `headers` (without `authorization`, +`proxy-authorization` and `cookie`) and `body` — the decoded JSON object, the form parameters, or `raw` for +anything else (`rawBase64` when it is not UTF-8). The event type is `unknown` when neither the provider's header +nor its field names one, and the envelope carries `x-webhook-provider` and `x-webhook-event-id` headers. + + + +```php +payload['body']['data']['object'] ?? []; + // record the payment … + } +} +``` + +On `firefly.eda.provider=memory` the listener runs inside the webhook request; on `queue` or a broker it runs in +a worker and the endpoint answers as soon as the event is on the bus. + +## Idempotency + +`firefly.webhooks.idempotency.store` picks where accepted delivery ids are remembered for `idempotency.ttl`: + +| Store | Backing | Notes | +|---|---|---| +| `cache` (default) | the Laravel cache store `idempotency.cache-store` names (`''` = default), `add()` with a TTL | atomic on redis, memcached, database and array stores | +| `database` | `firefly_webhook_receipts` on `idempotency.connection` | portable table, primary key (provider, key), case-sensitive keys on every driver; run the migration and prune it daily | +| `memory` | the process | one process only — under PHP-FPM it forgets every request | + + + +```bash +php artisan vendor:publish --tag=firefly-webhooks-migrations +php artisan migrate +php artisan firefly:webhooks:prune +``` + +A claim is atomic: two simultaneous deliveries of one webhook produce one 202 and one 200, never two publishes. + +## Behind a proxy + +A Twilio signature covers the exact URL Twilio called. When a load balancer rewrites the host or the scheme, set +the provider's `public-url` to the public endpoint; the query string of the request is appended to it. + +## Receiving firefly/callbacks deliveries + +A LaraFly application that sends [callbacks](callbacks.md) signs them the way the `hmac` validator checks with a +timestamp header, so the receiving side is configuration only: + + + +```php +// config/firefly.php +return [ + 'webhooks' => [ + 'enabled' => true, + 'providers' => [ + 'orders-service' => [ + 'secret' => env('ORDERS_CALLBACK_SECRET'), + 'signature-header' => 'X-Firefly-Signature', + 'timestamp-header' => 'X-Firefly-Timestamp', + 'idempotency-header' => 'X-Firefly-Delivery', + 'event-type-header' => 'X-Firefly-Event', + ], + ], + ], +]; +``` + +## Configuration (`firefly.webhooks.*`) + +| Key | Default | Meaning | +|---|---|---| +| `firefly.webhooks.enabled` | `false` | Master gate: no route, no bean while off. | +| `firefly.webhooks.path` | `webhooks` | Route prefix. | +| `firefly.webhooks.max-body-bytes` | `1048576` | Larger bodies answer 413 before any signature work. | +| `firefly.webhooks.destination` | `webhooks.{provider}` | EDA destination template. | +| `firefly.webhooks.event-type` | `webhook.{provider}.{type}` | EDA event-type template. | +| `firefly.webhooks.replay.tolerance` | `5m` | Default replay window for providers that sign a timestamp. | +| `firefly.webhooks.idempotency.store` | `cache` | `memory`, `cache` or `database`. | +| `firefly.webhooks.idempotency.ttl` | `24h` | How long an accepted delivery id is remembered. | +| `firefly.webhooks.idempotency.cache-store` | `''` | Cache store for `cache` (`''` = default). | +| `firefly.webhooks.idempotency.connection` | `null` | Connection for `database` (`null` = default). | +| `firefly.webhooks.providers..*` | — | See [Providers](#providers). | diff --git a/mkdocs.yml b/mkdocs.yml index df54cf58..925f2e4f 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -141,6 +141,8 @@ nav: - Event-Driven Architecture: modules/eda.md - EDA Brokers: modules/eda-brokers.md - Messaging: modules/messaging.md + - Webhooks: modules/webhooks.md + - Callbacks: modules/callbacks.md - CQRS: - CQRS (Command/Query): modules/cqrs.md - Security: diff --git a/packages/actuator/README.md b/packages/actuator/README.md index 0569a60d..b80fcf94 100644 --- a/packages/actuator/README.md +++ b/packages/actuator/README.md @@ -6,6 +6,8 @@ illuminate Router under `/actuator`, and a HAL index at `/actuator` listing ever first-party `HealthIndicator` SPI (`Ping`/`DiskSpace`/opt-in `Db`) feeds `/actuator/health` (+ liveness/ readiness groups, 503 on DOWN), alongside `/actuator/info`, masked `/actuator/env`, `/actuator/beans`, `/actuator/conditions`, `/actuator/mappings`, `/actuator/loggers`, and `/actuator/scheduledtasks`. +Opt-in `cache`, `queue` and `redis` health indicators probe each configured cache store, queue connection and +Redis connection. Dependency-light, always-on, and secured entirely by `firefly/security`'s `HttpSecurity` config with zero code edge — and secure-by-default: `health`/`info` ship unexposed until configured, so an unconfigured endpoint 404s rather than leaking data. diff --git a/packages/callbacks/README.md b/packages/callbacks/README.md index 5f1c59f1..7638592b 100644 --- a/packages/callbacks/README.md +++ b/packages/callbacks/README.md @@ -9,4 +9,6 @@ recorded in an execution log (memory or database). `php artisan firefly:callback deliveries; `php artisan firefly:callbacks:replay` sends logged deliveries again. Off until `firefly.callbacks.enabled` is true. +See [Callbacks](../../docs/modules/callbacks.md) for the full reference. + Apache-2.0 © Firefly Software Solutions Inc. diff --git a/packages/cqrs/README.md b/packages/cqrs/README.md index 8c50c9ec..36e5ce61 100644 --- a/packages/cqrs/README.md +++ b/packages/cqrs/README.md @@ -7,6 +7,8 @@ no-op-by-default seams (validation over the shipped `Validator`, authorization, query cache), and the domain->integration-event bridge that re-emits every committed `Firefly\Domain\DomainEvent` onto the `firefly/eda` `EventPublisher`. `#[Transactional]` handlers get transaction semantics for free from the firefly/data proxy — cqrs writes no interception code. +With `firefly.cqrs.query.cache.provider=cache` the query cache is real — `#[CachedQuery]` results on the Laravel +cache, evicted by `#[InvalidatesQuery]` events committed in process or arriving over the bus. Read models / projections are a future `firefly/eventsourcing` package. Apache-2.0 © Firefly Software Solutions Inc. diff --git a/packages/eda-kafka/README.md b/packages/eda-kafka/README.md index 38c8e0f7..d632fd83 100644 --- a/packages/eda-kafka/README.md +++ b/packages/eda-kafka/README.md @@ -13,6 +13,9 @@ Every record it writes to a `.DLT` carries `x-dlt-reason`, `x-dlt-source-topic` the same three headers, spelled the same way, that PyFly stamps on the records it dead-letters, so a topic both frameworks publish to is readable from one `kcat -C -t .DLT -f '%h'`. +The package also carries the Kafka listener container for `firefly/messaging` (`firefly.messaging.provider=kafka`), +built on the same factories. + See [EDA Brokers](../../docs/modules/eda-brokers.md) for the full adapter reference. Apache-2.0 © Firefly Software Solutions Inc. diff --git a/packages/eda-rabbitmq/README.md b/packages/eda-rabbitmq/README.md index cfb3b45d..1c2729a3 100644 --- a/packages/eda-rabbitmq/README.md +++ b/packages/eda-rabbitmq/README.md @@ -7,6 +7,9 @@ opt-in package, auto-configured behind `firefly.eda.provider=rabbitmq`, with a b (dead-letter exchange) for exhausted retries and a `RabbitMqHealthIndicator` feeding `/actuator/health`. +The package also carries the RabbitMQ listener container for `firefly/messaging` (`firefly.messaging.provider=rabbitmq`), +on the same connection factory. + See [EDA Brokers](../../docs/modules/eda-brokers.md) for the full adapter reference. Apache-2.0 © Firefly Software Solutions Inc. diff --git a/packages/messaging/README.md b/packages/messaging/README.md index 0c15018f..c436410f 100644 --- a/packages/messaging/README.md +++ b/packages/messaging/README.md @@ -3,7 +3,8 @@ LaraFly's lower-level raw-bytes messaging: a `MessageBrokerPort` (topic + bytes value + key + consumer group) with an in-memory default adapter and a Laravel-queue async adapter, a broker-agnostic `#[MessageListener]` attribute (carrying `retries`/`retryDelay`/`deadLetterTopic`) compiled into a `MessageListenerManifest` and wired -onto the broker at boot, and a bytes-aware retry / dead-letter helper. Real brokers (Kafka/RabbitMQ) ship as their -own `firefly/messaging-` packages at SP-4. +onto the broker at boot, and a bytes-aware retry / dead-letter helper. Kafka and RabbitMQ listener containers — +retries with back-off, dead-letter topics and exchanges — ship in `firefly/eda-kafka` and `firefly/eda-rabbitmq` +(`firefly.messaging.provider=kafka|rabbitmq`), consumed by `php artisan firefly:messaging:consume`. Apache-2.0 © Firefly Software Solutions Inc. diff --git a/packages/resilience/README.md b/packages/resilience/README.md index a5654876..857db693 100644 --- a/packages/resilience/README.md +++ b/packages/resilience/README.md @@ -9,4 +9,7 @@ Reflection-free and config-driven; the six also apply as attributes (`#[Retry]`, method-interceptor chain shared with `#[Transactional]`, at order 200 — inside method security, outside the transaction — gated by `firefly.resilience.method.enabled`. +The store is `firefly.resilience.store.provider` — `cache` (default) or `memory` — and a cache store without atomic +locks refuses the boot unless `firefly.resilience.store.allow-lockless` is true. + Apache-2.0 © Firefly Software Solutions Inc. diff --git a/packages/webhooks/README.md b/packages/webhooks/README.md index c58f9b87..02e05ec1 100644 --- a/packages/webhooks/README.md +++ b/packages/webhooks/README.md @@ -7,4 +7,6 @@ delivery id on an idempotency store (memory, the Laravel cache, or a portable ta and published to the `firefly/eda` event bus as `webhook.{provider}.{type}` on `webhooks.{provider}`. A refusal is a problem+json document with a stable code. Off until `firefly.webhooks.enabled` is true. +See [Webhooks](../../docs/modules/webhooks.md) for the full reference. + Apache-2.0 © Firefly Software Solutions Inc. diff --git a/skeleton/config/firefly.php b/skeleton/config/firefly.php index 74e52a1f..97014f38 100644 --- a/skeleton/config/firefly.php +++ b/skeleton/config/firefly.php @@ -2716,4 +2716,90 @@ // 'server' => ['enabled' => false, 'path' => '/feature-flags/flagd.json', 'token' => env('FIREFLY_FEATURE_FLAGS_SERVER_TOKEN', ''), 'allow-anonymous' => false], ], + /* + |-------------------------------------------------------------------------- + | Webhooks — firefly/webhooks + |-------------------------------------------------------------------------- + | + | Inbound webhooks: POST {path}/{provider} for every provider below. A delivery is size-checked, + | signature-verified, held to the replay window when the provider signs a timestamp, deduplicated on its + | delivery id, and published to firefly/eda as `event-type` on `destination` (both templates over + | {provider} and {type}). Refusals answer problem+json. See docs/modules/webhooks.md. + | + | `validator`: hmac (default) | stripe | github | twilio | none. Each kind defaults its own header names; + | override any of signature-header, signature-prefix, algorithm (sha256), encoding (hex|base64), + | timestamp-header, tolerance, idempotency-header, idempotency-field, event-type-header, event-type-field, + | destination, public-url (Twilio behind a proxy). A signing kind without a secret refuses the boot. + | + | The idempotency `store` is 'cache' by default ('memory' forgets every PHP-FPM request; 'database' is the + | firefly_webhook_receipts table — `php artisan vendor:publish --tag=firefly-webhooks-migrations`, pruned by + | `php artisan firefly:webhooks:prune`). + | + | Defaults: enabled false, path 'webhooks', max-body-bytes 1048576, destination 'webhooks.{provider}', + | event-type 'webhook.{provider}.{type}', replay.tolerance '5m', idempotency.store 'cache', + | idempotency.ttl '24h', idempotency.cache-store '', idempotency.connection null, providers []. + | + */ + + 'webhooks' => [ + 'enabled' => env('FIREFLY_WEBHOOKS_ENABLED', false), + + // 'path' => 'webhooks', + // 'max-body-bytes' => 1048576, + // 'destination' => 'webhooks.{provider}', + // 'event-type' => 'webhook.{provider}.{type}', + // 'replay' => ['tolerance' => '5m'], + // 'idempotency' => [ + // 'store' => env('FIREFLY_WEBHOOKS_IDEMPOTENCY_STORE', 'cache'), + // 'ttl' => '24h', + // 'cache-store' => '', + // 'connection' => null, + // ], + // 'providers' => [ + // 'stripe' => ['validator' => 'stripe', 'secret' => env('STRIPE_WEBHOOK_SECRET')], + // 'github' => ['validator' => 'github', 'secret' => env('GITHUB_WEBHOOK_SECRET')], + // 'twilio' => ['validator' => 'twilio', 'secret' => env('TWILIO_AUTH_TOKEN'), 'public-url' => 'https://api.example.com/webhooks/twilio'], + // 'partner' => ['secret' => env('PARTNER_WEBHOOK_SECRET'), 'timestamp-header' => 'X-Firefly-Timestamp', 'signature-header' => 'X-Firefly-Signature', 'idempotency-header' => 'X-Firefly-Delivery'], + // ], + ], + + /* + |-------------------------------------------------------------------------- + | Callbacks — firefly/callbacks + |-------------------------------------------------------------------------- + | + | Outbound callbacks: every firefly/eda event whose type matches `event-types` is POSTed to each matching + | subscription, signed with `signature.timestamp-header` and `signature.header` (sha256 HMAC over + | "."), retried with exponential back-off on 408/429/5xx and transport errors, through a + | circuit breaker per endpoint, and recorded in the execution log. Attempts run where the event is consumed: + | use the queue or a broker eda provider so they run in workers. See docs/modules/callbacks.md. + | + | registry.store / log.store: 'memory' (subscriptions come from `subscriptions`) or 'database' (the + | firefly_callback_* tables — `php artisan vendor:publish --tag=firefly-callbacks-migrations`). + | `php artisan firefly:callbacks:list` and `php artisan firefly:callbacks:replay` read the log. + | + | Defaults: enabled false, event-types [], registry.store 'memory', log.store 'memory', connection null, + | subscriptions [], retry.max-attempts 3, retry.initial-delay '1s', retry.multiplier 2.0, retry.max-delay + | '30s', http.timeout '10s', http.require-https true, http.allowed-hosts [] (any), circuit-breaker.enabled + | true, circuit-breaker.failure-threshold 5, circuit-breaker.wait-duration-in-open '30s', signature.header + | 'X-Firefly-Signature', signature.timestamp-header 'X-Firefly-Timestamp'. + | + */ + + 'callbacks' => [ + 'enabled' => env('FIREFLY_CALLBACKS_ENABLED', false), + + // 'event-types' => ['order.*', 'invoice.paid'], + // 'registry' => ['store' => env('FIREFLY_CALLBACKS_REGISTRY', 'memory')], + // 'log' => ['store' => env('FIREFLY_CALLBACKS_LOG', 'memory')], + // 'connection' => null, + // 'subscriptions' => [ + // ['id' => 'partner-orders', 'url' => 'https://partner.example.com/hooks', 'event-types' => ['order.*'], 'secret' => env('PARTNER_CALLBACK_SECRET')], + // ], + // 'retry' => ['max-attempts' => 3, 'initial-delay' => '1s', 'multiplier' => 2.0, 'max-delay' => '30s'], + // 'http' => ['timeout' => '10s', 'require-https' => true, 'allowed-hosts' => ['example.com']], + // 'circuit-breaker' => ['enabled' => true, 'failure-threshold' => 5, 'wait-duration-in-open' => '30s'], + // 'signature' => ['header' => 'X-Firefly-Signature', 'timestamp-header' => 'X-Firefly-Timestamp'], + ], + ]; diff --git a/tests/Support/DocsCodeAudit.php b/tests/Support/DocsCodeAudit.php index d3609444..118b2055 100644 --- a/tests/Support/DocsCodeAudit.php +++ b/tests/Support/DocsCodeAudit.php @@ -157,7 +157,8 @@ final class DocsCodeAudit /** * Prefixes whose NEXT segment is chosen by the application, not by the framework: a feature switch's own * name, a registration id, a provider id, a registered client's id, an in-memory user's name, an actuator - * endpoint id, a health group's name. For these the guard checks the TAIL after the user's segment + * endpoint id, a health group's name, a webhook provider id, a callback subscription's position, an event + * type mapped to the queries it invalidates. For these the guard checks the TAIL after the user's segment * instead — so `firefly.security.oauth2.client.registration.google.client-id` is accepted because some * real key ends in `.client-id` under that prefix, while a misspelt tail is still caught. * @@ -172,6 +173,8 @@ final class DocsCodeAudit * @var list */ public const array USER_KEY_PREFIXES = [ + 'firefly.callbacks.subscriptions.', + 'firefly.cqrs.query.cache.invalidation.', 'firefly.feature.', 'firefly.feature-flags.evaluators.', 'firefly.feature-flags.flags.', @@ -181,6 +184,7 @@ final class DocsCodeAudit 'firefly.security.oauth2.client.registration.', 'firefly.security.oauth2.server.clients.', 'firefly.security.users.', + 'firefly.webhooks.providers.', ]; /** @var list */ From 509b11ebd7204afbbece15c3d6d18bb9c284d0e8 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 21:34:26 -0700 Subject: [PATCH 43/47] fix(lumen): boot EdaConsumerServiceProvider in the Postgres outbox harness so firefly:eda:consume exists --- .../Integration/OutboxPostgresIntegrationTestCase.php | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/samples/lumen/tests/Integration/OutboxPostgresIntegrationTestCase.php b/samples/lumen/tests/Integration/OutboxPostgresIntegrationTestCase.php index d2d8ee32..9ea78060 100644 --- a/samples/lumen/tests/Integration/OutboxPostgresIntegrationTestCase.php +++ b/samples/lumen/tests/Integration/OutboxPostgresIntegrationTestCase.php @@ -4,6 +4,7 @@ namespace Lumen\Tests\Integration; +use Firefly\Eda\EdaConsumerServiceProvider; use Firefly\Eda\Postgres\EdaPostgresServiceProvider; use Firefly\Eda\Postgres\Outbox\OutboxSchema; use Illuminate\Contracts\Config\Repository; @@ -32,10 +33,16 @@ public static function postgresAvailable(): bool return extension_loaded('pdo_pgsql') && getenv('FIREFLY_PG_DSN') !== false; } - /** @return list> */ + /** + * Adds the postgres outbox provider plus EdaConsumerServiceProvider, which registers firefly:eda:consume. A real + * app gets the latter through package auto-discovery; Testbench does not discover the root package's providers, + * so the harness lists it explicitly or the round-trip's consumer step finds no such command. + * + * @return list> + */ protected function fireflyProviders(): array { - return [...parent::fireflyProviders(), EdaPostgresServiceProvider::class]; + return [...parent::fireflyProviders(), EdaPostgresServiceProvider::class, EdaConsumerServiceProvider::class]; } /** @return array */ From 8f2109b484d3dc3568d12472a4949b0cf44e17e9 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 21:42:06 -0700 Subject: [PATCH 44/47] fix(callbacks): refuse delivery to an inactive subscription in the deliverer, so replay no longer reaches a deactivated endpoint without --include-inactive --- .../src/Console/ReplayCallbacksCommand.php | 15 ++++++++--- .../src/Delivery/CallbackDeliverer.php | 9 ++++++- .../src/Delivery/CallbackDispatcher.php | 14 +++++++--- .../tests/CapstoneCallbacksCommandsTest.php | 26 +++++++++++++++++++ .../tests/Delivery/CallbackDelivererTest.php | 19 ++++++++++++++ .../tests/Delivery/CallbackDispatcherTest.php | 23 ++++++++++++++++ 6 files changed, 98 insertions(+), 8 deletions(-) diff --git a/packages/callbacks/src/Console/ReplayCallbacksCommand.php b/packages/callbacks/src/Console/ReplayCallbacksCommand.php index 3b970a57..facae1ba 100644 --- a/packages/callbacks/src/Console/ReplayCallbacksCommand.php +++ b/packages/callbacks/src/Console/ReplayCallbacksCommand.php @@ -14,13 +14,14 @@ /** * Sends logged deliveries again: the executions named by id, and/or with --failed the most recent failed ones. Each * is re-sent to its original URL with its ORIGINAL body, re-signed with a fresh timestamp, under the subscription's - * current attempt budget and the endpoint's breaker; its attempts keep counting. Exits 1 if any replay did not end + * current attempt budget and the endpoint's breaker; its attempts keep counting. A deactivated subscription is + * refused (and its execution left as logged) unless --include-inactive is passed, which the output announces. Exits 1 if any replay did not end * delivered, so a scheduled replay surfaces in the scheduler's output. */ final class ReplayCallbacksCommand extends Command { /** @var string */ - protected $signature = 'firefly:callbacks:replay {ids?* : execution ids, as firefly:callbacks:list --executions prints them} {--failed : replay the most recent failed deliveries} {--limit=50 : with --failed, how many}'; + protected $signature = 'firefly:callbacks:replay {ids?* : execution ids, as firefly:callbacks:list --executions prints them} {--failed : replay the most recent failed deliveries} {--limit=50 : with --failed, how many} {--include-inactive : also replay to deactivated subscriptions}'; /** @var string */ protected $description = 'Send logged callback deliveries again.'; @@ -69,9 +70,14 @@ public function handle(Container $container): int return self::SUCCESS; } + $includeInactive = (bool) $this->option('include-inactive'); + if ($includeInactive) { + $this->warn('--include-inactive: inactive subscriptions are replayed too.'); + } + $allDelivered = true; foreach ($executions as $execution) { - $result = $dispatcher->replay($execution); + $result = $dispatcher->replay($execution, $includeInactive); if ($result === null) { $this->warn("{$execution->id}: subscription {$execution->subscriptionId} no longer exists; skipped."); $allDelivered = false; @@ -80,6 +86,9 @@ public function handle(Container $container): int } $line = "{$result->id}: {$result->status->value} after {$result->attempts} attempt(s)".($result->lastError !== null ? " — {$result->lastError}" : ''); + if (! $includeInactive && $result->lastError === "subscription [{$execution->subscriptionId}] is inactive") { + $line .= '; pass --include-inactive to replay it anyway'; + } if ($result->status === CallbackStatus::Delivered) { $this->info($line); } else { diff --git a/packages/callbacks/src/Delivery/CallbackDeliverer.php b/packages/callbacks/src/Delivery/CallbackDeliverer.php index a99daedf..b5d36dee 100644 --- a/packages/callbacks/src/Delivery/CallbackDeliverer.php +++ b/packages/callbacks/src/Delivery/CallbackDeliverer.php @@ -15,6 +15,9 @@ /** * Sends one execution to one subscription and answers how it ended — it never throws. * + * * An inactive subscription is refused before anything is sent: failed, "subscription [id] is inactive". This is + * the one delivery path, so dispatch and replay alike fail closed; only a caller that explicitly includes + * inactive subscriptions (`firefly:callbacks:replay --include-inactive`) gets past it. * * The URL must be http(s) — https unless firefly.callbacks.http.require-https is false — and, when * firefly.callbacks.http.allowed-hosts names hosts, on one of them or a subdomain (PyFly's authorized domains: * the guard against a subscription aimed at an internal address). @@ -51,8 +54,12 @@ public function __construct( $this->clock = $clock ?? static fn (): int => time(); } - public function deliver(CallbackSubscription $subscription, CallbackExecution $execution): CallbackExecution + public function deliver(CallbackSubscription $subscription, CallbackExecution $execution, bool $includeInactive = false): CallbackExecution { + if (! $subscription->active && ! $includeInactive) { + return $execution->withOutcome(CallbackStatus::Failed, $execution->attempts, null, "subscription [{$subscription->id}] is inactive", ($this->clock)()); + } + $refusal = $this->refusal($execution->url); if ($refusal !== null) { return $execution->withOutcome(CallbackStatus::Failed, $execution->attempts, null, $refusal, ($this->clock)()); diff --git a/packages/callbacks/src/Delivery/CallbackDispatcher.php b/packages/callbacks/src/Delivery/CallbackDispatcher.php index a78a0f0b..d60a4d4c 100644 --- a/packages/callbacks/src/Delivery/CallbackDispatcher.php +++ b/packages/callbacks/src/Delivery/CallbackDispatcher.php @@ -61,16 +61,22 @@ public function dispatch(EventEnvelope $envelope): array return $results; } - /** Sends a logged execution again (`firefly:callbacks:replay`); null when its subscription no longer exists. */ - public function replay(CallbackExecution $execution): ?CallbackExecution + /** + * Sends a logged execution again (`firefly:callbacks:replay`); null when its subscription no longer exists. A + * deactivated subscription is refused by the deliverer (failed, "is inactive") and the log is left as it was, + * unless $includeInactive is set. + */ + public function replay(CallbackExecution $execution, bool $includeInactive = false): ?CallbackExecution { $subscription = $this->subscriptions->find($execution->subscriptionId); if ($subscription === null) { return null; } - $result = $this->deliverer->deliver($subscription, $execution); - $this->log->record($result); + $result = $this->deliverer->deliver($subscription, $execution, $includeInactive); + if ($subscription->active || $includeInactive) { + $this->log->record($result); + } return $result; } diff --git a/packages/callbacks/tests/CapstoneCallbacksCommandsTest.php b/packages/callbacks/tests/CapstoneCallbacksCommandsTest.php index b368020f..0a838b5f 100644 --- a/packages/callbacks/tests/CapstoneCallbacksCommandsTest.php +++ b/packages/callbacks/tests/CapstoneCallbacksCommandsTest.php @@ -4,6 +4,8 @@ use Firefly\Callbacks\Execution\CallbackStatus; use Firefly\Callbacks\Execution\ExecutionLog; +use Firefly\Callbacks\Subscription\CallbackSubscription; +use Firefly\Callbacks\Subscription\SubscriptionRegistry; use Firefly\Callbacks\Tests\Support\CallbacksArtisan; use Firefly\Callbacks\Tests\Support\CallbacksCapstoneTestCase; use Illuminate\Support\Facades\Http; @@ -63,3 +65,27 @@ ->assertExitCode(1); Http::assertSentCount(5); }); + +it('refuses to replay to a deactivated subscription unless --include-inactive is passed, and says so', function () { + /** @var CallbacksCapstoneTestCase $this */ + Http::fake(['partner.example.com/*' => Http::sequence()->push('', 503)->push('', 503)->push('', 503)->push('', 204)]); + $this->events->publish('orders', 'order.created', ['id' => 11]); + $failed = $this->app()->make(ExecutionLog::class)->recent(1, CallbackStatus::Failed)[0]; + + $registry = $this->app()->make(SubscriptionRegistry::class); + $current = $registry->find('sub-orders'); + expect($current)->not->toBeNull(); + /** @var CallbackSubscription $current */ + $registry->save(new CallbackSubscription($current->id, $current->name, $current->url, $current->eventTypes, $current->secret, $current->headers, false, $current->maxAttempts)); + + CallbacksArtisan::pending($this->artisan('firefly:callbacks:replay', ['--failed' => true])) + ->expectsOutputToContain('subscription [sub-orders] is inactive; pass --include-inactive to replay it anyway') + ->assertExitCode(1); + Http::assertSentCount(3); + + CallbacksArtisan::pending($this->artisan('firefly:callbacks:replay', ['ids' => [$failed->id], '--include-inactive' => true])) + ->expectsOutputToContain('--include-inactive: inactive subscriptions are replayed too') + ->expectsOutputToContain("{$failed->id}: delivered after 4 attempt(s)") + ->assertExitCode(0); + Http::assertSentCount(4); +}); diff --git a/packages/callbacks/tests/Delivery/CallbackDelivererTest.php b/packages/callbacks/tests/Delivery/CallbackDelivererTest.php index 242be1c9..bc21ccc2 100644 --- a/packages/callbacks/tests/Delivery/CallbackDelivererTest.php +++ b/packages/callbacks/tests/Delivery/CallbackDelivererTest.php @@ -133,3 +133,22 @@ function w4Partner(?int $maxAttempts = null, string $url = 'https://partner.exam ->and($third->lastError)->toContain('circuit open for https://partner.example.com:443') ->and($sender->sent)->toHaveCount(2); }); + +it('refuses to deliver to an inactive subscription without sending anything, unless the caller includes inactive ones', function () { + $sender = new ScriptedCallbackSender([204]); + [$deliverer] = w4Deliverer($sender); + $inactive = new CallbackSubscription('sub-1', 'partner', 'https://partner.example.com/hooks', ['order.*'], 'shared', [], false); + $execution = w4Execution($inactive); + + $refused = $deliverer->deliver($inactive, $execution); + + expect($refused->status)->toBe(CallbackStatus::Failed) + ->and($refused->attempts)->toBe(0) + ->and($refused->lastError)->toContain('subscription [sub-1] is inactive') + ->and($sender->sent)->toBe([]); + + $forced = $deliverer->deliver($inactive, $execution, includeInactive: true); + + expect($forced->status)->toBe(CallbackStatus::Delivered) + ->and($sender->sent)->toHaveCount(1); +}); diff --git a/packages/callbacks/tests/Delivery/CallbackDispatcherTest.php b/packages/callbacks/tests/Delivery/CallbackDispatcherTest.php index 9a024408..5dfe0fb5 100644 --- a/packages/callbacks/tests/Delivery/CallbackDispatcherTest.php +++ b/packages/callbacks/tests/Delivery/CallbackDispatcherTest.php @@ -70,3 +70,26 @@ function w4Dispatcher(ScriptedCallbackSender $sender): array ->and($log->find($failed->id)?->status)->toBe(CallbackStatus::Delivered) ->and($dispatcher->replay($orphan))->toBeNull(); }); + +it('refuses to replay to a deactivated subscription and leaves the log untouched, unless inactive ones are included', function () { + $sender = new ScriptedCallbackSender([500, 204]); + $subscriptions = new InMemorySubscriptionRegistry([new CallbackSubscription('orders', 'orders', 'https://a.example.com/hooks', ['order.*'], 's')]); + $log = new InMemoryExecutionLog; + $deliverer = new CallbackDeliverer(new CallbacksSettings(maxAttempts: 1), $sender, new CallbackSigner, null, static function (float $s): void {}); + $dispatcher = new CallbackDispatcher($subscriptions, $log, $deliverer); + $failed = $dispatcher->dispatch(new EventEnvelope('order.created', 'orders', [], [], 'evt-4'))[0]; + + $subscriptions->save(new CallbackSubscription('orders', 'orders', 'https://a.example.com/hooks', ['order.*'], 's', [], false)); + $refused = $dispatcher->replay($failed); + + expect($refused?->status)->toBe(CallbackStatus::Failed) + ->and($refused?->lastError)->toContain('is inactive') + ->and($sender->sent)->toHaveCount(1) + ->and($log->find($failed->id))->toEqual($failed); + + $forced = $dispatcher->replay($failed, includeInactive: true); + + expect($forced?->status)->toBe(CallbackStatus::Delivered) + ->and($sender->sent)->toHaveCount(2) + ->and($log->find($failed->id)?->status)->toBe(CallbackStatus::Delivered); +}); From 67d3ff5399163c3b5c48b27ed7d8f9ddc6385ff1 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 21:43:06 -0700 Subject: [PATCH 45/47] fix(callbacks): deliver and replay to the subscription's current URL and run the destination refusal on it, recording a changed endpoint --- .../src/Console/ReplayCallbacksCommand.php | 11 +++++--- .../src/Delivery/CallbackDeliverer.php | 7 ++++++ .../src/Execution/CallbackExecution.php | 19 ++++++++++++++ .../tests/CapstoneCallbacksCommandsTest.php | 22 ++++++++++++++++ .../tests/Delivery/CallbackDelivererTest.php | 25 +++++++++++++++++++ .../tests/Delivery/CallbackDispatcherTest.php | 16 ++++++++++++ 6 files changed, 97 insertions(+), 3 deletions(-) diff --git a/packages/callbacks/src/Console/ReplayCallbacksCommand.php b/packages/callbacks/src/Console/ReplayCallbacksCommand.php index facae1ba..154fc9a0 100644 --- a/packages/callbacks/src/Console/ReplayCallbacksCommand.php +++ b/packages/callbacks/src/Console/ReplayCallbacksCommand.php @@ -13,9 +13,10 @@ /** * Sends logged deliveries again: the executions named by id, and/or with --failed the most recent failed ones. Each - * is re-sent to its original URL with its ORIGINAL body, re-signed with a fresh timestamp, under the subscription's - * current attempt budget and the endpoint's breaker; its attempts keep counting. A deactivated subscription is - * refused (and its execution left as logged) unless --include-inactive is passed, which the output announces. Exits 1 if any replay did not end + * is re-sent to its subscription's CURRENT URL (a changed endpoint is reported, and recorded on the execution) with + * its ORIGINAL body, re-signed with a fresh timestamp, under the subscription's current attempt budget and the + * endpoint's breaker; its attempts keep counting. A deactivated subscription is refused, its execution left as + * logged, unless --include-inactive is passed, which the output announces. Exits 1 if any replay did not end * delivered, so a scheduled replay surfaces in the scheduler's output. */ final class ReplayCallbacksCommand extends Command @@ -85,6 +86,10 @@ public function handle(Container $container): int continue; } + if ($result->url !== $execution->url) { + $this->warn("{$result->id}: destination changed from {$execution->url} to {$result->url}"); + } + $line = "{$result->id}: {$result->status->value} after {$result->attempts} attempt(s)".($result->lastError !== null ? " — {$result->lastError}" : ''); if (! $includeInactive && $result->lastError === "subscription [{$execution->subscriptionId}] is inactive") { $line .= '; pass --include-inactive to replay it anyway'; diff --git a/packages/callbacks/src/Delivery/CallbackDeliverer.php b/packages/callbacks/src/Delivery/CallbackDeliverer.php index b5d36dee..b637323b 100644 --- a/packages/callbacks/src/Delivery/CallbackDeliverer.php +++ b/packages/callbacks/src/Delivery/CallbackDeliverer.php @@ -18,6 +18,9 @@ * * An inactive subscription is refused before anything is sent: failed, "subscription [id] is inactive". This is * the one delivery path, so dispatch and replay alike fail closed; only a caller that explicitly includes * inactive subscriptions (`firefly:callbacks:replay --include-inactive`) gets past it. + * * The destination is the subscription's CURRENT URL, never the one stored on the execution: an endpoint changed + * since the execution was logged is delivered to (and checked) at its new address, and the returned execution + * carries that URL, so the log records where it was actually sent. * * The URL must be http(s) — https unless firefly.callbacks.http.require-https is false — and, when * firefly.callbacks.http.allowed-hosts names hosts, on one of them or a subdomain (PyFly's authorized domains: * the guard against a subscription aimed at an internal address). @@ -60,6 +63,10 @@ public function deliver(CallbackSubscription $subscription, CallbackExecution $e return $execution->withOutcome(CallbackStatus::Failed, $execution->attempts, null, "subscription [{$subscription->id}] is inactive", ($this->clock)()); } + if ($execution->url !== $subscription->url) { + $execution = $execution->withUrl($subscription->url); + } + $refusal = $this->refusal($execution->url); if ($refusal !== null) { return $execution->withOutcome(CallbackStatus::Failed, $execution->attempts, null, $refusal, ($this->clock)()); diff --git a/packages/callbacks/src/Execution/CallbackExecution.php b/packages/callbacks/src/Execution/CallbackExecution.php index a7f74bca..69708829 100644 --- a/packages/callbacks/src/Execution/CallbackExecution.php +++ b/packages/callbacks/src/Execution/CallbackExecution.php @@ -57,6 +57,25 @@ public static function pending(CallbackSubscription $subscription, EventEnvelope return new self($id, $subscription->id, $envelope->eventId, $envelope->eventType, $subscription->url, $body, CallbackStatus::Pending, 0, null, null, $now, $now); } + /** The same execution aimed at another destination: the subscription's current URL, after its endpoint changed. */ + public function withUrl(string $url): self + { + return new self( + $this->id, + $this->subscriptionId, + $this->eventId, + $this->eventType, + $url, + $this->body, + $this->status, + $this->attempts, + $this->responseStatus, + $this->lastError, + $this->createdAt, + $this->updatedAt, + ); + } + public function withOutcome(CallbackStatus $status, int $attempts, ?int $responseStatus, ?string $lastError, int $now): self { return new self( diff --git a/packages/callbacks/tests/CapstoneCallbacksCommandsTest.php b/packages/callbacks/tests/CapstoneCallbacksCommandsTest.php index 0a838b5f..6456af7d 100644 --- a/packages/callbacks/tests/CapstoneCallbacksCommandsTest.php +++ b/packages/callbacks/tests/CapstoneCallbacksCommandsTest.php @@ -8,6 +8,7 @@ use Firefly\Callbacks\Subscription\SubscriptionRegistry; use Firefly\Callbacks\Tests\Support\CallbacksArtisan; use Firefly\Callbacks\Tests\Support\CallbacksCapstoneTestCase; +use Illuminate\Http\Client\Request; use Illuminate\Support\Facades\Http; uses(CallbacksCapstoneTestCase::class); @@ -89,3 +90,24 @@ ->assertExitCode(0); Http::assertSentCount(4); }); + +it('replays to the subscription\'s current URL and reports the destination change', function () { + /** @var CallbacksCapstoneTestCase $this */ + Http::fake(['partner.example.com/*' => Http::response('', 503), 'moved.example.com/*' => Http::response('', 204)]); + $this->events->publish('orders', 'order.created', ['id' => 12]); + $failed = $this->app()->make(ExecutionLog::class)->recent(1, CallbackStatus::Failed)[0]; + + $registry = $this->app()->make(SubscriptionRegistry::class); + $current = $registry->find('sub-orders'); + expect($current)->not->toBeNull(); + /** @var CallbackSubscription $current */ + $registry->save(new CallbackSubscription($current->id, $current->name, 'https://moved.example.com/hooks/orders', $current->eventTypes, $current->secret, $current->headers, true, $current->maxAttempts)); + + CallbacksArtisan::pending($this->artisan('firefly:callbacks:replay', ['ids' => [$failed->id]])) + ->expectsOutputToContain("{$failed->id}: destination changed from https://partner.example.com/hooks/orders to https://moved.example.com/hooks/orders") + ->expectsOutputToContain("{$failed->id}: delivered after 4 attempt(s)") + ->assertExitCode(0); + + Http::assertSent(static fn (Request $request): bool => $request->url() === 'https://moved.example.com/hooks/orders'); + expect($this->app()->make(ExecutionLog::class)->find($failed->id)?->url)->toBe('https://moved.example.com/hooks/orders'); +}); diff --git a/packages/callbacks/tests/Delivery/CallbackDelivererTest.php b/packages/callbacks/tests/Delivery/CallbackDelivererTest.php index bc21ccc2..15397367 100644 --- a/packages/callbacks/tests/Delivery/CallbackDelivererTest.php +++ b/packages/callbacks/tests/Delivery/CallbackDelivererTest.php @@ -152,3 +152,28 @@ function w4Partner(?int $maxAttempts = null, string $url = 'https://partner.exam expect($forced->status)->toBe(CallbackStatus::Delivered) ->and($sender->sent)->toHaveCount(1); }); + +it('delivers to the subscription\'s current URL, not the one stored on the execution, and records the change', function () { + $sender = new ScriptedCallbackSender([204]); + [$deliverer] = w4Deliverer($sender); + $execution = w4Execution(w4Partner(url: 'https://old.example.com/hooks')); + + $result = $deliverer->deliver(w4Partner(url: 'https://new.example.com/hooks'), $execution); + + expect($result->status)->toBe(CallbackStatus::Delivered) + ->and($sender->sent[0]['url'])->toBe('https://new.example.com/hooks') + ->and($result->url)->toBe('https://new.example.com/hooks'); +}); + +it('runs the destination refusal on the subscription\'s current URL, whatever the stored one passed', function () { + $sender = new ScriptedCallbackSender([204]); + [$deliverer] = w4Deliverer($sender, new CallbacksSettings(allowedHosts: ['partner.example.com'])); + $execution = w4Execution(w4Partner()); + + $result = $deliverer->deliver(w4Partner(url: 'https://169.254.169.254/latest/meta-data'), $execution); + + expect($result->status)->toBe(CallbackStatus::Failed) + ->and($result->lastError)->toContain('allowed-hosts') + ->and($result->url)->toBe('https://169.254.169.254/latest/meta-data') + ->and($sender->sent)->toBe([]); +}); diff --git a/packages/callbacks/tests/Delivery/CallbackDispatcherTest.php b/packages/callbacks/tests/Delivery/CallbackDispatcherTest.php index 5dfe0fb5..4fe3865d 100644 --- a/packages/callbacks/tests/Delivery/CallbackDispatcherTest.php +++ b/packages/callbacks/tests/Delivery/CallbackDispatcherTest.php @@ -93,3 +93,19 @@ function w4Dispatcher(ScriptedCallbackSender $sender): array ->and($sender->sent)->toHaveCount(2) ->and($log->find($failed->id)?->status)->toBe(CallbackStatus::Delivered); }); + +it('replays to the subscription\'s current URL after its endpoint changed, and logs the new destination', function () { + $sender = new ScriptedCallbackSender([500, 204]); + $subscriptions = new InMemorySubscriptionRegistry([new CallbackSubscription('orders', 'orders', 'https://a.example.com/hooks', ['order.*'], 's')]); + $log = new InMemoryExecutionLog; + $deliverer = new CallbackDeliverer(new CallbacksSettings(maxAttempts: 1), $sender, new CallbackSigner, null, static function (float $s): void {}); + $dispatcher = new CallbackDispatcher($subscriptions, $log, $deliverer); + $failed = $dispatcher->dispatch(new EventEnvelope('order.created', 'orders', [], [], 'evt-5'))[0]; + + $subscriptions->save(new CallbackSubscription('orders', 'orders', 'https://moved.example.com/hooks', ['order.*'], 's')); + $replayed = $dispatcher->replay($failed); + + expect($replayed?->status)->toBe(CallbackStatus::Delivered) + ->and($sender->sent[1]['url'])->toBe('https://moved.example.com/hooks') + ->and($log->find($failed->id)?->url)->toBe('https://moved.example.com/hooks'); +}); From af31188b28ec5093b48f147ba79bc2ef15352ec3 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 21:43:22 -0700 Subject: [PATCH 46/47] docs(callbacks): document that replay refuses inactive subscriptions without --include-inactive and sends to the current URL --- CHANGELOG.md | 7 +++++++ docs/modules/callbacks.md | 12 +++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 44b60bb9..2c6bcb0c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,13 @@ All notable changes to LaraFly are documented here. This project uses CalVer (`Y `CacheResilienceStore` constructed by hand over such a store throws from `withLock()` unless built with `allowLockless: true`. +### Fixed + +- **`firefly:callbacks:replay` honors deactivation and endpoint changes.** The deliverer refuses an inactive + subscription on every path (replay needs `--include-inactive`, which the output reports), and every delivery goes + to the subscription's current URL — checked against `http.require-https` / `http.allowed-hosts` — instead of the + URL stored on the execution, recording the change. + ## [26.10.1] - 2026-10-06 ### Added diff --git a/docs/modules/callbacks.md b/docs/modules/callbacks.md index b5220e55..d61de422 100644 --- a/docs/modules/callbacks.md +++ b/docs/modules/callbacks.md @@ -88,7 +88,9 @@ it with the `hmac` validator and `timestamp-header: X-Firefly-Timestamp`. `circuit-breaker.wait-duration-in-open`, and deliveries to that endpoint fail at once with `circuit open` instead of waiting on a partner that is down. - URLs must be https unless `http.require-https` is false, and on a host in `http.allowed-hosts` (or a subdomain - of one) when that list is not empty. + of one) when that list is not empty. The URL checked and sent to is always the subscription's current one. +- An inactive subscription is never delivered to: it does not match events, and the deliverer refuses it on any + path that reaches it (see `--include-inactive` below for the one explicit exception). - Attempts run where the event is consumed. On `firefly.eda.provider=memory` that is the request that published the event — choose `queue` or a broker so deliveries run in workers. - An event the broker redelivers maps to the same execution: one already delivered is not sent again, one that @@ -105,11 +107,19 @@ php artisan firefly:callbacks:list php artisan firefly:callbacks:list --executions --status=failed --limit=50 php artisan firefly:callbacks:replay --failed php artisan firefly:callbacks:replay 9f2c4e… +php artisan firefly:callbacks:replay 9f2c4e… --include-inactive ``` `replay` re-sends the logged body re-signed with a fresh timestamp; its attempts keep counting, and it exits 1 if any replay did not end delivered — schedule it and the scheduler's output says so. +- It sends to the subscription's **current** URL, not the one stored on the execution, and runs the + `http.require-https` / `http.allowed-hosts` checks on that URL. When the endpoint changed since the execution was + logged, the output says `destination changed from to ` and the execution records the new URL. +- A deactivated subscription (`active` false) is refused — `subscription [] is inactive`, nothing sent, the + execution left as logged — like every other delivery. `--include-inactive` replays to it anyway, and the output + announces that the option is on. + ```bash From 1578f124593397c9b8e8410a94e2e3838208b611 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Tue, 13 Oct 2026 01:23:12 -0700 Subject: [PATCH 47/47] docs(publishing): count the 35 component names the root package replaces --- docs/publishing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/publishing.md b/docs/publishing.md index 8175d246..3ef294ce 100644 --- a/docs/publishing.md +++ b/docs/publishing.md @@ -7,7 +7,7 @@ cross-repository `ACCESS_TOKEN` are needed. ## Package identity and compatibility -The root package replaces all 30 former component names, including `firefly/firefly`, with +The root package replaces all 35 component names, including `firefly/firefly`, with `self.version`. Packagist's `firefly` vendor namespace belongs to another publisher, so the public package uses the Firefly Framework organization's namespace. `firefly/lumen` is a sample, not a replacement; `firefly/skeleton` is a bundled project template, not a second published package.