Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -82,10 +82,12 @@ engine = create_async_engine("postgresql+asyncpg://localhost/app")
broker = OutboxBroker(engine, outbox_table=outbox_table)
app = FastStream(broker)


@broker.subscriber("orders", max_workers=4)
async def handle(order_id: int) -> None:
print(f"order {order_id}")


# Producer side — share the caller's open transaction:
session_factory = async_sessionmaker(engine, expire_on_commit=False)
async with session_factory() as session, session.begin():
Expand Down
5 changes: 2 additions & 3 deletions docs/introduction/how-it-works.md
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ transaction — the row must commit with the caller's domain writes:

```python
async with session_factory() as session, session.begin():
session.add(order) # domain write
session.add(order) # domain write
await broker.publish(order.id, queue="orders", session=session)
# session.begin() commits both atomically on exit
```
Expand Down Expand Up @@ -127,8 +127,7 @@ immune to worker / DB clock skew.
When the invariant fires, the broker emits a WARNING with structured fields:

```python
extra={"event": "lease_lost", "phase": "terminal" | "retry",
"row_id": ..., "queue": ..., "deliveries_count": ...}
extra = {"event": "lease_lost", "phase": "terminal" | "retry", "row_id": ..., "queue": ..., "deliveries_count": ...}
```

Recurring `event=lease_lost` records mean `lease_ttl_seconds < handler P99`
Expand Down
107 changes: 60 additions & 47 deletions docs/operations/alembic.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,29 +16,41 @@ against an empty schema):

```python
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('outbox',
sa.Column('id', sa.BigInteger(), autoincrement=True, nullable=False),
sa.Column('queue', sa.String(length=255), nullable=False),
sa.Column('payload', sa.LargeBinary(), nullable=False),
sa.Column('headers', postgresql.JSONB(astext_type=sa.Text()), nullable=True),
sa.Column('attempts_count', sa.BigInteger(), server_default='0', nullable=False),
sa.Column('deliveries_count', sa.BigInteger(), server_default='0', nullable=False),
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
sa.Column('next_attempt_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
sa.Column('first_attempt_at', sa.DateTime(timezone=True), nullable=True),
sa.Column('last_attempt_at', sa.DateTime(timezone=True), nullable=True),
sa.Column('acquired_at', sa.DateTime(timezone=True), nullable=True),
sa.Column('acquired_token', sa.Uuid(), nullable=True),
sa.Column('timer_id', sa.String(length=255), nullable=True),
sa.CheckConstraint('(acquired_token IS NULL) = (acquired_at IS NULL)', name='outbox_lease_ck'),
sa.PrimaryKeyConstraint('id')
op.create_table(
"outbox",
sa.Column("id", sa.BigInteger(), autoincrement=True, nullable=False),
sa.Column("queue", sa.String(length=255), nullable=False),
sa.Column("payload", sa.LargeBinary(), nullable=False),
sa.Column("headers", postgresql.JSONB(astext_type=sa.Text()), nullable=True),
sa.Column("attempts_count", sa.BigInteger(), server_default="0", nullable=False),
sa.Column("deliveries_count", sa.BigInteger(), server_default="0", nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.Column("next_attempt_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.Column("first_attempt_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("last_attempt_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("acquired_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("acquired_token", sa.Uuid(), nullable=True),
sa.Column("timer_id", sa.String(length=255), nullable=True),
sa.CheckConstraint("(acquired_token IS NULL) = (acquired_at IS NULL)", name="outbox_lease_ck"),
sa.PrimaryKeyConstraint("id"),
)
op.create_index(
"outbox_lease_idx",
"outbox",
["queue", "acquired_at"],
unique=False,
postgresql_where=sa.text("acquired_token IS NOT NULL"),
)
op.create_index(
"outbox_pending_idx",
"outbox",
["queue", "next_attempt_at"],
unique=False,
postgresql_where=sa.text("acquired_token IS NULL"),
)
op.create_index(
"outbox_timer_id_uq", "outbox", ["queue", "timer_id"], unique=True, postgresql_where=sa.text("timer_id IS NOT NULL")
)
op.create_index('outbox_lease_idx', 'outbox', ['queue', 'acquired_at'], unique=False,
postgresql_where=sa.text('acquired_token IS NOT NULL'))
op.create_index('outbox_pending_idx', 'outbox', ['queue', 'next_attempt_at'], unique=False,
postgresql_where=sa.text('acquired_token IS NULL'))
op.create_index('outbox_timer_id_uq', 'outbox', ['queue', 'timer_id'], unique=True,
postgresql_where=sa.text('timer_id IS NOT NULL'))
# ### end Alembic commands ###
```

Expand Down Expand Up @@ -89,21 +101,22 @@ The captured `upgrade()` body:

```python
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('outbox_dlq',
sa.Column('id', sa.BigInteger(), autoincrement=True, nullable=False),
sa.Column('original_id', sa.BigInteger(), nullable=False),
sa.Column('queue', sa.String(length=255), nullable=False),
sa.Column('payload', sa.LargeBinary(), nullable=False),
sa.Column('headers', postgresql.JSONB(astext_type=sa.Text()), nullable=True),
sa.Column('deliveries_count', sa.BigInteger(), nullable=False),
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
sa.Column('failed_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
sa.Column('failure_reason', sa.String(length=64), nullable=False),
sa.Column('last_exception', sa.String(), nullable=True),
sa.Column('timer_id', sa.String(length=255), nullable=True),
sa.PrimaryKeyConstraint('id')
op.create_table(
"outbox_dlq",
sa.Column("id", sa.BigInteger(), autoincrement=True, nullable=False),
sa.Column("original_id", sa.BigInteger(), nullable=False),
sa.Column("queue", sa.String(length=255), nullable=False),
sa.Column("payload", sa.LargeBinary(), nullable=False),
sa.Column("headers", postgresql.JSONB(astext_type=sa.Text()), nullable=True),
sa.Column("deliveries_count", sa.BigInteger(), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("failed_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.Column("failure_reason", sa.String(length=64), nullable=False),
sa.Column("last_exception", sa.String(), nullable=True),
sa.Column("timer_id", sa.String(length=255), nullable=True),
sa.PrimaryKeyConstraint("id"),
)
op.create_index('outbox_dlq_queue_failed_idx', 'outbox_dlq', ['queue', 'failed_at'], unique=False)
op.create_index("outbox_dlq_queue_failed_idx", "outbox_dlq", ["queue", "failed_at"], unique=False)
# ### end Alembic commands ###
```

Expand Down Expand Up @@ -268,11 +281,11 @@ pointer to this section. Re-running autogenerate produces an empty
```python
# Drop first ONLY if the constraint exists with a wrong predicate; skip the
# drop if it is absent entirely.
op.drop_constraint('outbox_lease_ck', 'outbox', type_='check')
op.drop_constraint("outbox_lease_ck", "outbox", type_="check")
op.create_check_constraint(
'outbox_lease_ck',
'outbox',
'(acquired_token IS NULL) = (acquired_at IS NULL)',
"outbox_lease_ck",
"outbox",
"(acquired_token IS NULL) = (acquired_at IS NULL)",
)
```

Expand All @@ -288,13 +301,13 @@ three indexes and their expected shape:
| `outbox_timer_id_uq` | `queue, timer_id` | yes | `timer_id IS NOT NULL` |

```python
op.drop_index('outbox_timer_id_uq', table_name='outbox')
op.drop_index("outbox_timer_id_uq", table_name="outbox")
op.create_index(
'outbox_timer_id_uq',
'outbox',
['queue', 'timer_id'],
"outbox_timer_id_uq",
"outbox",
["queue", "timer_id"],
unique=True,
postgresql_where=sa.text('timer_id IS NOT NULL'),
postgresql_where=sa.text("timer_id IS NOT NULL"),
)
```

Expand Down Expand Up @@ -339,7 +352,7 @@ faster, no vacuum debt.
# ### Convert outbox_dlq to range-partitioned by failed_at ###

# Rename the existing table out of the way.
op.rename_table('outbox_dlq', 'outbox_dlq_old')
op.rename_table("outbox_dlq", "outbox_dlq_old")

# Create the partitioned parent. The PRIMARY KEY must include the
# partition key, so we extend it to (id, failed_at).
Expand Down Expand Up @@ -381,7 +394,7 @@ op.execute("""
WHERE failed_at >= '2026-05-01';
""")

op.drop_table('outbox_dlq_old')
op.drop_table("outbox_dlq_old")
```

`validate_schema()` continues to work against the partitioned table —
Expand Down
6 changes: 2 additions & 4 deletions docs/usage/dlq.md
Original file line number Diff line number Diff line change
Expand Up @@ -129,12 +129,10 @@ transform (or drop) the stored text:

```python
# Store only the exception class — no message, no payload.
broker = OutboxBroker(engine, outbox_table=t, dlq_table=dlq,
last_exception_renderer=lambda exc: type(exc).__name__)
broker = OutboxBroker(engine, outbox_table=t, dlq_table=dlq, last_exception_renderer=lambda exc: type(exc).__name__)

# Or drop the detail entirely:
broker = OutboxBroker(engine, outbox_table=t, dlq_table=dlq,
last_exception_renderer=lambda exc: None)
broker = OutboxBroker(engine, outbox_table=t, dlq_table=dlq, last_exception_renderer=lambda exc: None)
```

The renderer runs per terminal failure; its output is still length-capped
Expand Down
6 changes: 2 additions & 4 deletions docs/usage/fastapi.md
Original file line number Diff line number Diff line change
Expand Up @@ -63,8 +63,7 @@ router = OutboxRouter(engine, outbox_table=outbox_table)
async def handle(
body: dict,
session: AsyncSession = Depends(get_session),
) -> None:
... # domain writes on `session` commit with any chained outbox publishes
) -> None: ... # domain writes on `session` commit with any chained outbox publishes


@router.post("/orders")
Expand Down Expand Up @@ -116,8 +115,7 @@ async def handle(
msg: OutboxMessage,
broker: OutboxBroker,
session: AsyncSession = Depends(get_session),
) -> None:
...
) -> None: ...
```

They resolve via FastStream's `Context()` paths but go through FastAPI's
Expand Down
10 changes: 8 additions & 2 deletions docs/usage/messaging-service.md
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,9 @@ class OutboxEventProducer:

async def send_chat_event(self, event: ChatEvent) -> None:
await self.outbox_broker.publish(
event, queue="chat-events", session=self.session,
event,
queue="chat-events",
session=self.session,
)
```

Expand Down Expand Up @@ -250,7 +252,11 @@ from faststream_outbox import TestOutboxBroker


async def test_create_message_relays_event_to_kafka(
outbox_broker, kafka_broker, create_message_use_case, command, kafka_publisher,
outbox_broker,
kafka_broker,
create_message_use_case,
command,
kafka_publisher,
) -> None:
async with TestOutboxBroker(outbox_broker), TestKafkaBroker(kafka_broker):
await create_message_use_case(command)
Expand Down
6 changes: 3 additions & 3 deletions docs/usage/publisher.md
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ raises `NotImplementedError`.

```python
async with session_factory() as session, session.begin():
session.add(order) # domain write
session.add(order) # domain write
await broker.publish(
{"order_id": order.id},
queue="orders",
Expand Down Expand Up @@ -97,9 +97,9 @@ orders_pub = broker.publisher("orders", headers={"source": "checkout"})


async def checkout(order: Order, session: AsyncSession) -> None:
session.add(order) # domain write
session.add(order) # domain write
await orders_pub.publish({"order_id": order.id}, session=session)
await session.commit() # row + domain commit together
await session.commit() # row + domain commit together
```

Per-call `headers` are merged with the publisher's static headers
Expand Down
1 change: 1 addition & 0 deletions docs/usage/relay.md
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,7 @@ are **not** forwarded to the foreign publish. Two ways to override:
from faststream.response import Response
from faststream_outbox.annotations import OutboxMessage


@publisher_kafka
@broker_outbox.subscriber("outbox_queue")
async def relay(body: dict, msg: OutboxMessage) -> Response:
Expand Down
4 changes: 2 additions & 2 deletions docs/usage/subscriber.md
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ reclaim of *actually* stuck rows everywhere. Instead, segregate slow work
onto its own subscriber with a longer TTL:

```python
@broker.subscriber("slow_q", lease_ttl_seconds=600) # 10 minutes
@broker.subscriber("slow_q", lease_ttl_seconds=600) # 10 minutes
async def heavy_job(msg): ...


Expand Down Expand Up @@ -235,7 +235,7 @@ async def handle(msg: OutboxMessage, body: dict) -> None:
await write_audit(body)
await msg.ack()
except TransientError:
await msg.nack() # retry
await msg.nack() # retry
except PermanentError:
await msg.reject() # terminal delete
```
Expand Down
2 changes: 1 addition & 1 deletion docs/usage/testing.md
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ tb = TestOutboxBroker(broker, run_loops=True)
async with tb:
tb.feed("orders", json.dumps({"order_id": 1}).encode())
# the real fetch + worker loops pick the row up asynchronously
async with asyncio.timeout(1.0): # fail fast instead of hanging forever
async with asyncio.timeout(1.0): # fail fast instead of hanging forever
while not received:
await asyncio.sleep(0.01)

Expand Down
12 changes: 9 additions & 3 deletions docs/usage/timers.md
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,9 @@ await broker.publish(

# Fire at a specific UTC instant:
await broker.publish(
{"x": 1}, queue="orders", session=session,
{"x": 1},
queue="orders",
session=session,
activate_at=dt.datetime(2027, 6, 1, 9, tzinfo=dt.UTC),
)
```
Expand Down Expand Up @@ -63,15 +65,19 @@ id is a silent no-op (returns `None`):

```python
first = await broker.publish(
{"order_id": 1}, queue="orders", session=session,
{"order_id": 1},
queue="orders",
session=session,
activate_in=dt.timedelta(seconds=30),
timer_id="order-confirm-1",
)
assert first is not None

# Re-publish — no row inserted, no NOTIFY emitted, returns None.
second = await broker.publish(
{"order_id": 1}, queue="orders", session=session,
{"order_id": 1},
queue="orders",
session=session,
activate_in=dt.timedelta(seconds=30),
timer_id="order-confirm-1",
)
Expand Down
2 changes: 1 addition & 1 deletion faststream_outbox/metrics/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ def _safe_emit(recorder: MetricsRecorder, event: str, tags: Mapping[str, typing.
"""
try:
recorder(event, tags)
except Exception: # noqa: BLE001
except Exception:
_logger.log(logging.DEBUG, "metrics recorder raised", exc_info=True)


Expand Down
2 changes: 2 additions & 0 deletions planning/changes/2026-06-03.02-faststream-0.7-migration.md
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,7 @@ _parser: AsyncCallable
_decoder: AsyncCallable
codec: CodecProto # NEW in 0.7


# faststream/_internal/endpoint/subscriber/usecase.py — add_call signature
def add_call(
self,
Expand All @@ -171,6 +172,7 @@ def add_call(
codec_: Optional[CodecProto] = None, # NEW; outbox passes nothing
) -> Self: ...


# faststream/_internal/testing/broker.py — create_publisher_fake_subscriber
@abstractmethod
def create_publisher_fake_subscriber(
Expand Down
8 changes: 6 additions & 2 deletions planning/changes/2026-06-04.02-foreign-broker-relay.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ Rabbit, NATS, Redis, Confluent — any FastStream broker) is the destination:
```python
publisher_kafka = broker_kafka.publisher("kafka_topic")


@publisher_kafka
@broker_outbox.subscriber("outbox_queue", max_workers=10, retry_strategy=...)
async def relay(body: dict) -> dict:
Expand Down Expand Up @@ -47,7 +48,7 @@ relevant body is:
```python
for p in chain(
self.__get_response_publisher(message), # → (OutboxFakePublisher,)
h.handler._publishers, # → (KafkaPublisher, …) from @pub
h.handler._publishers, # → (KafkaPublisher, …) from @pub
):
await p._publish(
result_msg.as_publish_command(),
Expand Down Expand Up @@ -97,12 +98,14 @@ kafka_router = KafkaRouter()

publisher_kafka = kafka_router.publisher("kafka_topic")


@publisher_kafka
@broker_outbox.subscriber("outbox_queue")
async def relay(body: dict) -> dict:
return body

broker_kafka.include_router(kafka_router) # wires producer into router publisher

broker_kafka.include_router(kafka_router) # wires producer into router publisher
app = FastStream(broker_outbox, on_startup=[broker_kafka.connect])
```

Expand Down Expand Up @@ -208,6 +211,7 @@ relay subscriber forwards to Kafka. The relay block:
```python
publisher_kafka = broker_kafka.publisher("kafka_topic")


@publisher_kafka
@broker_outbox.subscriber("outbox_queue")
async def relay(body: dict) -> dict:
Expand Down
Loading
Loading