diff --git a/docs/design/unpublished_s3_object_ownership.md b/docs/design/unpublished_s3_object_ownership.md new file mode 100644 index 0000000000000..548ccabfb7e01 --- /dev/null +++ b/docs/design/unpublished_s3_object_ownership.md @@ -0,0 +1,364 @@ +# Unpublished CN S3 object ownership and remote handoff + +Issue: [#29257](https://github.com/matrixorigin/matrixone/issues/29257). +Implementation: [#29290](https://github.com/matrixorigin/matrixone/pull/29290). +Status: **bounded-admission revision reviewed by an independent GPT-6 medium +agent at commit `4014befc34f726913260d4677a22bae370ae7387`: PASS, 0 design +blockers. Lightweight retry-shell revision reviewed at +`1107d213cf0fdcb53ede12064442ccfc655bc2fb`: PASS, 0 design blockers. +Implementation tests and independent code review passed locally; deployment +alerts remain a rollout condition.** + +This document records the design embodied by an already-open implementation PR. +It is retrospective and does not establish that the earlier commits passed a +design-first gate. Link the exact reviewed document revision and an independent +design decision in the PR before treating that gate as satisfied. A material +change to the ownership or wire contract needs a new design revision. + +## Problem and acceptance boundary + +CN writers can persist an S3 object before its metadata enters `txn.writes`. +`SyncAndTakeResults` can detach the only stats from the sinker, and `Sinker.Close` +discards remaining persisted/unpublished names without deleting them. A later +serialization, forwarding, or workspace-append error then leaves an object that +ordinary rollback GC cannot find. The same boundary occurs in Insert, +MultiUpdate, remote DELETE, workspace dumps/rewrite, clone readers, and tombstone +transfer. It is not a request to make every S3 object globally discoverable. + +The registration boundary is the successful append of the corresponding +metadata entry to the transaction workspace. Before that append, exactly one +effective cleanup responsibility must remain; after it, normal transaction +rollback/GC owns the object. Duplicate name-only owners may exist during +handoff, but the transaction deduplicates by object name and never lets a later +owner delete a name already accepted by the first owner. + +Required invariants: + +1. A failed Sync, metadata copy, or append cannot silently discard persisted + names. Successful metadata copy alone is not registration. +2. A producer may release a completed writer's buffers only after a lightweight + name owner has been retained by its workspace. A failed transfer leaves the + writer responsible and retryable. +3. A remote worker may retire its owner only after the coordinator has retained + each exported batch's names and sent an explicit receipt for every batch. + An ordinary credit ACK, a missing ACK, or an empty stream is not proof. +4. Aborted, stopped, canceled, or failed streams keep the worker in cleanup + mode. Accepted names are never deleted by the unpublished-object cleaner. +5. Failed deletion retains the exact writer/name owner for a later in-process + retry and returns a cleanup error; it is not interpreted as success. A + remote mirror whose handler has exited transfers that retry obligation to + its CN server before the stream is removed. A local transaction does the + same before terminal rollback retires its workspace. + +## Ownership transitions + +| Boundary | Before | Successful transition | Failure | +|---|---|---|---| +| Writer Sync / result extraction | Writer/sinker owns persisted names | Writer tracks detached names alongside sinker results | Writer keeps both sets and abort cleanup deletes them | +| Metadata copied to producer output | Writer owns names | Workspace retains a copied name-only owner; writer may reset/close | Writer remains cleanup-pending and cannot close as though published | +| Local workspace append | Workspace name owner | Append accepts only names in that entry; transaction rollback/GC takes over | Unaccepted names remain for workspace cleanup | +| Remote output decode | Worker workspace owns names | Coordinator parses and retains names before forwarding/discarding output | No ownership receipt; worker cleans on terminal failure | +| Batch ACK | Worker and coordinator have temporary overlapping responsibility | Coordinator sends `batch_ack_s3_ownership_retained=true` only after retention; worker records contiguous receipts | Missing/unmarked receipt cannot retire worker ownership | +| Remote terminal drain | Worker still owns unproven names | All sent batches have contiguous receipts, no pending credits, no abort/stop; worker accepts its owners and completes | Worker cleans; cleanup error is returned with the original execution error | + +`CNS3Writer.TransferPersistedObjects` reads results already produced by Sync; it +does not Sync again. Persistent Insert/MultiUpdate/remote DELETE producers call +it only after their output metadata has been copied. One-shot MultiUpdate +writers close after successful Sync and copy; only object names survive until +workspace append. A writer whose deletion fails remains open for retry, so an +outage can temporarily retain its buffers. Normal successful spills do not. + +The transaction workspace keeps both name owners and fallback cleanup +callbacks for writers/flows whose immediate cleanup failed. It detaches the +ledger while performing file-service I/O, then reattaches failed entries for +the next lifecycle callback. The name index prevents duplicate owners from +deleting an accepted object. Cleanup is independent of operator reuse. If the +remote terminal cleanup fails, the handler detaches the workspace's cleanup +records into a callback and registers it with the owning CN server before +returning its error. The callback does not capture the transaction itself. +One CN worker +retries failed callbacks after the stream has exited; a successful callback +releases the reference, including the consumed slot in the queue backing array. +The server stops transaction producers before closing cleanup admission, joins +the worker, and retries tasks in rounds within one cleanup deadline before CN +I/O dependencies close. Individual attempts have a thirty-second bound. A +failed task does not prevent healthy tasks from being attempted in that round. +If the total deadline expires, CN shutdown remains fail-stop and preserves its +dependencies, but the cleanup worker resumes so later storage recovery can +still delete the objects. The CN-close error remains visible; the existing +one-shot service shutdown contract does not automatically resume its tail. + +Terminal local rollback transfers detached unpublished cleanup records to +that CN worker before `delTransaction`, because frontend rollback invalidates +the transaction handle even when cleanup reports an error. A clone reader's +retained callback uses the current retry-attempt context rather than the +expired request context captured when the reader was created. + +## Remote receipt protocol and ordering + +The existing batch-credit ACK has a cumulative sequence and may be sent by an +older coordinator without retaining S3 names. Field 19 of `pipeline.Message` +adds an optional boolean `batch_ack_s3_ownership_retained`. Absence decodes to +false. It is a per-batch ownership receipt, not a new credit or a claim that +the transaction append succeeded. + +The coordinator classifies the actual producer output, including grouped +`PreScopes`, ordinary `UpdateWriteS3` where coordinator `IsRemote` is false, +Insert S3 output, and remote DELETE tombstone metadata. Traversal stops at +operators that have already consumed/registered the metadata. For a classified +batch, it parses names and retains them in the coordinator's transaction +workspace **before** forwarding and ACKing. A parse/retention failure sends no +ownership receipt and fails the stream. A no-output stream rejects unexpected +S3 output rather than acknowledging it as a successful handoff. + +The worker tracks two frontiers under the batch-flow mutex: cumulative +`ackedSeq` frees flow-control credits, while contiguous `ownershipAckedSeq` +proves explicit receipts. Only the latter can authorize S3 lease release. CN +ingress handles ACKs synchronously in the per-session receive loop, preserving +wire order before the next ACK handler runs; non-ACK pipeline work remains +asynchronous. Without this serialization, ACK 2 could consume cumulative +credits before ACK 1 updates the ownership frontier. The worker's finalization +predicate requires actual output (`nextSeq > 0`), every receipt, no pending +credits, and no abort or receiver StopSending. An invalid ACK closes the +connection; late ACKs after lifecycle removal or abort do not create ownership. + +The negotiated credit window is at most eight batches and 64 MiB of pending +wire payload; a single oversized batch can occupy an otherwise empty window. +ACK processing adds constant state, no receipt map, blocked ACK goroutines, or +new wait barrier. Object-name ownership is **not** bounded by that credit +window: it grows with the unpublished objects in a transaction and is released +by registration or cleanup. S3 deletion outages can prolong writer retention. +The CN retry queue holds one callback per failed stream or local rollback. +Each callback owns the remaining name owners and fallback cleanup callbacks, +not a bound method holding the entire transaction. Fallback writer cleanup can +still retain writer buffers and references until deletion succeeds. Memory +therefore grows with concurrent failed cleanups during a storage outage; this +is an explicit limit +of the in-memory repair, not a fixed-capacity queue that could drop ownership. +The single retry worker bounds retry concurrency. + +## Failure, rollout, and limits + +- A lost/failed ACK, connection loss, invalid sequence, cancellation, or + StopSending cannot prove takeover. The worker keeps its owner and attempts + cleanup; the coordinator's overlapping owner is reduced only if its local + transaction actually appends the corresponding metadata. Cleanup retries are + in-process, not a distributed durable handoff log. +- Pipeline teardown starts one execution-scoped cleanup deadline before Reset + or Free. All pipeline contexts share the budget without changing execution + cancellation; nested writer, MultiUpdate, transfer and remote finalizer + cleanup preserve it instead of starting another ten-minute timer per writer. + The first pipeline entering teardown starts the clock; later pipelines may + defer expired cleanup to the CN queue even if their own execution ran longer. + Prepared execution/query retry creates a fresh budget. Failed Reset transfers + cleanup to the workspace immediately, including Insert, Delete, MultiUpdate, + partition writers and clone readers, since prepared execution skips Free. + The CN + retry worker uses a fresh bounded attempt after the handler exits. A failed + S3 delete is returned and its in-memory owner stays queued; a process crash + or persistent storage outage can still leave an orphan. Durable recovery is + a separate design and is not claimed here. The budget bounds context-aware + object deletion, not an uncooperative file service or arbitrary CPU teardown. +- A new worker with an old coordinator sees an unmarked ACK and fails closed + rather than releasing its owner. An old worker with a new coordinator still + has its old pre-receipt gap; this PR cannot retrofit it. A mixed-version + cluster therefore does not have the full guarantee. Roll out coordinators + before workers, drain old workers before claiming the guarantee, and do not + treat wire-field tolerance as safe ownership interoperability. +- Retaining before downstream append can temporarily duplicate name ledgers + on worker and coordinator. The object itself is not copied. The extra ACK + field is emitted only when true; normal non-S3 streams keep credit behavior. + No new global registry, background sweeper, or S3 listing is introduced. + +Alternatives rejected: accepting any cumulative ACK as takeover (old +coordinator can ACK without retention); retaining every completed writer until +transaction end (unaccounted sinker memory across repeated spills); and a +receipt map or waiting ACK goroutines (additional unbounded/terminal-wakeup +state when per-session ingress order can be preserved). + +## Validation and review gate + +Deterministic regressions cover post-persistence/pre-append failures for local +producers and transaction callers, repeated spills and MPool release, +coordinator classification/retention for connector and dispatch output, +wire-field round trip and absent-field compatibility, marked/unmarked/gapped +ACKs, ACK ingress ordering, no-output and StopSending behavior, accepted versus +unaccepted names, and failed cleanup retries after stream exit. Deadline tests +check that nested cleanup preserves the outer bound. Focused race tests cover the +ownership and ingress paths. The PR's CI runs UT, coverage and multi-CN BVT; +passing them does not prove every mixed-version/cancel combination. + +Design review must explicitly decide whether the mixed-version fail-closed +behavior and in-memory-only deletion retry are acceptable for this bug fix, +whether an unbounded outage backlog requires a separate durable-recovery or +backpressure design before rollout, and verify that no terminal path accepts +ownership without all receipts. +The earlier implementation did not pass a design-first gate. The bounded +admission revision below received independent model review at the exact commit +above. That review does not replace maintainer approval, and a material +implementation deviation requires another design review. + +## Proposed revision: bounded in-process cleanup debt + +**Status: implemented and validated locally.** The ticket ledger, pre-`Sync` +admission, name-only retry state, and primary-error preservation are now in the +PR worktree. This revision supersedes the earlier unbounded-queue limit, +heavy-writer fallback, and cleanup-error reporting; other ownership and +receipt invariants remain in force. The +product choice is to stop admitting new CN S3 uploads when cleanup debt reaches +its limit; an already-uploaded object must never be abandoned because a queue is +full. Read-only queries and writes that do not upload new S3 objects remain +eligible to run. The existing ten-minute cleanup deadline is a limit on an +attempt, not on the lifetime of an ownership obligation. + +### Budget and admission point + +The CN owns a fixed-capacity ledger of **65,536 object-name tickets**. A ticket +is reserved immediately before each CN file sinker's `Sync` can persist its +named object, including synchronous and pipelined sinkers. The reservation uses +the name already available from `FSinkerImpl.ActiveObjectName`; an unavailable +name or missing CN admission service fails closed before `Sync`. A canceled +reservation releases its ticket only if no write started. Once `Sync` starts, +both success and commit-ambiguous failure retain the ticket until either +metadata registration is confirmed or deletion succeeds. No failed-cleanup +queue append can be the first admission decision. + +The bound covers object-name tickets, not object bytes on S3. Each retained +record must contain a validated bounded-length name, one file-service handle, +and a small fixed-size ownership state. Callbacks handed to the CN retry worker +must close/drain their sinkers and release `mpool` buffers first, then retain +only file-service/name records. A lightweight `CNS3Writer` or `TransferFlow` +shell may hold those records so existing retry callbacks remain valid; their +sinkers, batches, `mpool` buffers, table, and request context must be nil or +released. A callback with no object-name tickets is not queued. The worker may group tickets per failed +stream or rollback, but its queue entry count cannot exceed the ticket count. +The 65,536 default is a concrete admission ceiling; benchmarked retained bytes +per ticket and operational headroom must be recorded before rollout. In-flight +writer buffers remain outside this ticket budget; their existing controls are +path-specific, and this proposal does not claim a global bound for them. The +ticket budget specifically covers debt that outlives those writers. Changing +the ceiling requires a separately reviewed configuration/capacity decision. + +All production CN S3 upload paths use this admission hook: Insert, MultiUpdate, +remote DELETE, transaction dumps and rewrites, clone readers, and tombstone +transfer. The existing shared sinker remains usable by TN and other callers +without CN admission. CN constructors supply the service-specific hook; direct +`TransferFlow` sinker creation supplies it explicitly. A path with no hook must +not silently fall back to unmetered CN persistence. The name/ticket moves with +the first effective cleanup owner; duplicate ownership references share the +same ticket, rather than consuming or releasing it twice. + +For a remote handoff, the worker retains its ticket while the coordinator +reserves its own ticket **before** recording each received name and emitting +field-19 ownership receipt. The two CNs may briefly each charge the same +object; the worker releases its charge only after contiguous receipts prove +takeover. If the coordinator has no capacity, it sends no ownership receipt, +fails the stream, and the worker remains responsible for deletion. An old +coordinator cannot issue a valid receipt. Admission is local to each CN; no +cross-CN shared counter or new wire field is required. + +Tickets are released exactly once after a successful workspace append accepts +the corresponding object name, or after `DeleteUnpublishedObjects` succeeds or +reports the object absent. A failed Delete, timed-out cleanup, stream removal, +transaction retirement, or CN shutdown timeout retains the ticket and its +name-only owner. A permit exhausted during a transaction returns an explicit +resource-exhausted error **before the next object upload**; the transaction +rolls back and retries deletion of objects it already owns. While S3 deletion +remains unavailable, later CN uploads remain backpressured. Admission must not +wait while holding a workspace, sinker, or batch-flow lock; a nonblocking +reservation failure avoids a capacity/deletion deadlock. + +For a name-only retry owner, successful bounded Delete batches retire their +names and tickets even if a later batch in the same attempt fails or times out. +An uncertain batch remains wholly owned; its response cannot prove which keys +were deleted. The next attempt starts with the remaining names. This makes +cleanup progress monotonic when each individual batch can complete within the +attempt budget, without changing TN sinker cleanup's full-snapshot contract. +The single CN retry worker may continue immediately after a successful task; +failed attempts retain their delay and rotate behind other work. Updating queue +metrics must not scan the whole backlog on every enqueue or completion. + +This does not make ownership durable. A CN process crash can still lose its +in-memory ledger and leave an object behind. The issue acceptance for this PR +therefore covers live-CN rollback/stream/teardown failures, not process-crash +recovery. Durable journaling of names before upload is a separate design if +crash recovery is required. Operators must not treat the finite ledger as a +substitute for S3 availability or durable GC. + +### Rollout and observable behavior + +Roll out ACK-capable coordinators before workers that require receipts. Drain +old workers before claiming the full ownership guarantee. New workers must +reject S3 handoff to an old coordinator before releasing ownership, and the +release plan must state that those remote writes can fail during an unordered +upgrade. A capability/version gate at scheduling time is preferable if mixed +versions must continue serving all writes; that is a separate compatibility +decision and must be implemented and validated before making that promise. + +The PR exports `mo_unpublished_s3_tickets{state="used|high_water"}` and +`mo_unpublished_s3_admission_failures_total` by CN service. It also exports +`mo_unpublished_s3_pending_cleanup_tasks`, +`mo_unpublished_s3_oldest_cleanup_age_seconds`, and a process-level +`mo_unpublished_s3_delete_failures_total`. The used-ticket gauge is an upper +bound on pending cleanup names, including objects in flight before metadata +acceptance. Configure alerts before rollout at 80% of 65,536 tickets for five +minutes, on any sustained nonzero oldest cleanup age above five minutes, and +on Delete failure growth. Object names are never metric labels. During a +storage outage, the system trades +write availability for bounded memory and retained ownership; successful +cleanup releases tickets and automatically resumes uploads. + +### Primary errors and transaction terminal behavior + +Cleanup ownership transfer is separate from the SQL result. When a statement +already has a business error, and its remaining cleanup records were accepted +by the workspace or CN retry worker, return that original error unchanged +through RPC; log and count the Delete failure as cleanup debt. Do not +`errors.Join` the two into a new error that loses the original MySQL code. +Apply the same rule to remote finalization, tombstone transfer, and remote +DELETE writers. With no primary error, report a cleanup error only if no live +owner accepted the obligation. A failed queue admission is an ownership +invariant failure, not an ordinary successful cleanup handoff; CN shutdown must +stop transaction producers before closing retry admission, and the transaction +must not silently drop the owner if that invariant is violated. + +At Commit and Rollback, delete unpublished objects with one bounded teardown +budget detached from a canceled request context. Nested writer and transfer +cleanup share that budget; an already-expired execution deadline must not make +the first rollback cleanup attempt instantly fail. If deletion fails but the CN +worker accepts the name-only record, Commit/Rollback proceed according to their +business outcome and the worker owns the remaining garbage. If transfer fails, +surface that failure and preserve the owner before any transaction handle is +retired. This changes only garbage cleanup reporting, not whether registered +transaction writes commit or roll back. + +### Alternatives and proof required + +An unbounded in-memory queue preserves local throughput during an outage but +can exhaust CN memory, so it is rejected. A cap enforced only when enqueueing +after an upload can drop the sole owner and is rejected. A durable ownership +journal would allow continued admission until its disk budget is reached and +would improve crash recovery, but adds atomic record-before-upload and +record-after-registration transitions, restart replay, disk-pressure handling, +and upgrade/migration semantics; it is out of scope for this revision. A +single-worker retry rate, larger timeout, or queue-slice compaction cannot +prove a capacity bound. + +Before implementation approval, deterministic tests must prove: reservation +precedes every real CN object Sync; no-capacity fails before upload; ambiguous +Sync retains its ticket; successful registration and deletion each release once; +failed deletion and terminal handoff retain the name/ticket without writer +buffers; 65,536 occupied tickets reject the next upload; concurrent reserve/ +release and retry survive race testing; remote coordinator capacity failure +withholds the receipt and leaves worker cleanup-owned; both mixed-version +directions behave as documented. Benchmarks must report retained bytes per +ticket and a repeated-spill hot-path cost. The existing package, race, and +multi-CN CI evidence is reusable only for behavior whose implementation and +test inputs have not changed. + +The progress refinement also requires a deterministic multi-batch failure +followed by recovery for workspace owners, writer shells, and transfer flows; +an ambiguous failed batch must retain its ticket. Queue tests must show prompt +drain after successful callbacks, rotation and bounded retries after failures, +and unchanged ownership during close. Recheck S3 handoff and transaction +consumers because their admission release points share the same ledger. diff --git a/pkg/cnservice/server.go b/pkg/cnservice/server.go index 7bd110636d396..9f88f947709e1 100644 --- a/pkg/cnservice/server.go +++ b/pkg/cnservice/server.go @@ -544,6 +544,8 @@ func (s *service) closeService() error { // otherwise retirement can wait for the work we have not stopped yet. s.closeSiriusRuntime, s.closeMongoDBRuntime, + // All transaction producers must stop before cleanup admission closes. + func() error { return s.colexecServer.CloseUnpublishedS3Cleanup(context.Background()) }, ) if s.closeErr != nil { return @@ -860,14 +862,7 @@ func (s *service) handleRequest( } } - // start a goroutine to handle one received message. - owned = false - cancelOwned = false - go func() { - defer release() - if value.Cancel != nil { - defer value.Cancel() - } + invoke := func() { s.pipelines.counter.Add(1) defer s.pipelines.counter.Add(-1) @@ -885,6 +880,24 @@ func (s *service) handleRequest( s._txnClient, s.aicm, s.acquireMessage) + } + // The connection read loop calls handleRequest in wire order. Ownership + // receipts are per batch, so ACKs must not be processed out of order by + // separate handler goroutines. + if msg.GetCmd() == pipeline.Method_PipelineBatchAck { + invoke() + return nil + } + + // start a goroutine to handle one received message. + owned = false + cancelOwned = false + go func() { + defer release() + if value.Cancel != nil { + defer value.Cancel() + } + invoke() }() return nil } diff --git a/pkg/cnservice/server_test.go b/pkg/cnservice/server_test.go index 6ad598bfc8e15..574432cfe6016 100644 --- a/pkg/cnservice/server_test.go +++ b/pkg/cnservice/server_test.go @@ -56,6 +56,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/pb/timestamp" "github.com/matrixorigin/matrixone/pkg/queryservice" qclient "github.com/matrixorigin/matrixone/pkg/queryservice/client" + "github.com/matrixorigin/matrixone/pkg/sql/colexec" "github.com/matrixorigin/matrixone/pkg/sql/compile" "github.com/matrixorigin/matrixone/pkg/txn/client" "github.com/matrixorigin/matrixone/pkg/txn/trace" @@ -325,6 +326,42 @@ func TestServiceCloseWithdrawalErrorIsLocallyComplete(t *testing.T) { } } +func TestServiceCloseCompletesAfterTransientS3CleanupFailure(t *testing.T) { + moruntime.RunTest(t.Name(), func(rt moruntime.Runtime) { + withdrawalErr := errors.New("withdrawal failed") + hc := &failingWithdrawalHeartbeatClient{testHAKClient: &testHAKClient{}, err: withdrawalErr} + mc := clusterservice.NewMOCluster(t.Name(), hc, time.Hour) + t.Cleanup(mc.Close) + ctrl := gomock.NewController(t) + ls := mock_lock.NewMockLockService(ctrl) + ls.EXPECT().Close().Return(nil).Times(2) + executor := colexec.NewServer(t.Name()) + var attempts atomic.Int32 + cleanup := func(context.Context) error { + if attempts.Add(1) == 1 { + return errors.New("temporary Delete failure") + } + return nil + } + sv := &service{ + cfg: &Config{UUID: t.Name()}, logger: zap.NewNop(), config: util.NewConfigData(nil), + stopper: stopper.NewStopper(t.Name()), bootstrapService: &testBootService{}, + mo: closeErrorMOServer{}, _hakeeperClient: hc, moCluster: mc, + server: closeOnlyRPCServer{}, lockService: ls, colexecServer: executor, + incrservice: closeOnlyIncrService{onClose: func() { + require.NoError(t, executor.RetryUnpublishedS3Cleanup(cleanup)) + }}, + viewMetadataAdmissionGeneration: 1, + } + // Remote withdrawal is diagnostic; it must not hide whether local + // teardown advanced past the recovered S3 cleanup and closed its tail. + require.ErrorIs(t, sv.Close(), withdrawalErr) + require.True(t, sv.CloseComplete()) + require.Equal(t, int32(2), attempts.Load()) + require.Equal(t, 1, hc.closed) + }) +} + func TestMakeRSSCacheEvictorEvictsMemoryCacheOnly(t *testing.T) { oldMemoryEvictor := evictMemoryCachesToCapacityPercent defer func() { @@ -1541,6 +1578,55 @@ func TestPipelineAdmissionRejectCancelsRequestOnce(t *testing.T) { require.Equal(t, int32(1), cancelCount.Load()) } +func TestPipelineBatchAckRunsBeforeIngressReturns(t *testing.T) { + started := make(chan struct{}) + releaseHandler := make(chan struct{}) + var releaseOnce sync.Once + t.Cleanup(func() { releaseOnce.Do(func() { close(releaseHandler) }) }) + var cancelCount atomic.Int32 + s := &service{cfg: &Config{}} + s.requestHandler = func( + _ context.Context, + _ string, + _ morpc.Message, + _ morpc.ClientSession, + _ engine.Engine, + _ fileservice.FileService, + _ lockservice.LockService, + _ qclient.QueryClient, + _ logservice.CNHAKeeperClient, + _ udf.Service, + _ client.TxnClient, + _ *defines.AutoIncrCacheManager, + _ func() morpc.Message, + ) error { + close(started) + <-releaseHandler + return nil + } + returned := make(chan error, 1) + go func() { + returned <- s.handleRequest(context.Background(), morpc.RPCMessage{ + Message: &pipeline.Message{Cmd: pipeline.Method_PipelineBatchAck, Sid: pipeline.Status_Last}, + Cancel: func() { cancelCount.Add(1) }, + }, 0, nil) + }() + select { + case <-started: + case <-time.After(time.Second): + t.Fatal("ACK handler did not start") + } + select { + case <-returned: + t.Fatal("ingress returned before ACK processing completed") + case <-time.After(50 * time.Millisecond): + } + releaseOnce.Do(func() { close(releaseHandler) }) + require.NoError(t, <-returned) + require.Equal(t, int32(1), cancelCount.Load()) + require.Zero(t, s.pipelines.counter.Load()) +} + func TestHandleRequestPropagatesConfiguredRPCMaxMessageSize(t *testing.T) { const configuredLimit = 32 * 1024 s := &service{cfg: &Config{UUID: t.Name()}} diff --git a/pkg/objectio/ioutil/sink_pool.go b/pkg/objectio/ioutil/sink_pool.go index 102cefd1aa9d4..6e4835c8307c5 100644 --- a/pkg/objectio/ioutil/sink_pool.go +++ b/pkg/objectio/ioutil/sink_pool.go @@ -194,7 +194,7 @@ func (p *SinkPool) runSyncWorker() { syncStart := time.Now() stats, err := job.fSinker.Sync(r.ctx) atomic.AddInt64(&r.syncNs, int64(time.Since(syncStart))) - if err != nil { + if err != nil && !objectSyncWasNotStarted(err) { if name := activeFileSinkerObjectName(job.fSinker); name != "" { r.mu.Lock() r.unpublished = append(r.unpublished, name) diff --git a/pkg/objectio/ioutil/sink_pool_test.go b/pkg/objectio/ioutil/sink_pool_test.go index 5fc2a703bffcd..7480b98df79e1 100644 --- a/pkg/objectio/ioutil/sink_pool_test.go +++ b/pkg/objectio/ioutil/sink_pool_test.go @@ -16,6 +16,7 @@ package ioutil import ( "context" + "errors" "fmt" "sync" "sync/atomic" @@ -151,6 +152,67 @@ func mockFactory(sinkErr, syncErr error) FileSinkerFactory { } } +func TestAdmittedFileSinkerReservesBeforeSync(t *testing.T) { + base := &mockFileSinker{activeName: "object", syncStart: make(chan struct{})} + admitted := &admittedFileSinker{ + FileSinker: base, + reserve: func(name string) error { + require.Equal(t, "object", name) + return errors.New("cleanup capacity exhausted") + }, + } + _, err := admitted.Sync(context.Background()) + require.ErrorContains(t, err, "cleanup capacity exhausted") + select { + case <-base.syncStart: + t.Fatal("Sync ran despite failed pre-upload admission") + default: + } +} + +func TestSinkerRejectedAdmissionDoesNotCreateCleanupDebt(t *testing.T) { + for _, pipeline := range []bool{false, true} { + t.Run(fmt.Sprintf("pipeline=%t", pipeline), func(t *testing.T) { + proc := testutil.NewProc(t) + fs, err := fileservice.NewMemoryFS("shared", fileservice.DisabledCacheConfig, nil) + require.NoError(t, err) + attrs, typs, _ := mockSchema(3, -1) + base := &mockFileSinker{activeName: "not-written", syncStart: make(chan struct{})} + admissionErr := errors.New("cleanup capacity exhausted") + opts := []SinkerOption{ + WithMemorySizeThreshold(1), + WithObjectSyncAdmission(func(name string) error { + if name != "not-written" { + return fmt.Errorf("unexpected object name %q", name) + } + return admissionErr + }, func(string) {}), + } + if pipeline { + opts = append(opts, WithPipelineFlush()) + } + sinker := NewSinker(-1, attrs, typs, + func(*mpool.MPool, fileservice.FileService) FileSinker { return base }, + proc.Mp(), fs, opts...) + bat := containers.MockBatch(typs, 8192, -1, nil) + err = sinker.Write(context.Background(), containers.ToCNBatch(bat)) + if err == nil { + err = sinker.Sync(context.Background()) + } + require.ErrorIs(t, err, admissionErr) + files, cleanupErr := sinker.DeletePersisted(context.Background()) + require.NoError(t, cleanupErr) + require.Empty(t, files, "a rejected upload must not enter cleanup debt") + select { + case <-base.syncStart: + t.Fatal("file Sync started despite rejected admission") + default: + } + _ = sinker.Close() + }) + } +} + // ---------- SinkPool.Submit tests ---------- func TestSinkPool_SubmitSuccess(t *testing.T) { diff --git a/pkg/objectio/ioutil/sinker.go b/pkg/objectio/ioutil/sinker.go index 42bda58faac3d..9963302e59e22 100644 --- a/pkg/objectio/ioutil/sinker.go +++ b/pkg/objectio/ioutil/sinker.go @@ -32,6 +32,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/logutil" "github.com/matrixorigin/matrixone/pkg/objectio" "github.com/matrixorigin/matrixone/pkg/objectio/mergeutil" + metricv2 "github.com/matrixorigin/matrixone/pkg/util/metric/v2" "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/containers" ) @@ -107,6 +108,47 @@ func WithChunkedColumnPolicy(policy objectio.ChunkedColumnPolicy) SinkerOption { } } +// WithObjectSyncAdmission reserves cleanup capacity before each file Sync. +// Release runs only after this sinker successfully deletes an object. +func WithObjectSyncAdmission(reserve func(string) error, release func(string)) SinkerOption { + return func(sinker *Sinker) { + sinker.config.reserveObject = reserve + sinker.config.releaseObject = release + } +} + +type admittedFileSinker struct { + FileSinker + reserve func(string) error +} + +// objectSyncAdmissionError means no object write began. The ordinary Sync +// error path must retain the name because persistence may be ambiguous. +type objectSyncAdmissionError struct{ cause error } + +func (e *objectSyncAdmissionError) Error() string { return e.cause.Error() } +func (e *objectSyncAdmissionError) Unwrap() error { return e.cause } + +func objectSyncWasNotStarted(err error) bool { + var admissionErr *objectSyncAdmissionError + return errors.As(err, &admissionErr) +} + +func (s *admittedFileSinker) ActiveObjectName() string { + return activeFileSinkerObjectName(s.FileSinker) +} + +func (s *admittedFileSinker) Sync(ctx context.Context) (*objectio.ObjectStats, error) { + name := s.ActiveObjectName() + if name == "" { + return nil, &objectSyncAdmissionError{moerr.NewInvalidStateNoCtx("file sinker has no object name before Sync")} + } + if err := s.reserve(name); err != nil { + return nil, &objectSyncAdmissionError{err} + } + return s.FileSinker.Sync(ctx) +} + type FileSinker interface { Sink(context.Context, *batch.Batch) error Sync(context.Context) (*objectio.ObjectStats, error) @@ -353,6 +395,12 @@ func NewSinker( return fileSinker } } + if reserve := sinker.config.reserveObject; reserve != nil { + factory := sinker.fSinker.factory + sinker.fSinker.factory = func(mp *mpool.MPool, fs fileservice.FileService) FileSinker { + return &admittedFileSinker{FileSinker: factory(mp, fs), reserve: reserve} + } + } sinker.fillDefaults() return sinker @@ -406,6 +454,8 @@ type Sinker struct { tailSizeCap int offHeap bool chunkedColumnPolicy objectio.ChunkedColumnPolicy + reserveObject func(string) error + releaseObject func(string) } fSinker struct { executor FileSinker @@ -493,8 +543,33 @@ func DeleteUnpublishedObjects( fs fileservice.FileService, files ...string, ) (int, error) { + unique, _, err := deleteUnpublishedObjectBatches(ctx, fs, files) + return len(unique), err +} + +// DeleteUnpublishedObjectsWithProgress reports only names in fully confirmed +// Delete batches. A failed batch can have partial effects, so it and every +// unattempted batch remain owned. The remaining slice does not retain the +// backing array of completed names after the caller stores it for retry. +func DeleteUnpublishedObjectsWithProgress( + ctx context.Context, + fs fileservice.FileService, + files ...string, +) (completed, remaining []string, err error) { + unique, completedCount, err := deleteUnpublishedObjectBatches(ctx, fs, files) + if err != nil { + return unique[:completedCount], append([]string(nil), unique[completedCount:]...), err + } + return unique, nil, nil +} + +func deleteUnpublishedObjectBatches( + ctx context.Context, + fs fileservice.FileService, + files []string, +) (unique []string, completed int, err error) { seen := make(map[string]struct{}, len(files)) - unique := make([]string, 0, len(files)) + unique = make([]string, 0, len(files)) for _, file := range files { if file == "" { continue @@ -513,14 +588,15 @@ func DeleteUnpublishedObjects( err := fs.Delete(deleteCtx, unique[start:end]...) cancel() if err != nil && !moerr.IsMoErrCode(err, moerr.ErrFileNotFound) { - return len(unique), errors.Join( + metricv2.UnpublishedS3DeleteFailuresCounter.Inc() + return unique, start, errors.Join( moerr.NewInternalErrorf( ctx, "delete unpublished objects [%d:%d]", start, end), err, ) } } - return len(unique), nil + return unique, len(unique), nil } // DeletePersisted deletes every object that this sinker has persisted, or may @@ -557,6 +633,11 @@ func (sinker *Sinker) DeletePersisted(ctx context.Context) ([]string, error) { if err != nil { return files, err } + if sinker.config.releaseObject != nil { + for _, name := range files { + sinker.config.releaseObject(name) + } + } sinker.staged.persisted = sinker.staged.persisted[:0] sinker.result.persisted = sinker.result.persisted[:0] @@ -795,7 +876,7 @@ func (sinker *Sinker) syncFileSinker(ctx context.Context, fSinker FileSinker) er stats, err := fSinker.Sync(ctx) atomic.AddInt64(&sinker.timing.syncNs, int64(time.Since(syncStart))) if err != nil { - if name := activeFileSinkerObjectName(fSinker); name != "" { + if name := activeFileSinkerObjectName(fSinker); name != "" && !objectSyncWasNotStarted(err) { sinker.staged.unpublished = append(sinker.staged.unpublished, name) } return err diff --git a/pkg/objectio/ioutil/sinker_test.go b/pkg/objectio/ioutil/sinker_test.go index d0c6e4da249d6..211b08a0d63e5 100644 --- a/pkg/objectio/ioutil/sinker_test.go +++ b/pkg/objectio/ioutil/sinker_test.go @@ -206,6 +206,43 @@ type persistThenErrorFS struct { persisted string } +type failSecondUnpublishedDeleteBatchFS struct { + fileservice.FileService + batchSizes []int +} + +func (fs *failSecondUnpublishedDeleteBatchFS) Delete(_ context.Context, names ...string) error { + fs.batchSizes = append(fs.batchSizes, len(names)) + if len(fs.batchSizes) == 2 { + return errors.New("second Delete batch failed") + } + return nil +} + +func TestDeleteUnpublishedObjectsReportsOnlyConfirmedBatches(t *testing.T) { + names := make([]string, 1001) + for i := range names { + names[i] = fmt.Sprintf("unpublished-%d", i) + } + fs := &failSecondUnpublishedDeleteBatchFS{} + completed, remaining, err := DeleteUnpublishedObjectsWithProgress( + context.Background(), fs, append(names, "", names[0])...) + require.ErrorContains(t, err, "second Delete batch failed") + require.Len(t, completed, 1000) + require.Len(t, remaining, 1) + require.Equal(t, []int{1000, 1}, fs.batchSizes) + retried, stillPending, err := DeleteUnpublishedObjectsWithProgress(context.Background(), fs, remaining...) + require.NoError(t, err) + require.Len(t, retried, 1) + require.Empty(t, stillPending) + require.Equal(t, []int{1000, 1, 1}, fs.batchSizes) + + legacyFS := &failSecondUnpublishedDeleteBatchFS{} + count, err := DeleteUnpublishedObjects(context.Background(), legacyFS, names...) + require.ErrorContains(t, err, "second Delete batch failed") + require.Equal(t, len(names), count, "the existing cleanup count keeps its full-snapshot contract") +} + func (fs *persistThenErrorFS) Write(ctx context.Context, vector fileservice.IOVector) error { if err := fs.FileService.Write(ctx, vector); err != nil { return err @@ -237,3 +274,31 @@ func TestSinkerDeletePersistedAfterAmbiguousSyncFailure(t *testing.T) { _, err = baseFS.StatFile(ctx, fs.persisted) require.True(t, moerr.IsMoErrCode(err, moerr.ErrFileNotFound)) } + +func TestSinkerAdmissionRetainsAmbiguousSyncUntilDelete(t *testing.T) { + ctx := context.Background() + proc := testutil.NewProc(t) + baseFS, err := fileservice.NewMemoryFS("shared", fileservice.DisabledCacheConfig, nil) + require.NoError(t, err) + fs := &persistThenErrorFS{FileService: baseFS} + attrs, typs, seqnums := mockSchema(3, 2) + reserved := make(map[string]struct{}) + sinker := NewSinker(2, attrs, typs, NewFSinkerImplFactory(seqnums, 2, true, false, 0), proc.Mp(), fs, + WithObjectSyncAdmission(func(name string) error { + if len(reserved) == 1 { + return errors.New("cleanup capacity exhausted") + } + reserved[name] = struct{}{} + return nil + }, func(name string) { delete(reserved, name) }), + ) + t.Cleanup(func() { require.NoError(t, sinker.Close()) }) + bat := containers.MockBatch(typs, 1000, 2, nil) + require.NoError(t, sinker.Write(ctx, containers.ToCNBatch(bat))) + require.ErrorContains(t, sinker.Sync(ctx), "injected post-persist sync failure") + require.Contains(t, reserved, fs.persisted, "reservation must precede persistence") + files, err := sinker.DeletePersisted(ctx) + require.NoError(t, err) + require.Equal(t, []string{fs.persisted}, files) + require.Empty(t, reserved, "successful deletion releases the ticket") +} diff --git a/pkg/pb/pipeline/pipeline.pb.go b/pkg/pb/pipeline/pipeline.pb.go index 7636f8332ac6d..0a68ef9460953 100644 --- a/pkg/pb/pipeline/pipeline.pb.go +++ b/pkg/pb/pipeline/pipeline.pb.go @@ -273,9 +273,12 @@ type Message struct { AcceptedBatchCreditBytes uint64 `protobuf:"varint,16,opt,name=accepted_batch_credit_bytes,json=acceptedBatchCreditBytes,proto3" json:"accepted_batch_credit_bytes,omitempty"` BatchSequence uint64 `protobuf:"varint,17,opt,name=batch_sequence,json=batchSequence,proto3" json:"batch_sequence,omitempty"` BatchAckSequence uint64 `protobuf:"varint,18,opt,name=batch_ack_sequence,json=batchAckSequence,proto3" json:"batch_ack_sequence,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + // ACK only: the receiving workspace retained cleanup ownership for this + // batch's S3 objects. This receipt is not cumulative like batch credits. + BatchAckS3OwnershipRetained bool `protobuf:"varint,19,opt,name=batch_ack_s3_ownership_retained,json=batchAckS3OwnershipRetained,proto3" json:"batch_ack_s3_ownership_retained,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *Message) Reset() { *m = Message{} } @@ -436,6 +439,13 @@ func (m *Message) GetBatchAckSequence() uint64 { return 0 } +func (m *Message) GetBatchAckS3OwnershipRetained() bool { + if m != nil { + return m.BatchAckS3OwnershipRetained + } + return false +} + type Connector struct { PipelineId int32 `protobuf:"varint,1,opt,name=pipeline_id,json=pipelineId,proto3" json:"pipeline_id,omitempty"` ConnectorIndex int32 `protobuf:"varint,2,opt,name=connector_index,json=connectorIndex,proto3" json:"connector_index,omitempty"` @@ -6986,591 +6996,593 @@ func init() { func init() { proto.RegisterFile("pipeline.proto", fileDescriptor_7ac67a7adf3df9c7) } var fileDescriptor_7ac67a7adf3df9c7 = []byte{ - // 9340 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x7d, 0x4d, 0x6f, 0x1c, 0xc7, - 0xb6, 0x98, 0x87, 0xe4, 0x70, 0x66, 0xce, 0x7c, 0xb2, 0xf8, 0x35, 0xfa, 0xb0, 0x24, 0x8f, 0x2d, - 0x9b, 0x96, 0x65, 0x4a, 0xa6, 0xed, 0x7b, 0x6d, 0xdf, 0xe7, 0xeb, 0x47, 0x51, 0xa2, 0xcd, 0x6b, - 0x51, 0xe2, 0x6d, 0x52, 0xd7, 0x80, 0x81, 0xa4, 0xd1, 0xec, 0xae, 0x19, 0xf6, 0x65, 0x4f, 0x77, - 0xab, 0xab, 0x47, 0x22, 0xb5, 0x4a, 0x36, 0x01, 0x92, 0x3f, 0x90, 0x8b, 0x00, 0x01, 0x6e, 0x90, - 0x2c, 0x12, 0x64, 0xf7, 0x82, 0x97, 0x65, 0xd6, 0x0f, 0x41, 0x16, 0x6f, 0x95, 0x55, 0x10, 0x04, - 0xf7, 0x02, 0x59, 0x24, 0x40, 0x10, 0x04, 0x48, 0x90, 0xcd, 0x03, 0x82, 0x73, 0x4e, 0x55, 0x7f, - 0x0c, 0x47, 0x92, 0xed, 0x04, 0x6f, 0x93, 0xb7, 0xeb, 0x3e, 0xe7, 0x54, 0x75, 0x4d, 0xd5, 0xa9, - 0xf3, 0x5d, 0x35, 0xd0, 0x89, 0xfd, 0x58, 0x06, 0x7e, 0x28, 0x37, 0xe3, 0x24, 0x4a, 0x23, 0x51, - 0x37, 0xef, 0x97, 0x3f, 0x1c, 0xf9, 0xe9, 0xc9, 0xe4, 0x78, 0xd3, 0x8d, 0xc6, 0x77, 0x46, 0xd1, - 0x28, 0xba, 0x43, 0x04, 0xc7, 0x93, 0x21, 0xbd, 0xd1, 0x0b, 0x3d, 0x71, 0xc3, 0xcb, 0x10, 0x44, - 0xee, 0xa9, 0x79, 0x8e, 0x03, 0x27, 0xd4, 0xcf, 0xdd, 0xd4, 0x1f, 0x4b, 0x95, 0x3a, 0xe3, 0x58, - 0x03, 0x1a, 0xe9, 0x99, 0xc6, 0x0d, 0x7e, 0xbf, 0x08, 0xb5, 0x7d, 0xa9, 0x94, 0x33, 0x92, 0x62, - 0x00, 0xf3, 0xca, 0xf7, 0xfa, 0x95, 0x1b, 0x95, 0x8d, 0xce, 0x56, 0x6f, 0x33, 0x1b, 0xd6, 0x61, - 0xea, 0xa4, 0x13, 0x65, 0x21, 0x12, 0x69, 0xdc, 0xb1, 0xd7, 0x9f, 0x9b, 0xa6, 0xd9, 0x97, 0xe9, - 0x49, 0xe4, 0x59, 0x88, 0x14, 0x3d, 0x98, 0x97, 0x49, 0xd2, 0x9f, 0xbf, 0x51, 0xd9, 0x68, 0x59, - 0xf8, 0x28, 0x04, 0x2c, 0x78, 0x4e, 0xea, 0xf4, 0x17, 0x08, 0x44, 0xcf, 0xe2, 0x1d, 0xe8, 0xc4, - 0x49, 0xe4, 0xda, 0x7e, 0x38, 0x8c, 0x6c, 0xc2, 0x56, 0x09, 0xdb, 0x42, 0xe8, 0x5e, 0x38, 0x8c, - 0xee, 0x23, 0x55, 0x1f, 0x6a, 0x4e, 0xe8, 0x04, 0xe7, 0x4a, 0xf6, 0x17, 0x09, 0x6d, 0x5e, 0x45, - 0x07, 0xe6, 0x7c, 0xaf, 0x5f, 0xbb, 0x51, 0xd9, 0x58, 0xb0, 0xe6, 0x7c, 0x0f, 0xbf, 0x31, 0x99, - 0xf8, 0x5e, 0xbf, 0xce, 0xdf, 0xc0, 0x67, 0x31, 0x80, 0x56, 0x28, 0xa5, 0xf7, 0x28, 0x4a, 0x2d, - 0x19, 0x07, 0xe7, 0xfd, 0xc6, 0x8d, 0xca, 0x46, 0xdd, 0x2a, 0xc1, 0xc4, 0x65, 0xa8, 0x7b, 0xf2, - 0x78, 0x32, 0xda, 0x57, 0xa3, 0x3e, 0xdc, 0xa8, 0x6c, 0x34, 0xac, 0xec, 0x5d, 0x1c, 0xc1, 0x7a, - 0x22, 0x9f, 0x4e, 0xa4, 0x4a, 0xa5, 0x67, 0xa7, 0xd2, 0x49, 0xbc, 0xe8, 0x79, 0x68, 0x8f, 0x23, - 0x4f, 0xf6, 0x9b, 0x34, 0x03, 0x57, 0x8b, 0xb3, 0x94, 0x48, 0x67, 0x7c, 0xa4, 0x89, 0xf6, 0x23, - 0x4f, 0x5a, 0xab, 0x59, 0xe3, 0x22, 0x58, 0x58, 0xb0, 0xe6, 0xb8, 0xae, 0x8c, 0x2f, 0x76, 0xda, - 0xfa, 0x01, 0x9d, 0xae, 0x98, 0xb6, 0xa5, 0x3e, 0xbf, 0x82, 0xab, 0xf9, 0x48, 0x8f, 0x9d, 0xd4, - 0x3d, 0xb1, 0xdd, 0x44, 0x7a, 0x7e, 0x6a, 0xbb, 0xd1, 0x24, 0x4c, 0xfb, 0xed, 0x1b, 0x95, 0x8d, - 0xb6, 0x75, 0x29, 0xa3, 0xb9, 0x87, 0x24, 0x3b, 0x44, 0xb1, 0x83, 0x04, 0xaf, 0xe8, 0xe0, 0xf8, - 0x3c, 0x95, 0xaa, 0xdf, 0xa1, 0x89, 0x9e, 0xd9, 0xc1, 0x3d, 0x24, 0x10, 0x5f, 0xc2, 0x95, 0xec, - 0x57, 0xcd, 0x18, 0x40, 0x97, 0x06, 0xd0, 0x37, 0x24, 0x17, 0xbe, 0xff, 0xd2, 0xe6, 0xfc, 0xf9, - 0x1e, 0x7d, 0x7e, 0x56, 0x73, 0xfe, 0xfa, 0x4d, 0xe8, 0x70, 0x2b, 0x85, 0x03, 0x0c, 0x5d, 0xd9, - 0x5f, 0xa2, 0x16, 0x6d, 0x82, 0x1e, 0x6a, 0xa0, 0xb8, 0x0d, 0x82, 0xc9, 0x1c, 0xf7, 0x34, 0x27, - 0x15, 0x44, 0xda, 0x23, 0xcc, 0xb6, 0x7b, 0x6a, 0xa8, 0xbf, 0x58, 0xf8, 0xdd, 0xef, 0xaf, 0xbf, - 0x31, 0x78, 0x02, 0x8d, 0x9d, 0x28, 0x0c, 0xa5, 0x9b, 0x46, 0x89, 0xb8, 0x0e, 0x4d, 0xb3, 0x38, - 0xb6, 0xde, 0x2b, 0x55, 0x0b, 0x0c, 0x68, 0xcf, 0x13, 0xef, 0x41, 0xd7, 0x35, 0xd4, 0xb6, 0x1f, - 0x7a, 0xf2, 0x8c, 0x36, 0x4b, 0xd5, 0xea, 0x64, 0xe0, 0x3d, 0x84, 0x0e, 0xfe, 0xdb, 0x3c, 0xd4, - 0x0e, 0x4f, 0x26, 0xc3, 0x61, 0x20, 0xc5, 0x3b, 0xd0, 0xd6, 0x8f, 0x3b, 0x51, 0xb0, 0xe7, 0x9d, - 0xe9, 0x7e, 0xcb, 0x40, 0x71, 0x03, 0x9a, 0x1a, 0x70, 0x74, 0x1e, 0x4b, 0xdd, 0x6d, 0x11, 0x54, - 0xee, 0x67, 0xdf, 0x0f, 0x69, 0x0f, 0xce, 0x5b, 0x65, 0xe0, 0x14, 0x95, 0x73, 0x46, 0xdb, 0xb2, - 0x4c, 0xe5, 0xd0, 0xd7, 0xb6, 0x03, 0xff, 0x99, 0xb4, 0xe4, 0x68, 0x27, 0x4c, 0x69, 0x73, 0x56, - 0xad, 0x22, 0x48, 0x6c, 0xc1, 0xaa, 0xe2, 0x26, 0x76, 0xe2, 0x84, 0x23, 0xa9, 0xec, 0x89, 0x1f, - 0xa6, 0x3f, 0xfb, 0xa4, 0xbf, 0x78, 0x63, 0x7e, 0x63, 0xc1, 0x5a, 0xd6, 0x48, 0x8b, 0x70, 0x4f, - 0x08, 0x25, 0xee, 0xc2, 0xca, 0x54, 0x1b, 0x6e, 0x52, 0xbb, 0x31, 0xbf, 0x31, 0x6f, 0x89, 0x52, - 0x93, 0x3d, 0x6a, 0xf1, 0x00, 0x96, 0x92, 0x49, 0x88, 0x22, 0x6c, 0xd7, 0x0f, 0x52, 0x99, 0x1c, - 0xc6, 0xd2, 0xa5, 0x4d, 0xde, 0xdc, 0x5a, 0xdf, 0x24, 0x29, 0x67, 0x4d, 0xa3, 0xad, 0x8b, 0x2d, - 0xc4, 0xed, 0x6c, 0xf2, 0x1e, 0x9c, 0xc5, 0x09, 0x49, 0x82, 0xe6, 0x16, 0x70, 0x07, 0x08, 0xb1, - 0x8a, 0x68, 0x71, 0x0b, 0x96, 0xbc, 0xc4, 0xf1, 0x43, 0xdb, 0x09, 0x02, 0xfb, 0x78, 0xe2, 0x9e, - 0xca, 0x54, 0x91, 0x74, 0xa8, 0x5b, 0x5d, 0x42, 0x6c, 0x07, 0xc1, 0x3d, 0x06, 0x8b, 0x77, 0xa1, - 0xab, 0xd2, 0xc4, 0x0f, 0x47, 0xf6, 0x89, 0xa3, 0x4e, 0xec, 0x53, 0x79, 0x4e, 0xc2, 0xa1, 0x6e, - 0xb5, 0x19, 0xfc, 0x8d, 0xa3, 0x4e, 0xbe, 0x95, 0xe7, 0x83, 0xff, 0x3d, 0x07, 0xf5, 0xfb, 0xbe, - 0x8a, 0x91, 0xcb, 0xc4, 0x3a, 0xd4, 0x86, 0x93, 0xd0, 0xcd, 0x79, 0x68, 0x11, 0x5f, 0xf7, 0x3c, - 0xf1, 0x27, 0xd0, 0x0d, 0x22, 0xd7, 0x09, 0xec, 0x8c, 0x5d, 0xfa, 0x73, 0x37, 0xe6, 0x37, 0x9a, - 0x5b, 0xcb, 0xb9, 0x54, 0xc8, 0xd8, 0xd1, 0xea, 0x10, 0x6d, 0xce, 0x9e, 0x5f, 0x42, 0x2f, 0x91, - 0xe3, 0x28, 0x95, 0x85, 0xe6, 0xf3, 0xd4, 0x5c, 0xe4, 0xcd, 0xbf, 0x4b, 0x9c, 0xf8, 0x11, 0x8a, - 0x92, 0x2e, 0xd3, 0xe6, 0xcd, 0x3f, 0x2a, 0xac, 0xa8, 0x1c, 0xd9, 0xbe, 0x77, 0x66, 0xd3, 0x07, - 0xfa, 0x0b, 0x37, 0xe6, 0x37, 0xaa, 0xf9, 0xf2, 0xc8, 0xd1, 0x9e, 0x77, 0xf6, 0x10, 0x31, 0xe2, - 0x63, 0x58, 0x9b, 0x6e, 0xc2, 0xbd, 0xf6, 0xab, 0xd4, 0x66, 0xb9, 0xd4, 0xc6, 0x22, 0x94, 0x78, - 0x0b, 0x5a, 0xa6, 0x51, 0x8a, 0xac, 0xbc, 0xc8, 0xcc, 0xa5, 0x0a, 0xac, 0xbc, 0x0e, 0x35, 0x5f, - 0xd9, 0xca, 0x0f, 0x4f, 0x49, 0xc6, 0xd7, 0xad, 0x45, 0x5f, 0x1d, 0xfa, 0xe1, 0xa9, 0xb8, 0x04, - 0xf5, 0x44, 0xba, 0x8c, 0xa9, 0x13, 0xa6, 0x96, 0x48, 0x97, 0x50, 0xeb, 0x80, 0x8f, 0xb6, 0x9b, - 0x4a, 0x2d, 0xe9, 0x17, 0x13, 0xe9, 0xee, 0xa4, 0x72, 0xa0, 0xa0, 0xba, 0x2f, 0x93, 0x91, 0x44, - 0x61, 0x8f, 0x0d, 0x0f, 0x5d, 0x27, 0xa4, 0x79, 0xaf, 0x5b, 0xd9, 0x3b, 0xaa, 0x9a, 0xd8, 0x49, - 0x52, 0xdf, 0x09, 0x68, 0x6b, 0xd5, 0x2d, 0xf3, 0x2a, 0xae, 0x40, 0x43, 0xa5, 0x4e, 0x92, 0xe2, - 0xaf, 0xa3, 0x2d, 0x55, 0xb5, 0xea, 0x04, 0xc0, 0x5d, 0xb9, 0x0e, 0x35, 0x19, 0x7a, 0x84, 0x5a, - 0xe0, 0x95, 0x94, 0xa1, 0xb7, 0xe7, 0x9d, 0x0d, 0xfe, 0x55, 0x05, 0xda, 0xfb, 0x93, 0x20, 0xf5, - 0xb7, 0x93, 0xd1, 0x44, 0x8e, 0xc3, 0x14, 0x55, 0xd4, 0x7d, 0x5f, 0xa5, 0xfa, 0xcb, 0xf4, 0x2c, - 0x36, 0xa0, 0xf1, 0x75, 0x12, 0x4d, 0x62, 0xe2, 0x4a, 0x5e, 0xe9, 0x22, 0x57, 0xe6, 0x48, 0xe4, - 0xe0, 0xc7, 0x89, 0x27, 0x93, 0x7b, 0xe7, 0x44, 0x3b, 0x7f, 0x81, 0xb6, 0x88, 0x16, 0x57, 0xa1, - 0x71, 0x28, 0x63, 0x27, 0x71, 0x90, 0x05, 0x16, 0x48, 0xaf, 0xe5, 0x00, 0xfc, 0xad, 0x44, 0xbc, - 0xe7, 0xe9, 0x8d, 0x6d, 0x5e, 0x07, 0xff, 0xb4, 0x02, 0x8d, 0xed, 0xd1, 0x28, 0x91, 0x23, 0x27, - 0x25, 0x25, 0x1b, 0xc5, 0x34, 0xde, 0x79, 0x6b, 0x2e, 0x8a, 0x49, 0x91, 0xe3, 0x2f, 0xe0, 0x09, - 0xa2, 0x67, 0x71, 0x0d, 0x16, 0xe4, 0xec, 0x01, 0x11, 0x5c, 0xac, 0xc1, 0xa2, 0x1b, 0x85, 0x43, - 0x7f, 0xa4, 0xd5, 0xbf, 0x7e, 0x13, 0x5f, 0x40, 0x93, 0x9f, 0x98, 0x07, 0xaa, 0xa4, 0xfb, 0x2e, - 0x71, 0xf3, 0x6c, 0x04, 0x3b, 0x44, 0x81, 0x1c, 0x61, 0x81, 0x9b, 0x3d, 0x0f, 0xfe, 0xb8, 0x00, - 0x55, 0x9a, 0x19, 0x5c, 0x1b, 0x54, 0xe7, 0xb6, 0x7c, 0xe6, 0x04, 0x66, 0x49, 0x11, 0xf0, 0xe0, - 0x99, 0x13, 0x88, 0x1b, 0x50, 0xc5, 0x21, 0xa8, 0x19, 0x13, 0xcb, 0x08, 0xf1, 0x2e, 0x54, 0xf1, - 0xeb, 0xaa, 0x3c, 0x7a, 0xfc, 0xc6, 0xbd, 0x85, 0xbf, 0xf8, 0x8f, 0xd7, 0xdf, 0xb0, 0x18, 0x2d, - 0xde, 0x83, 0x05, 0x67, 0x34, 0x52, 0xb4, 0x11, 0x4a, 0x7b, 0x31, 0x1b, 0xa9, 0x45, 0x04, 0xe2, - 0x53, 0x68, 0xf0, 0xa2, 0x23, 0x75, 0x95, 0xa8, 0xd7, 0x0b, 0x66, 0x52, 0x91, 0x1f, 0xac, 0x9c, - 0x12, 0x97, 0xcb, 0x57, 0x5a, 0x02, 0xd1, 0x76, 0xa8, 0x5b, 0x39, 0x00, 0xed, 0x98, 0x38, 0x91, - 0xdb, 0x41, 0x10, 0xb9, 0x87, 0xfe, 0x0b, 0xa9, 0xad, 0x9e, 0x12, 0x4c, 0xbc, 0x0b, 0x9d, 0x03, - 0xe6, 0x57, 0x4b, 0xaa, 0x49, 0x90, 0x2a, 0x6d, 0x09, 0x4d, 0x41, 0xc5, 0x26, 0x88, 0x12, 0xe4, - 0x88, 0x7e, 0x7e, 0xe3, 0xc6, 0xfc, 0x46, 0xdb, 0x9a, 0x81, 0x11, 0x6f, 0x43, 0x7b, 0x84, 0x33, - 0x8d, 0x02, 0x6e, 0x18, 0x38, 0x68, 0x24, 0xcd, 0xa3, 0x11, 0x65, 0x80, 0xbb, 0x81, 0x33, 0xa2, - 0x1d, 0x12, 0xfb, 0x41, 0x60, 0x8f, 0xe5, 0x98, 0xa4, 0xdf, 0xbc, 0x55, 0x27, 0xc0, 0xbe, 0x1c, - 0x8b, 0xf7, 0x61, 0x89, 0x88, 0xed, 0xe3, 0xf3, 0x5c, 0x44, 0xb6, 0x48, 0x3a, 0x74, 0x08, 0x71, - 0xef, 0x5c, 0xcb, 0x48, 0xf1, 0x3e, 0xf4, 0xbc, 0xf3, 0xd0, 0x19, 0xfb, 0xae, 0x6d, 0xfa, 0x27, - 0xd3, 0x05, 0xc5, 0x2e, 0xc3, 0xbf, 0xd6, 0x60, 0x14, 0x3c, 0x72, 0x1c, 0xa7, 0xe7, 0x19, 0xa1, - 0xad, 0x24, 0xee, 0x50, 0x34, 0x55, 0x50, 0x97, 0x2c, 0x13, 0xd6, 0x90, 0x1f, 0xca, 0x74, 0xcf, - 0x53, 0xa8, 0xff, 0x2f, 0x36, 0x22, 0xdb, 0xa4, 0x6e, 0xf5, 0xa6, 0x1b, 0x0c, 0xfe, 0xe5, 0x02, - 0x2c, 0xee, 0x85, 0x4a, 0x26, 0x29, 0x0a, 0x0e, 0x67, 0x38, 0x94, 0x6e, 0x2a, 0x59, 0x60, 0x2f, - 0x58, 0xd9, 0x3b, 0xae, 0xdd, 0x51, 0xf4, 0x5d, 0xe2, 0xa7, 0xf2, 0xf0, 0x63, 0xbd, 0x33, 0x72, - 0x00, 0xaa, 0x12, 0xc7, 0xf3, 0x6c, 0x43, 0x6d, 0x27, 0xd1, 0x73, 0x45, 0x42, 0xa4, 0x6e, 0x75, - 0x1d, 0xcf, 0xdb, 0xd6, 0x70, 0x2b, 0x7a, 0xae, 0xc4, 0x5b, 0x30, 0x9f, 0xc8, 0x21, 0xed, 0x93, - 0xe6, 0x56, 0x97, 0x79, 0xf1, 0xf1, 0xf1, 0x6f, 0xa5, 0x9b, 0x5a, 0x72, 0x68, 0x21, 0x4e, 0xac, - 0x40, 0xd5, 0x49, 0xd3, 0x84, 0x79, 0xab, 0x61, 0xf1, 0x8b, 0xd8, 0x84, 0x65, 0x12, 0x56, 0xa9, - 0x1f, 0x85, 0x76, 0xea, 0x1c, 0x07, 0x92, 0x66, 0x82, 0x15, 0xf1, 0x52, 0x86, 0x3a, 0x42, 0x0c, - 0xce, 0xc3, 0x16, 0xac, 0x4e, 0xd3, 0x87, 0xce, 0x58, 0x2a, 0xd2, 0xc3, 0x0d, 0x6b, 0xb9, 0xdc, - 0xe2, 0x11, 0xa2, 0x90, 0x11, 0xf2, 0x36, 0x28, 0xee, 0xea, 0x24, 0x39, 0x5a, 0x19, 0x10, 0xa5, - 0xe1, 0x2a, 0x2c, 0xfa, 0xca, 0x96, 0xa1, 0xa7, 0x25, 0x70, 0xd5, 0x57, 0x0f, 0x42, 0x4f, 0x7c, - 0x00, 0x0d, 0xfe, 0x8a, 0x27, 0x87, 0xa4, 0x47, 0x9b, 0x5b, 0x1d, 0xbd, 0xd5, 0x10, 0x7c, 0x5f, - 0x0e, 0xad, 0x7a, 0xaa, 0x9f, 0xd0, 0xc6, 0x4a, 0x23, 0x5b, 0x9e, 0xa5, 0x32, 0x09, 0x9d, 0x40, - 0x2b, 0x53, 0x48, 0xa3, 0x07, 0x1a, 0x22, 0x3e, 0x85, 0x75, 0x83, 0xb5, 0x55, 0x3a, 0x4e, 0xed, - 0x49, 0xe8, 0x9f, 0xd9, 0xa1, 0x13, 0x46, 0x64, 0x41, 0xcf, 0x5b, 0x2b, 0x06, 0x7d, 0x98, 0x8e, - 0xd3, 0x27, 0xa1, 0x7f, 0xf6, 0xc8, 0x09, 0x23, 0xb1, 0x01, 0xbd, 0xac, 0x59, 0xfa, 0x82, 0x7e, - 0x30, 0x31, 0x57, 0xc3, 0xea, 0x18, 0xf8, 0xd1, 0x0b, 0xfc, 0xad, 0xc4, 0x5b, 0x05, 0xca, 0x68, - 0x38, 0x44, 0xde, 0x52, 0xd2, 0x25, 0x33, 0xb8, 0x6a, 0x2d, 0xe7, 0xf4, 0x8f, 0x09, 0x77, 0x28, - 0xdd, 0xc1, 0x9f, 0x57, 0xa0, 0x49, 0x1b, 0xfa, 0x49, 0xec, 0xa1, 0xec, 0x7c, 0x1b, 0xda, 0xe5, - 0x45, 0x67, 0xbe, 0x69, 0x39, 0xc5, 0x15, 0x5f, 0x83, 0xc5, 0x6d, 0x17, 0x27, 0x8f, 0x18, 0xa7, - 0x6d, 0xe9, 0x37, 0xf1, 0x73, 0xe8, 0x4e, 0xa8, 0x1b, 0xdb, 0x4d, 0xcf, 0xec, 0x00, 0x65, 0x2e, - 0x4b, 0x28, 0xcd, 0x15, 0xfc, 0x8d, 0x9d, 0xf4, 0xcc, 0x6a, 0x4f, 0xcc, 0xe3, 0x43, 0x94, 0xc6, - 0x77, 0x61, 0x25, 0x91, 0xc8, 0x31, 0xf6, 0x0b, 0x99, 0x44, 0x76, 0x2a, 0xc7, 0x71, 0x94, 0x90, - 0x06, 0xc7, 0x59, 0x14, 0x8c, 0xfb, 0x5e, 0x26, 0xd1, 0x91, 0xc6, 0x0c, 0xde, 0x84, 0xea, 0x76, - 0x92, 0x38, 0xe7, 0xc4, 0x5a, 0xf8, 0xd0, 0xaf, 0xd0, 0xde, 0xe4, 0x97, 0x81, 0x0b, 0xf3, 0xfb, - 0x4e, 0x2c, 0x6e, 0xc2, 0xdc, 0x38, 0x26, 0x4c, 0x73, 0x6b, 0xb5, 0x20, 0xd0, 0x9c, 0x78, 0x73, - 0x3f, 0x7e, 0x10, 0xa6, 0xc9, 0xb9, 0x35, 0x37, 0x8e, 0x2f, 0x7f, 0x0a, 0x35, 0xfd, 0x8a, 0x6e, - 0x20, 0x6e, 0xf4, 0x0a, 0xcd, 0x30, 0x3e, 0xe2, 0x07, 0x9e, 0x39, 0xc1, 0xc4, 0x98, 0xae, 0xfc, - 0xf2, 0xc5, 0xdc, 0x67, 0x95, 0xc1, 0xff, 0x5c, 0x80, 0xfa, 0x7d, 0x19, 0x48, 0xfa, 0xed, 0x03, - 0x68, 0x15, 0x77, 0x85, 0x99, 0xb7, 0xd2, 0x4e, 0x19, 0x40, 0x8b, 0x6d, 0x09, 0x6a, 0x25, 0xf5, - 0xb6, 0x2b, 0xc1, 0x50, 0xc9, 0xed, 0xb1, 0x91, 0x46, 0xfb, 0xad, 0x6d, 0x99, 0x57, 0xc4, 0x3c, - 0xd2, 0x98, 0x05, 0xc6, 0xe8, 0x57, 0x71, 0x15, 0x20, 0x89, 0x9e, 0xdb, 0x3e, 0x2b, 0x74, 0xd6, - 0x8d, 0xf5, 0x24, 0x7a, 0xbe, 0x87, 0x2a, 0xfd, 0xaf, 0x65, 0x9b, 0xfd, 0x1c, 0xfa, 0x85, 0x6d, - 0x86, 0xae, 0x82, 0xed, 0x87, 0xec, 0x12, 0xe9, 0x1d, 0x97, 0xf7, 0x49, 0x9e, 0xc4, 0x5e, 0x48, - 0xde, 0x90, 0x11, 0x1e, 0x8d, 0x57, 0x08, 0x8f, 0x99, 0xb2, 0x08, 0x66, 0xcb, 0xa2, 0x7b, 0x00, - 0x87, 0x72, 0x34, 0x96, 0x61, 0xba, 0xef, 0xc4, 0xfd, 0x26, 0x2d, 0xfc, 0x20, 0x5f, 0x78, 0xb3, - 0x5a, 0x9b, 0x39, 0x11, 0x73, 0x41, 0xa1, 0x15, 0xda, 0x79, 0xae, 0x13, 0xda, 0x69, 0x32, 0x09, - 0x5d, 0x27, 0x65, 0xff, 0xb6, 0x6e, 0x35, 0x5d, 0x27, 0x3c, 0xd2, 0xa0, 0x82, 0xc0, 0x68, 0x17, - 0x05, 0xc6, 0xbb, 0xd0, 0x8d, 0x13, 0x7f, 0xec, 0x24, 0xe7, 0xa8, 0x2d, 0x68, 0x31, 0x78, 0xeb, - 0xb5, 0x35, 0xf8, 0x5b, 0x79, 0xbe, 0xe7, 0x9d, 0x5d, 0xfe, 0x12, 0xba, 0x53, 0x03, 0xf8, 0x51, - 0x7c, 0xf7, 0xbb, 0x45, 0x68, 0x1c, 0x24, 0x52, 0x0b, 0xf9, 0xeb, 0xd0, 0x54, 0xee, 0x89, 0x1c, - 0x3b, 0x2c, 0x1b, 0xb8, 0x07, 0x60, 0x10, 0xc9, 0x85, 0x92, 0x18, 0x9b, 0x7b, 0x8d, 0x18, 0xeb, - 0xc1, 0x3c, 0xdb, 0x8b, 0xb8, 0x99, 0xf0, 0x31, 0x97, 0xdd, 0x0b, 0x45, 0xd9, 0x7d, 0x03, 0x5a, - 0x27, 0x8e, 0xb2, 0x9d, 0x49, 0x1a, 0xd9, 0x6e, 0x14, 0x10, 0xd3, 0xd5, 0x2d, 0x38, 0x71, 0xd4, - 0xf6, 0x24, 0x8d, 0x76, 0xa2, 0x40, 0xbc, 0x09, 0xe0, 0x46, 0x81, 0x16, 0x43, 0xda, 0x58, 0x6e, - 0xb8, 0x51, 0xc0, 0xb2, 0x07, 0xb9, 0x52, 0xaa, 0xd4, 0x1f, 0x3b, 0x7a, 0x49, 0xb5, 0xc7, 0x5d, - 0x23, 0x51, 0xb8, 0x94, 0xa1, 0xac, 0xe8, 0x39, 0xbb, 0xda, 0x77, 0xa1, 0xe3, 0x46, 0xe3, 0xd8, - 0x8e, 0x71, 0x66, 0xc9, 0x74, 0xab, 0x5f, 0xf0, 0x86, 0x5a, 0x48, 0x71, 0x70, 0x2a, 0xd9, 0x98, - 0xdc, 0x82, 0xae, 0x1b, 0x4c, 0x54, 0x2a, 0x13, 0xd4, 0xe1, 0x72, 0xb6, 0x03, 0xd5, 0xd6, 0x24, - 0xda, 0x00, 0x1d, 0x40, 0xdb, 0x57, 0x76, 0x14, 0x78, 0x36, 0x0b, 0x28, 0xcd, 0x67, 0x4d, 0x5f, - 0x3d, 0x0e, 0x3c, 0x2d, 0x22, 0x99, 0x26, 0x94, 0xcf, 0x0d, 0x4d, 0xd3, 0xd0, 0x3c, 0x92, 0xcf, - 0x35, 0xcd, 0xcb, 0x04, 0x5a, 0xeb, 0x65, 0x02, 0x0d, 0xe7, 0x03, 0x27, 0x34, 0x75, 0x92, 0x11, - 0x49, 0xed, 0x80, 0xfd, 0x20, 0xe6, 0xaf, 0xa5, 0x13, 0x47, 0x1d, 0x11, 0xe6, 0x50, 0x23, 0xd0, - 0xeb, 0xd1, 0xb4, 0x38, 0x79, 0xe1, 0x64, 0x7c, 0x2c, 0x13, 0x5a, 0x09, 0xe6, 0x38, 0xc1, 0x48, - 0x2b, 0x7a, 0xfe, 0x88, 0x50, 0xb8, 0x22, 0xb7, 0x60, 0x49, 0x37, 0x71, 0xdc, 0xd4, 0x7f, 0x26, - 0x89, 0xbc, 0x4b, 0xe4, 0x5d, 0x46, 0x6c, 0x13, 0x1c, 0x69, 0xdf, 0xcf, 0x68, 0xb5, 0x64, 0x41, - 0xda, 0x1e, 0xc7, 0x04, 0xb2, 0xae, 0xf7, 0x3c, 0x24, 0xdd, 0x81, 0x6b, 0x69, 0xe2, 0xb8, 0xa7, - 0xcc, 0x0c, 0x7e, 0xe8, 0x26, 0x12, 0x59, 0xdb, 0x1e, 0xc9, 0x50, 0x26, 0xb8, 0x86, 0x14, 0xd5, - 0xa8, 0x5b, 0x57, 0x88, 0x0a, 0xd9, 0x63, 0xcf, 0xd0, 0x7c, 0x6d, 0x48, 0xc4, 0x03, 0xb8, 0xfe, - 0xb2, 0xe6, 0xf8, 0xe9, 0xc9, 0x38, 0xa4, 0x80, 0x47, 0xd5, 0xba, 0xea, 0xcc, 0xec, 0x60, 0x87, - 0x68, 0x06, 0xff, 0x7e, 0x0e, 0x6a, 0x07, 0x91, 0x4a, 0xef, 0x8f, 0x03, 0x23, 0x5a, 0x2a, 0x3f, - 0x56, 0xb4, 0xcc, 0xcd, 0x16, 0x2d, 0x33, 0x36, 0xf7, 0xfc, 0x8c, 0xcd, 0x8d, 0x0a, 0xbb, 0x48, - 0x47, 0x9b, 0x92, 0x5d, 0x99, 0x4e, 0x4e, 0x48, 0x1b, 0xf3, 0x0a, 0x9a, 0xcf, 0xb6, 0xc7, 0xba, - 0x80, 0x37, 0x50, 0xdd, 0x57, 0x5a, 0x0f, 0x30, 0xd2, 0xa7, 0x3d, 0xae, 0x6d, 0xeb, 0xba, 0xaf, - 0xf4, 0x9e, 0xff, 0x1c, 0x2e, 0x65, 0x2d, 0xed, 0xe7, 0x7e, 0x7a, 0x12, 0x4d, 0x52, 0x7b, 0x48, - 0x71, 0x03, 0xa5, 0x3d, 0xcf, 0x35, 0xd3, 0xd3, 0x77, 0x8c, 0xe6, 0xa8, 0x02, 0x99, 0xfa, 0xc3, - 0x49, 0x10, 0xd8, 0xa9, 0x3c, 0x4b, 0xf5, 0x16, 0xea, 0xf3, 0xdc, 0xe8, 0x79, 0xdb, 0x9d, 0x04, - 0xc1, 0x91, 0x3c, 0x4b, 0x51, 0x4d, 0xd7, 0x87, 0xfa, 0x65, 0xf0, 0x0f, 0x17, 0x00, 0x1e, 0x46, - 0xee, 0x29, 0x73, 0x21, 0xfa, 0xb3, 0x46, 0x93, 0x68, 0x4d, 0x57, 0x4b, 0x59, 0x7f, 0x88, 0x2d, - 0x58, 0x33, 0xbf, 0x1f, 0xf7, 0x3f, 0xfa, 0xd6, 0xac, 0x0a, 0xb4, 0x20, 0x13, 0x1a, 0xcb, 0xf1, - 0x21, 0xd2, 0x03, 0xe2, 0xb3, 0x7c, 0x6e, 0xb1, 0x4d, 0x7a, 0x1e, 0xd3, 0xdc, 0xce, 0x72, 0x6d, - 0xda, 0x79, 0xf3, 0xa3, 0xf3, 0x58, 0xdc, 0x85, 0xd5, 0x44, 0x0e, 0x13, 0xa9, 0x4e, 0xec, 0x54, - 0x15, 0x3f, 0xc6, 0x6e, 0xed, 0x92, 0x46, 0x1e, 0xa9, 0xec, 0x5b, 0x77, 0x61, 0x95, 0x67, 0x6a, - 0x7a, 0x78, 0xac, 0x37, 0x97, 0x18, 0x59, 0x1c, 0xdd, 0x9b, 0x40, 0x41, 0x6a, 0xd6, 0x85, 0xc6, - 0xcf, 0x09, 0x68, 0x32, 0x8e, 0x03, 0x89, 0x96, 0xf4, 0xce, 0x89, 0x13, 0x8e, 0x50, 0x7e, 0xea, - 0xc9, 0xcf, 0x01, 0x62, 0x00, 0x0b, 0xfb, 0x91, 0x27, 0x69, 0xaa, 0x3b, 0x5b, 0x9d, 0x4d, 0x0a, - 0x77, 0xe3, 0x4c, 0x52, 0x5c, 0x94, 0x70, 0xe2, 0x3d, 0xa0, 0xee, 0x98, 0xfd, 0x2e, 0xca, 0xa8, - 0x3a, 0x22, 0x89, 0x07, 0xef, 0xc2, 0x6a, 0x3e, 0x12, 0xdb, 0x49, 0xed, 0xf4, 0x44, 0x92, 0x1a, - 0x62, 0x31, 0xb5, 0x94, 0x0d, 0x6a, 0x3b, 0x3d, 0x3a, 0x91, 0xa8, 0x92, 0x36, 0xa0, 0x16, 0x1d, - 0xff, 0xd6, 0xc6, 0x8d, 0xd0, 0x9c, 0xbd, 0x11, 0x16, 0xa3, 0xe3, 0xdf, 0x5a, 0x72, 0x28, 0x7e, - 0x56, 0x54, 0xe1, 0x53, 0x53, 0xd3, 0xa2, 0xa9, 0x59, 0xc9, 0xf0, 0x85, 0xd9, 0x19, 0x7c, 0x06, - 0x8b, 0xf8, 0x73, 0x1e, 0xc7, 0x62, 0x13, 0x6a, 0x2c, 0x1a, 0x94, 0x36, 0xb9, 0x56, 0x72, 0xcd, - 0x9b, 0xf3, 0x8e, 0x65, 0x88, 0x06, 0x16, 0x74, 0x33, 0x35, 0xf6, 0x24, 0xf4, 0x9f, 0x4e, 0xa4, - 0xf8, 0x0a, 0x96, 0xe2, 0x44, 0x6a, 0xb6, 0xb7, 0x27, 0xa7, 0x68, 0x48, 0xea, 0x1d, 0xbc, 0xa2, - 0xb9, 0x34, 0x6b, 0x71, 0x8a, 0x1c, 0xda, 0x89, 0x4b, 0xef, 0x83, 0xef, 0x61, 0x3d, 0xa3, 0x38, - 0x94, 0x6e, 0x14, 0x7a, 0x4e, 0x72, 0x4e, 0x16, 0xc7, 0x54, 0xdf, 0xea, 0xc7, 0xf4, 0x7d, 0x48, - 0x7d, 0xff, 0xf7, 0x0a, 0x34, 0x77, 0x27, 0x2f, 0x5e, 0x9c, 0xf3, 0x5e, 0x12, 0x2d, 0xa8, 0x3c, - 0xa2, 0x0e, 0xe6, 0xac, 0xca, 0x23, 0x34, 0x8a, 0x0f, 0x4e, 0x71, 0x5f, 0x13, 0x9f, 0x37, 0x2c, - 0xfd, 0x86, 0xce, 0xfa, 0xc1, 0xe9, 0xd1, 0x2b, 0x38, 0x9a, 0xd1, 0xe8, 0xac, 0xdd, 0x9b, 0xf8, - 0x01, 0x9a, 0x6c, 0x9a, 0x79, 0xb3, 0x77, 0x74, 0x7f, 0xf7, 0x86, 0x3c, 0x94, 0xdd, 0x24, 0x1a, - 0xf3, 0x64, 0x69, 0x91, 0x31, 0x03, 0x23, 0xbe, 0x86, 0x65, 0x1d, 0x4c, 0xd4, 0x52, 0xc1, 0x56, - 0xb1, 0x74, 0x89, 0x75, 0x7f, 0x54, 0x00, 0x72, 0xf0, 0x77, 0x17, 0xa1, 0x8e, 0x6e, 0xee, 0xaf, - 0x22, 0x3f, 0x14, 0x77, 0xa1, 0xf1, 0xdb, 0xc8, 0x0f, 0x39, 0xf2, 0xc1, 0x09, 0x97, 0x65, 0xee, - 0xeb, 0x51, 0xe4, 0xc9, 0x4d, 0xa4, 0xa1, 0x98, 0x47, 0xfd, 0xb7, 0xfa, 0x49, 0xab, 0xca, 0xc4, - 0x1f, 0x9d, 0xa4, 0x36, 0x02, 0xb5, 0x6c, 0x6d, 0xfa, 0xca, 0x42, 0x18, 0xf5, 0x7a, 0x15, 0x80, - 0xfc, 0xeb, 0x28, 0xb4, 0xe3, 0x53, 0xed, 0x63, 0xd6, 0x11, 0xf2, 0x38, 0x3c, 0x38, 0xc5, 0xbd, - 0xe7, 0x2b, 0x5b, 0xc7, 0xd8, 0xb4, 0x3f, 0x50, 0x88, 0x31, 0xbc, 0x03, 0x1d, 0xb4, 0xd5, 0xd4, - 0xa9, 0x1f, 0xdb, 0x71, 0x12, 0x1d, 0x9b, 0x49, 0x41, 0x0b, 0xee, 0xf0, 0xd4, 0x8f, 0x0f, 0x10, - 0x46, 0x26, 0x92, 0x8e, 0xdc, 0xa1, 0xd8, 0x66, 0x5b, 0x04, 0x34, 0x08, 0xe7, 0x97, 0xc2, 0x73, - 0x01, 0x7b, 0x2c, 0x35, 0x32, 0x7d, 0x6a, 0x89, 0x0c, 0xc8, 0x35, 0xb9, 0x04, 0x75, 0xdc, 0x0c, - 0x84, 0xaa, 0x33, 0xca, 0x8d, 0x18, 0xf5, 0x3e, 0x40, 0x20, 0x87, 0xa9, 0x8d, 0x5c, 0xc6, 0xc1, - 0x88, 0xa9, 0x30, 0x18, 0x62, 0x77, 0x10, 0x29, 0x3e, 0x80, 0x26, 0xcf, 0x02, 0xd3, 0xc2, 0x05, - 0x5a, 0x20, 0x34, 0x13, 0xdf, 0x82, 0x66, 0x18, 0x85, 0xb6, 0x7c, 0x4a, 0xd4, 0x7a, 0xdf, 0x96, - 0x3a, 0x0e, 0xa3, 0xf0, 0xc1, 0x53, 0x24, 0x16, 0x77, 0xf4, 0x18, 0x38, 0x1e, 0xd4, 0x7a, 0x49, - 0x3c, 0x88, 0x46, 0xc2, 0x91, 0x91, 0x8f, 0xcc, 0x48, 0xb8, 0x45, 0xfb, 0x25, 0x2d, 0x78, 0x3c, - 0xdc, 0xe4, 0x06, 0xb4, 0x68, 0xdd, 0xc7, 0x4e, 0x6c, 0xa7, 0xce, 0x48, 0x5b, 0x18, 0x80, 0xb0, - 0x7d, 0x27, 0x3e, 0x72, 0x46, 0xc2, 0x82, 0x4b, 0x53, 0xfc, 0x76, 0x8c, 0xac, 0xcb, 0xb3, 0xd6, - 0x35, 0xf1, 0xa4, 0xd9, 0x5c, 0xb7, 0x56, 0xe2, 0x3a, 0x62, 0x79, 0x9a, 0xdd, 0xcf, 0xe1, 0x92, - 0x1c, 0x53, 0x26, 0x66, 0x1c, 0x27, 0x52, 0xa9, 0x92, 0x99, 0xd8, 0x63, 0x1d, 0x87, 0x04, 0x3b, - 0x19, 0x3e, 0xb3, 0x15, 0xdf, 0x81, 0x8e, 0xa3, 0xa2, 0xa1, 0x6d, 0xa6, 0x3c, 0x20, 0x0b, 0xa4, - 0x6a, 0xb5, 0x10, 0x6a, 0xf1, 0x44, 0x07, 0xa8, 0xd0, 0x89, 0x4a, 0x0f, 0x55, 0x0e, 0x53, 0x32, - 0x31, 0xea, 0x56, 0x1b, 0xc1, 0x3c, 0x10, 0x39, 0x4c, 0x07, 0xff, 0x6c, 0x0e, 0xea, 0x0f, 0xa3, - 0x28, 0xfe, 0x89, 0x7b, 0xa0, 0xc8, 0x5b, 0x73, 0x2f, 0xe7, 0xad, 0xf9, 0x32, 0x6f, 0x4d, 0xf1, - 0xc0, 0xc2, 0x0f, 0xe7, 0x81, 0xea, 0x8f, 0xe6, 0x81, 0xc5, 0x9f, 0xc0, 0x03, 0xb5, 0x69, 0x1e, - 0x18, 0xdc, 0x85, 0xea, 0xa1, 0x4c, 0x1f, 0xc7, 0xa8, 0xcd, 0x8c, 0x8d, 0x6e, 0x14, 0x41, 0x49, - 0x9b, 0x69, 0xfb, 0x5c, 0x0d, 0xfe, 0xaa, 0x09, 0x8d, 0xfb, 0xd2, 0x9b, 0xf0, 0xcc, 0x16, 0xe7, - 0xa9, 0xf2, 0xf2, 0x79, 0x9a, 0x2b, 0xcf, 0x13, 0xaa, 0x4e, 0xb3, 0x07, 0x67, 0x04, 0x73, 0xeb, - 0x66, 0x0b, 0xe2, 0x66, 0xcd, 0x77, 0xa0, 0x8e, 0x88, 0x96, 0xe6, 0x33, 0xdb, 0x80, 0xaf, 0xe6, - 0xe6, 0xea, 0x4f, 0xe3, 0xe6, 0xb2, 0x1c, 0xbb, 0x10, 0x2b, 0x7d, 0xed, 0xf4, 0x4e, 0xcb, 0xb0, - 0xfa, 0x05, 0x19, 0xf6, 0x10, 0x96, 0xa3, 0xd0, 0xf6, 0x26, 0x71, 0xe0, 0xa3, 0x8f, 0x4a, 0x36, - 0x7e, 0x14, 0x92, 0x39, 0x41, 0xd9, 0xd9, 0x8c, 0x47, 0x1f, 0x87, 0xf7, 0x0d, 0x11, 0xc7, 0x6d, - 0xac, 0xa5, 0x68, 0x1a, 0x84, 0x5b, 0xc8, 0xc3, 0xa5, 0x21, 0x4b, 0x80, 0x6c, 0x58, 0x4e, 0x33, - 0xb7, 0x08, 0xba, 0x13, 0x05, 0xa4, 0xdb, 0x3e, 0x83, 0x6e, 0x4e, 0xc5, 0xcc, 0xd4, 0x7c, 0x09, - 0x33, 0xb5, 0x4d, 0x43, 0xe6, 0xa7, 0xbf, 0x0e, 0xb9, 0xf5, 0x21, 0x2c, 0x9b, 0x70, 0x94, 0x36, - 0x67, 0x68, 0x05, 0x3b, 0xc4, 0x41, 0x3d, 0x1d, 0x81, 0x22, 0x4b, 0x86, 0x96, 0xe8, 0x17, 0xb0, - 0x52, 0x20, 0x47, 0xf6, 0x2d, 0xca, 0xaf, 0x22, 0xaf, 0x2c, 0x65, 0x6d, 0xf1, 0xf5, 0x21, 0xe7, - 0x13, 0x9a, 0x9e, 0x0c, 0xcc, 0x87, 0xb4, 0xa7, 0xd4, 0xf0, 0x64, 0xa0, 0xd3, 0xa0, 0xfb, 0xf0, - 0x0e, 0x7a, 0x95, 0x88, 0x77, 0x9d, 0x38, 0x9d, 0x24, 0xd2, 0x8e, 0x03, 0xc7, 0x95, 0x27, 0x51, - 0xe0, 0xc9, 0x24, 0x1f, 0xdc, 0x12, 0x0d, 0xee, 0x7a, 0x14, 0xa0, 0x47, 0xb3, 0xc3, 0x94, 0x07, - 0x39, 0xa1, 0x19, 0xeb, 0x36, 0x5c, 0xbb, 0xd0, 0x1d, 0xaa, 0xba, 0xbc, 0x23, 0x41, 0x1d, 0x5d, - 0x2a, 0x77, 0x84, 0x24, 0xa6, 0x8b, 0x8f, 0x60, 0x95, 0xd7, 0x8e, 0x99, 0xfb, 0x54, 0xca, 0xd8, - 0x0e, 0x1c, 0x95, 0xf6, 0x97, 0xd9, 0xac, 0x20, 0x24, 0x31, 0xf0, 0xb7, 0x52, 0xc6, 0x0f, 0x1d, - 0xfe, 0x2a, 0x37, 0xd1, 0x9e, 0x07, 0xb5, 0x29, 0xcd, 0xed, 0x0a, 0x7f, 0x95, 0xa8, 0xd8, 0xfd, - 0xc0, 0xc6, 0x85, 0x49, 0xfe, 0x13, 0xb8, 0x52, 0xea, 0x62, 0xec, 0x24, 0xa7, 0xb9, 0x29, 0xde, - 0x5f, 0xa5, 0x79, 0x5b, 0x2f, 0xb4, 0xdf, 0x27, 0x02, 0x3d, 0x8b, 0x1f, 0xc3, 0x1a, 0x3a, 0xc9, - 0x91, 0x77, 0x3a, 0x99, 0x72, 0xda, 0xd6, 0x68, 0xd0, 0xe8, 0x42, 0x3f, 0xf6, 0x4e, 0x27, 0x25, - 0xc7, 0xed, 0xe7, 0xd0, 0x2f, 0xd1, 0xda, 0x09, 0x25, 0x0a, 0xec, 0x38, 0x52, 0xfd, 0x75, 0x8e, - 0x4d, 0x15, 0xa3, 0x9b, 0x9c, 0x46, 0x38, 0x88, 0x94, 0xd8, 0x85, 0x1b, 0xf1, 0xc9, 0xb9, 0xf2, - 0x29, 0xb1, 0x49, 0x06, 0xfd, 0xc5, 0x0e, 0xfa, 0xec, 0x94, 0x1a, 0x3a, 0xb6, 0xfb, 0xa7, 0xfa, - 0xf9, 0x0c, 0x2e, 0x19, 0xc6, 0x3a, 0x91, 0xee, 0x69, 0x79, 0xc6, 0x2e, 0xd1, 0x8c, 0xad, 0x6a, - 0x8e, 0x42, 0x7c, 0x61, 0xb6, 0x36, 0xa0, 0x47, 0xfa, 0xce, 0x1e, 0x46, 0x93, 0x50, 0xff, 0xd2, - 0xcb, 0xf4, 0x4b, 0x3b, 0x04, 0xdf, 0x45, 0x30, 0xfd, 0xc8, 0x0d, 0xe8, 0x91, 0xb6, 0xe4, 0x5d, - 0xcf, 0x94, 0x57, 0x98, 0x12, 0xe1, 0x7a, 0xa3, 0x23, 0xe5, 0xa7, 0xb0, 0xae, 0x89, 0x86, 0x7e, - 0xe8, 0x04, 0xc5, 0x1f, 0x73, 0x95, 0xcd, 0x7c, 0x46, 0xef, 0x22, 0x36, 0xff, 0x11, 0xbf, 0x06, - 0x31, 0x8c, 0x12, 0xe9, 0x8f, 0x42, 0x72, 0x6b, 0xe9, 0x97, 0xa8, 0xfe, 0x9b, 0xb4, 0x37, 0xde, - 0xcc, 0xed, 0xfc, 0xc7, 0xf7, 0xbf, 0x7d, 0xb2, 0xcb, 0x74, 0xdf, 0xca, 0x73, 0xfa, 0x3d, 0x7a, - 0x67, 0xf6, 0x86, 0x65, 0xb0, 0x1a, 0xfc, 0xa3, 0x45, 0xe8, 0x90, 0x36, 0xfe, 0x1b, 0x25, 0xf0, - 0x37, 0x4a, 0xe0, 0xff, 0x07, 0x25, 0x70, 0x0b, 0x96, 0xfc, 0x30, 0x9e, 0xa4, 0xb8, 0x83, 0x94, - 0x3d, 0x61, 0x2f, 0x8e, 0x83, 0x63, 0x5d, 0x42, 0x7c, 0x2b, 0xcf, 0x15, 0xbb, 0x70, 0x83, 0xbf, - 0x53, 0x81, 0xda, 0x41, 0x12, 0x79, 0x13, 0x37, 0xfd, 0x89, 0xbb, 0xa2, 0xcc, 0x6d, 0xf3, 0xaf, - 0xe3, 0xb6, 0x85, 0x0b, 0x16, 0xdd, 0xbf, 0xa8, 0x40, 0x43, 0x0f, 0xe1, 0xe1, 0xd6, 0x4f, 0x1c, - 0x44, 0x9e, 0x67, 0xaf, 0xcc, 0xcc, 0xb3, 0xbf, 0x76, 0x14, 0xc8, 0x84, 0xcf, 0xb8, 0x30, 0x29, - 0x8a, 0xf3, 0xa4, 0x7b, 0xc3, 0x6a, 0x31, 0xf4, 0x71, 0x4c, 0xb9, 0xf5, 0xe7, 0xd0, 0x20, 0x2f, - 0x9f, 0xa4, 0xc8, 0x1a, 0x2c, 0xb2, 0x54, 0xd3, 0x03, 0xd5, 0x6f, 0xaf, 0xde, 0xd3, 0x73, 0x3f, - 0x69, 0x4f, 0x0f, 0xfe, 0xed, 0x3c, 0xb4, 0x29, 0xe4, 0xb2, 0x3b, 0x09, 0x79, 0xd7, 0x64, 0x01, - 0xf3, 0x4a, 0x39, 0x60, 0xbe, 0x90, 0xc8, 0xd4, 0x24, 0xf5, 0x5b, 0xfc, 0x99, 0x9d, 0x28, 0xb8, - 0x2f, 0x87, 0x16, 0x61, 0x70, 0xaa, 0x9c, 0x64, 0xa4, 0x66, 0x95, 0x24, 0x20, 0x1c, 0x7f, 0x55, - 0xec, 0x24, 0xce, 0x58, 0x99, 0x92, 0x04, 0x7e, 0x13, 0x02, 0x16, 0x68, 0x6f, 0xf2, 0xb4, 0xd0, - 0xb3, 0x8e, 0x1e, 0x2a, 0x3f, 0x1c, 0x65, 0x82, 0xa6, 0x4e, 0xa5, 0x28, 0xa3, 0x40, 0x8a, 0xfb, - 0x20, 0x38, 0x45, 0x93, 0x48, 0x07, 0x8d, 0x0f, 0xea, 0x87, 0xa4, 0x4d, 0x73, 0x6b, 0x8d, 0x3f, - 0x4b, 0x73, 0x69, 0x11, 0xfa, 0x00, 0xb1, 0x56, 0xcf, 0x9f, 0x82, 0xcc, 0x98, 0x4c, 0xb6, 0x40, - 0x32, 0x4f, 0xf9, 0x07, 0x4f, 0x26, 0x99, 0x25, 0xc4, 0x2d, 0x5f, 0xc1, 0xf2, 0x70, 0x12, 0x04, - 0xa9, 0x3c, 0x4b, 0x6d, 0x15, 0x4d, 0x12, 0x57, 0xda, 0xaf, 0xc8, 0x0e, 0x2d, 0x19, 0xda, 0x43, - 0x22, 0xb5, 0xe4, 0x50, 0x7c, 0x09, 0x22, 0xeb, 0xc0, 0xfc, 0x46, 0x93, 0xbb, 0xbd, 0xd0, 0xbe, - 0x67, 0x48, 0xf5, 0xaf, 0x1d, 0x0e, 0xb6, 0x61, 0xd5, 0xe4, 0x6b, 0x51, 0xb4, 0x6d, 0xe1, 0xbe, - 0xa5, 0xd8, 0x91, 0x99, 0xe3, 0x4a, 0x61, 0x8e, 0x57, 0xa0, 0x5a, 0x2c, 0x95, 0xe3, 0x97, 0xc1, - 0x4d, 0x68, 0x0e, 0xfd, 0x40, 0xea, 0xbc, 0x07, 0x2e, 0x9a, 0xce, 0x80, 0x54, 0x28, 0xc1, 0xaf, - 0xdf, 0x06, 0x7f, 0x5e, 0x81, 0xf5, 0xd8, 0x49, 0x9e, 0x4e, 0x74, 0x84, 0x9d, 0x6b, 0x0d, 0xd4, - 0x89, 0x93, 0x78, 0xb8, 0x71, 0xa9, 0x0b, 0xee, 0x9d, 0x2b, 0xad, 0x1a, 0x08, 0xe1, 0xb1, 0xbc, - 0x0b, 0xdd, 0x42, 0x8b, 0xd4, 0x49, 0x4c, 0x64, 0xb5, 0x9d, 0x44, 0xcf, 0xa9, 0x12, 0xe0, 0x10, - 0x81, 0x62, 0x00, 0xed, 0x9c, 0x4e, 0x92, 0x66, 0xa4, 0x82, 0x25, 0x43, 0xf5, 0x20, 0xf4, 0x70, - 0xe7, 0x86, 0x93, 0x31, 0x9b, 0x0b, 0x5c, 0x50, 0x57, 0x0b, 0x27, 0x63, 0xb2, 0x13, 0x56, 0xa0, - 0xca, 0x55, 0x8c, 0x55, 0x82, 0xf3, 0xcb, 0xe0, 0x2f, 0xab, 0xb0, 0xbc, 0xe7, 0xca, 0x63, 0x99, - 0x8c, 0xee, 0x3b, 0xa9, 0xb3, 0xeb, 0x07, 0xf2, 0xc8, 0x51, 0xa7, 0xc8, 0x70, 0x34, 0xe6, 0xd8, - 0x49, 0x4f, 0xf4, 0x2c, 0xd5, 0x11, 0x70, 0xe0, 0xa4, 0x27, 0xa8, 0xb6, 0x08, 0x39, 0x8c, 0x92, - 0xb1, 0x8e, 0x03, 0x37, 0x2c, 0xfa, 0x8d, 0xbb, 0x04, 0xc9, 0x5a, 0x2b, 0xff, 0x85, 0xd4, 0xe5, - 0x7f, 0xd4, 0x9a, 0x6a, 0x44, 0xde, 0x82, 0x56, 0x22, 0xdd, 0x28, 0xf1, 0xb4, 0xef, 0xcf, 0xe3, - 0x6c, 0x32, 0x8c, 0x1d, 0xfe, 0x5b, 0x90, 0xe7, 0x31, 0x29, 0xd4, 0x65, 0xfb, 0xa6, 0x46, 0xa8, - 0x9b, 0x21, 0x90, 0xf3, 0xf6, 0x3c, 0xf1, 0xb7, 0xa0, 0x97, 0xd3, 0x52, 0x52, 0xcd, 0x78, 0xc0, - 0x5b, 0xb9, 0x19, 0x33, 0xe3, 0x27, 0x6e, 0x1e, 0x98, 0x56, 0xbf, 0xa1, 0x46, 0x9c, 0x38, 0xcc, - 0xbb, 0x67, 0xa8, 0x78, 0x1b, 0xda, 0x2a, 0x0e, 0xfc, 0x54, 0x33, 0x80, 0xd2, 0x45, 0x82, 0x2d, - 0x02, 0x72, 0xee, 0x4b, 0xcd, 0x5a, 0xc2, 0xfa, 0x0f, 0x5a, 0xc2, 0xc6, 0xc5, 0x25, 0x7c, 0x1f, - 0x7a, 0x6e, 0x22, 0x3d, 0x19, 0xa6, 0xbe, 0x13, 0xd8, 0xca, 0x8d, 0x62, 0xa3, 0xa6, 0xbb, 0x39, - 0xfc, 0x10, 0xc1, 0xe2, 0x67, 0xb0, 0xee, 0x46, 0x61, 0x2a, 0xc3, 0x34, 0x2b, 0x23, 0xd5, 0x99, - 0x23, 0x5d, 0xfe, 0xb2, 0xaa, 0xd1, 0xa6, 0x98, 0x94, 0x73, 0x47, 0xe2, 0x2e, 0xac, 0xf0, 0xf2, - 0x4c, 0x35, 0xe2, 0xba, 0x05, 0x41, 0x2b, 0x55, 0x6e, 0xa1, 0xb3, 0x59, 0x89, 0x54, 0xbe, 0x37, - 0x71, 0x02, 0x2d, 0x21, 0x0a, 0xd9, 0x2c, 0x4b, 0x63, 0x74, 0x28, 0x95, 0xf2, 0x65, 0x25, 0x5a, - 0x2a, 0xba, 0xa1, 0x50, 0x53, 0xc3, 0x12, 0x49, 0x89, 0xfa, 0x1b, 0x47, 0x9d, 0x5c, 0xbe, 0x07, - 0x2b, 0xb3, 0x16, 0xe4, 0x75, 0x89, 0xd4, 0x46, 0x21, 0x91, 0xaa, 0x4b, 0x65, 0xff, 0xc7, 0x1c, - 0xac, 0x9a, 0xf5, 0x26, 0x97, 0x23, 0x63, 0xea, 0xeb, 0xa4, 0xcf, 0xd1, 0x4d, 0xc9, 0xc2, 0x3d, - 0x0d, 0x0b, 0x18, 0x44, 0xb1, 0x9d, 0x0d, 0xe8, 0x69, 0x82, 0x9c, 0xf9, 0xf9, 0x2b, 0x1d, 0x2f, - 0xeb, 0x8a, 0xb6, 0x00, 0xfd, 0xc0, 0xa1, 0x4c, 0x70, 0x8e, 0x3c, 0xaa, 0x1c, 0xa7, 0x26, 0xc4, - 0xec, 0xf4, 0x03, 0x0d, 0xce, 0xb0, 0x1c, 0x55, 0xfd, 0x3c, 0x9d, 0x38, 0x81, 0x9f, 0x9e, 0xdb, - 0x43, 0x5f, 0x06, 0x1e, 0x65, 0xed, 0xb9, 0xa6, 0xb1, 0x67, 0x30, 0xbb, 0x88, 0xd8, 0xf3, 0x54, - 0x61, 0x24, 0x3a, 0x19, 0x9c, 0x6d, 0x00, 0x3d, 0x92, 0x43, 0x02, 0xef, 0x79, 0xb3, 0xf7, 0xca, - 0xe2, 0xec, 0xbd, 0xf2, 0x1e, 0x74, 0xa7, 0xd7, 0x9c, 0x13, 0xb4, 0x1d, 0x55, 0x5e, 0xef, 0x59, - 0x4c, 0x58, 0x9f, 0xc9, 0x84, 0x7a, 0xd2, 0xff, 0xd7, 0x1c, 0xac, 0xe8, 0x49, 0xe7, 0xd4, 0xdd, - 0xbe, 0x13, 0x53, 0x85, 0xd4, 0x0d, 0x68, 0x8d, 0x23, 0x36, 0xa1, 0x0a, 0xe2, 0x0f, 0xc6, 0x51, - 0x26, 0x8b, 0x37, 0xa0, 0xe7, 0x73, 0xcb, 0x6c, 0x5e, 0x4c, 0xb5, 0xb2, 0x86, 0xeb, 0x59, 0x41, - 0x2e, 0x54, 0xa1, 0x13, 0xab, 0x93, 0x28, 0xd5, 0xa4, 0x24, 0xc4, 0x79, 0xce, 0x97, 0x0c, 0x8a, - 0xa8, 0xc9, 0x92, 0xbd, 0x0d, 0xc2, 0x9d, 0x24, 0x09, 0xee, 0x8f, 0x02, 0x39, 0x27, 0xef, 0x7a, - 0x1a, 0x93, 0x53, 0xbf, 0x0d, 0xb5, 0x71, 0x94, 0x5b, 0x24, 0x25, 0x43, 0xd4, 0x5a, 0x1c, 0x47, - 0xc4, 0x21, 0x97, 0xd1, 0x6a, 0x7a, 0x3a, 0xf1, 0x13, 0xe9, 0x19, 0x3d, 0x6c, 0xde, 0xb5, 0x92, - 0x3e, 0xf1, 0x3d, 0x4f, 0x86, 0x3a, 0x71, 0x54, 0xf7, 0xd5, 0x37, 0xf4, 0x4e, 0xc5, 0xbc, 0x72, - 0xe8, 0xa0, 0x6b, 0x16, 0x4e, 0x02, 0xda, 0x15, 0x81, 0x2e, 0x1d, 0xed, 0x6a, 0xc4, 0xa3, 0x49, - 0x80, 0x3b, 0x22, 0xd0, 0x4b, 0x4a, 0xba, 0x04, 0x59, 0xd0, 0x3e, 0xf1, 0xc3, 0x94, 0x44, 0x45, - 0x83, 0x96, 0x14, 0x11, 0xc8, 0x84, 0xdf, 0xf8, 0x61, 0x3a, 0xf8, 0xe7, 0x73, 0xb0, 0xa6, 0x27, - 0xfe, 0x50, 0x4f, 0x80, 0xd6, 0xcf, 0xe4, 0x5d, 0x98, 0xe9, 0xd2, 0x79, 0xbd, 0x79, 0x0b, 0x0c, - 0x68, 0x8f, 0x06, 0x9c, 0x73, 0xd7, 0x9c, 0x2e, 0x29, 0x35, 0x7c, 0x75, 0x1b, 0xc4, 0x05, 0xbe, - 0x52, 0x3a, 0xac, 0xd9, 0x9b, 0x62, 0x2c, 0x25, 0x3e, 0x81, 0xb5, 0xb1, 0x4c, 0x1d, 0xda, 0x08, - 0x41, 0xe4, 0x3a, 0xd4, 0x8a, 0xb6, 0x3c, 0x4f, 0xf7, 0x8a, 0xc1, 0x3e, 0xd4, 0x48, 0xdc, 0xf4, - 0xf8, 0x8d, 0xb1, 0x13, 0xfa, 0x43, 0xa9, 0x52, 0xb2, 0x33, 0xb8, 0x05, 0x1b, 0x3e, 0x3d, 0x83, - 0x41, 0x4b, 0x82, 0xa8, 0xc9, 0x62, 0x1d, 0xf2, 0x22, 0x2e, 0x12, 0x4d, 0x2d, 0x91, 0x43, 0xbd, - 0x76, 0x6d, 0x5c, 0xab, 0xd0, 0x0f, 0x47, 0x7c, 0x88, 0xa1, 0xc6, 0x36, 0xa5, 0x01, 0xee, 0x47, - 0x9e, 0x1c, 0xfc, 0x6e, 0x21, 0xe3, 0xd1, 0x03, 0x0d, 0x3f, 0x4c, 0x9d, 0x94, 0xea, 0xf6, 0xb3, - 0xc1, 0xb3, 0x8e, 0xe4, 0xb9, 0x6a, 0x1b, 0x28, 0x97, 0xf7, 0x6f, 0xc2, 0x72, 0x79, 0xb4, 0x4c, - 0x3b, 0xc7, 0x25, 0x0e, 0xc5, 0xe1, 0x66, 0xc7, 0x01, 0x32, 0x7a, 0x26, 0xd5, 0x95, 0xf0, 0x06, - 0xca, 0x64, 0x1f, 0xe6, 0x93, 0xa0, 0x74, 0xa1, 0x80, 0xf4, 0xb4, 0x56, 0xcc, 0x7a, 0x55, 0x87, - 0x1a, 0x81, 0x5b, 0x33, 0x27, 0x8f, 0x93, 0x49, 0x28, 0x3d, 0xad, 0xd2, 0xbb, 0x19, 0xfc, 0x80, - 0xc0, 0x38, 0xe0, 0x4c, 0x32, 0x15, 0xba, 0x5e, 0xe4, 0xae, 0x3d, 0x2d, 0x99, 0xf2, 0xae, 0x91, - 0x47, 0x73, 0x7a, 0xdd, 0x37, 0x0b, 0x88, 0x6e, 0x46, 0xad, 0xfb, 0xfe, 0x39, 0xf4, 0x33, 0x5a, - 0xfe, 0x75, 0xf9, 0x07, 0xea, 0xac, 0x7c, 0x4c, 0x13, 0xfa, 0x99, 0xd9, 0x47, 0x3e, 0x86, 0xb5, - 0xe9, 0x86, 0xfa, 0x4b, 0x0d, 0x6a, 0xb6, 0x5c, 0x6a, 0x96, 0xff, 0x92, 0x6c, 0x7d, 0x5d, 0xc7, - 0x3d, 0x91, 0xf6, 0x89, 0xaf, 0x8b, 0xe1, 0xe7, 0xad, 0x25, 0x83, 0xda, 0x41, 0xcc, 0x37, 0x7e, - 0xaa, 0x66, 0xd0, 0x8f, 0x7d, 0xa5, 0xb4, 0x56, 0x2c, 0xd3, 0xef, 0xfb, 0x4a, 0x0d, 0xfe, 0x4b, - 0x17, 0x5a, 0xc6, 0x52, 0xa4, 0x3a, 0xec, 0xdb, 0x45, 0xa3, 0xbf, 0xb9, 0xd5, 0x33, 0xd6, 0x3b, - 0x92, 0x6c, 0xa7, 0x69, 0x62, 0x72, 0x7d, 0xec, 0x0c, 0x94, 0xec, 0x9d, 0x39, 0x32, 0x10, 0x72, - 0x7b, 0x67, 0x1b, 0x96, 0x0a, 0x16, 0xa4, 0x9d, 0x46, 0xa9, 0x13, 0x68, 0xa7, 0xa0, 0x50, 0xc3, - 0x56, 0x20, 0xb1, 0xba, 0xf8, 0xc2, 0xb6, 0xc5, 0x11, 0x52, 0xa3, 0xb3, 0xe1, 0x46, 0x81, 0x29, - 0xfc, 0x9d, 0x72, 0x36, 0x10, 0x43, 0xd5, 0x39, 0x89, 0x44, 0x3f, 0x57, 0x3d, 0x0d, 0xf4, 0x0e, - 0x6a, 0x30, 0xe4, 0xf0, 0x69, 0x90, 0x0d, 0x90, 0x8c, 0xf9, 0x45, 0xf2, 0x63, 0x68, 0x80, 0x64, - 0xa5, 0x7f, 0x08, 0xcd, 0x28, 0xf1, 0x47, 0x3e, 0xa5, 0x89, 0xd9, 0xc0, 0x99, 0xfe, 0x08, 0x30, - 0xc1, 0x0e, 0x7e, 0x6a, 0x00, 0x8b, 0x5a, 0xfd, 0x5f, 0xac, 0xd8, 0xd1, 0x18, 0x73, 0x1c, 0xc1, - 0x4d, 0x71, 0x38, 0xbc, 0x23, 0x1b, 0xf9, 0x71, 0x04, 0x37, 0x3d, 0x7c, 0x1a, 0x50, 0xa6, 0xfc, - 0x5d, 0xe8, 0x72, 0x35, 0x08, 0x6f, 0xa8, 0x40, 0x86, 0xb4, 0xa6, 0x55, 0xab, 0xcd, 0x60, 0x1c, - 0xdf, 0x43, 0x19, 0xea, 0xb2, 0x4f, 0x27, 0x08, 0xd0, 0x63, 0x8d, 0x1c, 0x4f, 0xd7, 0xe8, 0xb4, - 0x0c, 0xf0, 0x61, 0xe4, 0x78, 0xe2, 0x0b, 0xb8, 0x8c, 0x38, 0x9b, 0x8b, 0x6b, 0xc3, 0xc9, 0x58, - 0x26, 0xbe, 0x6b, 0x3b, 0x8a, 0x6a, 0x76, 0x74, 0xa9, 0xce, 0x1a, 0x52, 0x3c, 0x40, 0x82, 0x47, - 0x8c, 0xdf, 0x56, 0xdf, 0xcb, 0x24, 0x12, 0xdf, 0x53, 0xb6, 0x7c, 0x96, 0xf9, 0x6e, 0xc2, 0x12, - 0x6f, 0xe5, 0x6b, 0xf5, 0x12, 0x4a, 0xaa, 0x89, 0x43, 0x84, 0x65, 0x8c, 0x3e, 0x6a, 0x2f, 0xbe, - 0x05, 0x61, 0x14, 0x1c, 0x71, 0x7e, 0xea, 0xa8, 0x53, 0x2e, 0x10, 0x2e, 0x85, 0xda, 0x66, 0xd8, - 0xa8, 0x96, 0xd1, 0x8c, 0x08, 0x44, 0x80, 0x12, 0xbf, 0x86, 0x95, 0xac, 0x33, 0x6d, 0xcb, 0x50, - 0x77, 0x1c, 0xd0, 0xb8, 0x7e, 0xb1, 0xbb, 0x92, 0x09, 0x64, 0x99, 0x91, 0x30, 0x98, 0xbb, 0xfc, - 0x1a, 0xba, 0xa6, 0x4b, 0x9e, 0x75, 0xd5, 0xef, 0x51, 0x6f, 0xd7, 0x2e, 0xf4, 0x56, 0xd2, 0xed, - 0x99, 0x7e, 0x66, 0x28, 0xfe, 0xd0, 0x4c, 0x93, 0x1b, 0x2d, 0x43, 0xe1, 0x90, 0xe6, 0xd6, 0x8d, - 0x0b, 0x3d, 0x4d, 0x29, 0x2b, 0xcb, 0x0c, 0xc1, 0xc0, 0xc5, 0x47, 0xb0, 0x6a, 0x3a, 0x8b, 0xc8, - 0xc5, 0xb3, 0xfd, 0x88, 0xbc, 0x3f, 0xc1, 0x26, 0x96, 0x46, 0xb2, 0xfb, 0xb7, 0x17, 0xb1, 0xb7, - 0x78, 0xc5, 0x34, 0x61, 0x2d, 0x4c, 0x1e, 0x71, 0xf6, 0xa3, 0x96, 0x49, 0x77, 0xf5, 0x35, 0x09, - 0xeb, 0x65, 0xf4, 0x80, 0xcd, 0xf0, 0x37, 0xa0, 0x47, 0x55, 0xfc, 0xb8, 0xac, 0x51, 0xe2, 0xf9, - 0xa1, 0x13, 0xf4, 0x57, 0x38, 0xe6, 0x8a, 0x70, 0x2b, 0x7a, 0xfe, 0x98, 0xa1, 0xe2, 0x08, 0xd6, - 0xcc, 0x87, 0x32, 0x31, 0xa3, 0x50, 0x95, 0x50, 0xc0, 0x7b, 0xd6, 0xc4, 0x95, 0x14, 0x8e, 0x65, - 0x96, 0xb0, 0xac, 0x86, 0xee, 0xc3, 0xf5, 0xa9, 0xa5, 0x1d, 0x3b, 0x67, 0xf6, 0x58, 0x8e, 0xa3, - 0xe4, 0x5c, 0x2b, 0x90, 0x35, 0x12, 0x60, 0x57, 0x4a, 0x8b, 0xb8, 0xef, 0x9c, 0xed, 0x13, 0x0d, - 0xab, 0x93, 0xaf, 0xe0, 0xea, 0x54, 0x2f, 0x5c, 0x14, 0x2f, 0x43, 0xe7, 0x38, 0x90, 0x1e, 0x85, - 0xc8, 0xeb, 0xd6, 0xa5, 0x52, 0x17, 0x87, 0x48, 0xf1, 0x80, 0x09, 0xc4, 0x97, 0x40, 0xc2, 0x5e, - 0xd1, 0xa9, 0x3f, 0x5b, 0xb9, 0x4e, 0x48, 0x51, 0xf1, 0xac, 0xaa, 0x02, 0x79, 0x91, 0x8f, 0x04, - 0xa2, 0xa4, 0xb4, 0x3a, 0x39, 0x31, 0x49, 0xce, 0x4f, 0xa0, 0x65, 0x02, 0xcb, 0xd4, 0xf6, 0x12, - 0xb5, 0x5d, 0xe2, 0xb6, 0x3a, 0x94, 0x4c, 0x0d, 0x9b, 0xc3, 0xfc, 0x45, 0x6c, 0x02, 0x9c, 0x3a, - 0xc3, 0x53, 0x87, 0xdb, 0x5c, 0x2e, 0x3a, 0xf8, 0xdf, 0x22, 0x9c, 0x5a, 0x34, 0x4e, 0xcd, 0xa3, - 0xf8, 0x1c, 0x2e, 0x99, 0x5d, 0xf8, 0xfc, 0x24, 0x0a, 0xb4, 0xc1, 0x3e, 0x74, 0xc2, 0x68, 0x92, - 0xea, 0x40, 0xf9, 0x9a, 0x26, 0xf8, 0x0e, 0xf1, 0xb8, 0x01, 0x76, 0x09, 0x2b, 0x7e, 0x0d, 0xab, - 0x4e, 0x82, 0x6b, 0x2c, 0xcf, 0xa4, 0x3b, 0x61, 0xf3, 0x86, 0x0c, 0xdc, 0xab, 0x14, 0x55, 0x2d, - 0xec, 0xc8, 0x6d, 0x24, 0x7b, 0x60, 0xa8, 0xc8, 0xdc, 0xb5, 0x96, 0x9d, 0x8b, 0x40, 0xf1, 0x04, - 0xd6, 0xb9, 0x4b, 0xc3, 0xa9, 0x64, 0x21, 0xa7, 0xbe, 0x9c, 0x11, 0x51, 0xa7, 0x4e, 0x35, 0xd7, - 0x32, 0xd9, 0xb9, 0xc5, 0x03, 0x2a, 0x01, 0x7d, 0xa9, 0xc4, 0xf7, 0x70, 0x89, 0xbb, 0xd5, 0xfe, - 0xb2, 0x3e, 0x5c, 0xc8, 0x52, 0xe9, 0xda, 0xf4, 0x86, 0xa7, 0x8e, 0x2d, 0xa2, 0xa4, 0x5a, 0x5c, - 0x92, 0x3e, 0xd6, 0x9a, 0x33, 0x0b, 0xac, 0xc4, 0x67, 0xd0, 0xe7, 0xbe, 0xb5, 0x05, 0x38, 0xf4, - 0xc3, 0x91, 0x4c, 0xe2, 0x04, 0x4d, 0xcd, 0xeb, 0x14, 0x8f, 0xe2, 0x96, 0xec, 0x67, 0xec, 0xe6, - 0x58, 0xb1, 0x0d, 0x6f, 0x72, 0x4b, 0x37, 0x0a, 0x9f, 0xc9, 0x44, 0xe1, 0x04, 0xe2, 0x42, 0xd9, - 0xfa, 0xa5, 0x7f, 0x83, 0xaa, 0x99, 0x2f, 0x13, 0xd1, 0x4e, 0x46, 0x83, 0xbc, 0xfe, 0x1b, 0x7e, - 0x44, 0xc7, 0x95, 0xbb, 0x18, 0x46, 0x89, 0x8b, 0x6c, 0x9e, 0xca, 0xc4, 0x77, 0x02, 0xd4, 0x9e, - 0x6f, 0xd1, 0xda, 0xf1, 0x84, 0xec, 0x22, 0x76, 0x3f, 0x47, 0x8a, 0x5f, 0xc2, 0x15, 0x6e, 0xe7, - 0xf9, 0xa8, 0x47, 0x8e, 0x27, 0xa9, 0xf4, 0xf2, 0x65, 0xec, 0x0f, 0x98, 0xb5, 0x89, 0xe4, 0x7e, - 0x4e, 0x91, 0x2d, 0x96, 0xf6, 0x55, 0x8e, 0xa1, 0x41, 0x11, 0x3e, 0x62, 0xa4, 0xec, 0xec, 0x4d, - 0xe5, 0xd5, 0x67, 0x6f, 0x3e, 0x84, 0x96, 0x76, 0x64, 0x5f, 0x76, 0x98, 0xa7, 0xc9, 0x78, 0xce, - 0x85, 0xdf, 0x86, 0x06, 0x79, 0xb1, 0xf4, 0x8d, 0xeb, 0xd0, 0xe4, 0xa5, 0x3b, 0x0e, 0x22, 0xf7, - 0xd4, 0xf8, 0x9d, 0x04, 0xba, 0x87, 0x90, 0x01, 0x40, 0xfd, 0x49, 0xe8, 0x47, 0xe1, 0x76, 0x10, - 0x0c, 0xfe, 0xb8, 0x08, 0x0d, 0x34, 0x77, 0x29, 0x24, 0x29, 0x06, 0xd0, 0x26, 0x99, 0x44, 0x25, - 0x35, 0x63, 0x27, 0xd6, 0xa7, 0x8b, 0x9a, 0x08, 0x44, 0xaa, 0x7d, 0x27, 0x9e, 0xaa, 0xb8, 0x99, - 0x9b, 0xaa, 0xb8, 0x79, 0x8b, 0x8f, 0x1f, 0x33, 0xf3, 0x48, 0x73, 0xea, 0x83, 0x3a, 0xb8, 0xc7, - 0x20, 0x34, 0xc3, 0x89, 0xc4, 0x09, 0xc8, 0x74, 0x97, 0x68, 0xc9, 0x29, 0x5d, 0x9c, 0x43, 0x22, - 0x71, 0x5b, 0x23, 0x0e, 0x25, 0x9b, 0x1a, 0x85, 0x38, 0x74, 0x75, 0x3a, 0x0e, 0x7d, 0x0b, 0xc0, - 0x8d, 0x42, 0x8f, 0xbc, 0x83, 0xa9, 0x5a, 0x04, 0xae, 0x8c, 0xc9, 0xb1, 0x3f, 0x20, 0x43, 0xf2, - 0x1e, 0xf4, 0x32, 0x0a, 0x34, 0xfe, 0xdd, 0x30, 0x0b, 0xad, 0x68, 0x2a, 0x4b, 0x0e, 0x77, 0xc2, - 0x74, 0x3a, 0x95, 0xd2, 0xb8, 0x90, 0x4a, 0x79, 0x49, 0x0d, 0x15, 0xfc, 0xe8, 0x43, 0x9c, 0x97, - 0xa0, 0x4e, 0xc5, 0x9a, 0xde, 0x24, 0xd6, 0x66, 0x48, 0xcd, 0x57, 0x94, 0xf2, 0x7a, 0x59, 0xba, - 0xa6, 0xf5, 0xff, 0x2a, 0x5d, 0xd3, 0xfe, 0x61, 0xe9, 0x9a, 0xce, 0x0f, 0x4b, 0xd7, 0x4c, 0xa5, - 0x37, 0xba, 0xd3, 0xe9, 0x8d, 0x97, 0x66, 0x94, 0x7b, 0x2f, 0xcd, 0x28, 0xbf, 0x26, 0x1d, 0xbc, - 0xf4, 0xea, 0x74, 0xf0, 0xeb, 0xf3, 0xd1, 0xe2, 0x75, 0xf9, 0xe8, 0x77, 0xa1, 0xcb, 0xc5, 0xcb, - 0xe4, 0x64, 0x9f, 0xca, 0x73, 0xa5, 0xf3, 0xdf, 0x6d, 0x02, 0xa3, 0x8b, 0xfd, 0xad, 0x3c, 0x57, - 0x83, 0x27, 0x00, 0x14, 0x7d, 0xa0, 0x9f, 0xf6, 0x32, 0xde, 0xa8, 0xfc, 0xe8, 0xfa, 0xba, 0xbf, - 0xaa, 0x00, 0x1c, 0x3a, 0xe3, 0x98, 0xd3, 0x07, 0xe2, 0x4f, 0xa1, 0xa9, 0xe8, 0xad, 0x58, 0x5f, - 0x54, 0x10, 0xd9, 0x39, 0xa9, 0x7e, 0xe4, 0x33, 0x86, 0x2a, 0x7b, 0x26, 0xb6, 0xe6, 0x1e, 0xb2, - 0x5a, 0xe6, 0xaa, 0x21, 0xa0, 0xb0, 0xee, 0x4d, 0xe8, 0x68, 0x82, 0x58, 0x26, 0xae, 0x0c, 0xf9, - 0x98, 0x49, 0xc5, 0x6a, 0x33, 0xf4, 0x80, 0x81, 0xe2, 0xa3, 0x8c, 0xcc, 0x58, 0x43, 0x17, 0xb3, - 0xa5, 0xba, 0x89, 0x36, 0x87, 0x06, 0x5b, 0xe6, 0xa7, 0xd0, 0x40, 0xea, 0xb0, 0x80, 0xdf, 0xeb, - 0xbd, 0x21, 0x9a, 0x50, 0xd3, 0xbd, 0xf6, 0x2a, 0xa2, 0x0d, 0x0d, 0x3a, 0xd4, 0x4a, 0xb8, 0xb9, - 0xc1, 0x3f, 0x5e, 0x85, 0xe6, 0x5e, 0xa8, 0xd2, 0x64, 0xc2, 0x2c, 0x9c, 0x1f, 0xdd, 0xac, 0xd2, - 0xd1, 0x4d, 0x7d, 0x1c, 0x81, 0x7f, 0x06, 0x1d, 0x47, 0xf8, 0x10, 0x6a, 0xfa, 0x94, 0xb0, 0xce, - 0x29, 0xcd, 0x3c, 0x62, 0x6c, 0x68, 0xc4, 0x26, 0xd4, 0x3d, 0x7d, 0x7c, 0x59, 0x17, 0x51, 0x15, - 0xce, 0x14, 0x9b, 0x83, 0xcd, 0x56, 0x46, 0x23, 0xde, 0x82, 0x79, 0x67, 0x34, 0xd2, 0x01, 0x9d, - 0x6e, 0x4e, 0x4a, 0xf6, 0xb9, 0x85, 0x38, 0x71, 0x07, 0x1a, 0x24, 0x3e, 0xa9, 0xa0, 0x71, 0x71, - 0xba, 0x4f, 0x53, 0x2d, 0xc9, 0x12, 0x95, 0xd2, 0x51, 0x77, 0xa0, 0x11, 0x44, 0x51, 0xcc, 0x0d, - 0x6a, 0xd3, 0x0d, 0x4c, 0x69, 0x99, 0x55, 0x0f, 0x4c, 0x91, 0xd9, 0xbb, 0xb0, 0x88, 0x9e, 0x5f, - 0x14, 0x6b, 0x8f, 0xa9, 0x30, 0x0e, 0x2a, 0xb1, 0xb2, 0xaa, 0x8a, 0x2a, 0xad, 0xb6, 0x00, 0x98, - 0xff, 0xa9, 0xe7, 0xc6, 0xf4, 0x74, 0x64, 0x69, 0x75, 0xdc, 0xa4, 0x26, 0xc3, 0x7e, 0x0f, 0x7a, - 0x9c, 0x42, 0x2d, 0xb4, 0x04, 0x53, 0x06, 0x6e, 0x5a, 0x96, 0xb3, 0xf2, 0x56, 0x27, 0x29, 0x67, - 0xe9, 0x3f, 0x80, 0x5a, 0xcc, 0x79, 0x41, 0x92, 0x30, 0x64, 0xad, 0x99, 0xa6, 0x3a, 0x61, 0x68, - 0x19, 0x0a, 0xf1, 0x4b, 0xe8, 0x70, 0xb9, 0xf2, 0x50, 0x27, 0xc8, 0x28, 0xa8, 0x5b, 0x3a, 0x60, - 0x5a, 0xca, 0x9f, 0x59, 0xed, 0xb4, 0x94, 0x4e, 0xfb, 0x05, 0xb4, 0xf3, 0x73, 0x73, 0x68, 0xec, - 0x75, 0x4d, 0xa2, 0xca, 0x34, 0x2f, 0x3a, 0xe2, 0x56, 0x4b, 0x16, 0xdd, 0xf2, 0x0d, 0x58, 0xd4, - 0x25, 0xf4, 0x3d, 0x6a, 0x55, 0xb8, 0xfc, 0x83, 0x8b, 0x66, 0x2d, 0x8d, 0xc7, 0xb9, 0xcc, 0xab, - 0x83, 0xc9, 0x67, 0x28, 0xcd, 0x65, 0x56, 0x1a, 0x6c, 0x35, 0xb2, 0xaa, 0x60, 0xf1, 0xa0, 0x5c, - 0xad, 0xcc, 0xf9, 0xdc, 0x65, 0x6a, 0x7a, 0x69, 0x46, 0x53, 0xce, 0xec, 0x5a, 0xdd, 0x78, 0xaa, - 0xe8, 0xf9, 0x36, 0xd4, 0xa3, 0xc4, 0xa3, 0x63, 0x2a, 0x54, 0x40, 0x93, 0x59, 0xbf, 0xfa, 0x68, - 0x34, 0x09, 0x8f, 0x5a, 0xc4, 0x2f, 0x68, 0x58, 0xc4, 0x49, 0x44, 0x66, 0x23, 0x89, 0xb8, 0xd5, - 0x8b, 0x86, 0x85, 0xc6, 0x93, 0x80, 0x7b, 0x07, 0x6a, 0xe6, 0x60, 0xc0, 0xda, 0x05, 0x4a, 0x83, - 0x12, 0x1f, 0x43, 0xb7, 0x2c, 0xd0, 0x54, 0x7f, 0xfd, 0x02, 0x75, 0xa7, 0x24, 0xbf, 0x50, 0x1b, - 0x57, 0x03, 0x7f, 0xec, 0xa7, 0xda, 0xdc, 0x2f, 0x1d, 0x54, 0x26, 0x04, 0xba, 0xfe, 0x3a, 0xfb, - 0x75, 0xe9, 0xa2, 0xeb, 0xaf, 0x33, 0x64, 0x7d, 0xa8, 0xf9, 0x6a, 0xd7, 0x4f, 0x54, 0xaa, 0x4b, - 0x5b, 0xcc, 0xab, 0x58, 0x83, 0x45, 0x5f, 0xa1, 0x9a, 0xd0, 0x06, 0xba, 0x7e, 0x13, 0xb7, 0x60, - 0x51, 0x1f, 0x9a, 0xb8, 0x71, 0x61, 0x47, 0xeb, 0x03, 0x5e, 0x96, 0xa6, 0x10, 0xef, 0x43, 0x8d, - 0x2a, 0xe6, 0xa3, 0x98, 0x2c, 0xc5, 0x12, 0x07, 0x70, 0xd9, 0xba, 0xb5, 0x18, 0x70, 0xf9, 0xfa, - 0x07, 0x50, 0x33, 0x46, 0xca, 0x60, 0x9a, 0xab, 0xb5, 0xb1, 0x62, 0x19, 0x0a, 0x71, 0x13, 0xaa, - 0x63, 0x94, 0x63, 0xfd, 0xb7, 0xa7, 0x77, 0x28, 0x8b, 0x37, 0xc6, 0x8a, 0xbf, 0x0d, 0x97, 0x8b, - 0x35, 0xe7, 0xa6, 0x20, 0x5d, 0xc7, 0xb6, 0x6f, 0x52, 0xdb, 0xb7, 0x66, 0xb0, 0x4a, 0xb9, 0x74, - 0xdd, 0x5a, 0x8f, 0x5f, 0x52, 0xd3, 0xfe, 0x69, 0x26, 0xee, 0x71, 0x77, 0xf5, 0xdf, 0x35, 0x7e, - 0xd7, 0x45, 0x85, 0x61, 0x94, 0x00, 0xe9, 0x99, 0xcf, 0xa0, 0x35, 0x9c, 0xbc, 0x78, 0x71, 0x6e, - 0xf2, 0x32, 0xef, 0x51, 0xbb, 0x42, 0x74, 0xa9, 0x50, 0xe6, 0x6e, 0x35, 0x87, 0x85, 0x9a, 0xf7, - 0x75, 0xa8, 0xb9, 0xa1, 0xed, 0x78, 0x5e, 0xd2, 0xdf, 0xe0, 0x32, 0x77, 0x37, 0xdc, 0xf6, 0x3c, - 0xba, 0x63, 0x24, 0x8a, 0x25, 0x1d, 0xd4, 0xb7, 0x7d, 0xaf, 0xff, 0x3e, 0x2b, 0x1e, 0x03, 0xda, - 0xf3, 0xe8, 0x12, 0x12, 0x13, 0x92, 0xf1, 0xbd, 0xfe, 0x2d, 0x7d, 0x09, 0x89, 0x06, 0xed, 0x79, - 0x68, 0x78, 0xa2, 0xff, 0x6a, 0x20, 0xfd, 0x0f, 0x38, 0xd7, 0x35, 0x76, 0xce, 0x0e, 0x34, 0x08, - 0x37, 0x29, 0x67, 0x75, 0x49, 0x6c, 0xdd, 0x9e, 0xde, 0xa4, 0x59, 0x05, 0x80, 0xd5, 0xf0, 0xb3, - 0x62, 0x00, 0xda, 0xd8, 0x24, 0x8a, 0xec, 0x60, 0xab, 0xff, 0xe1, 0xc5, 0x8d, 0xad, 0x0b, 0x1c, - 0x70, 0x63, 0x9b, 0x5a, 0x87, 0x2d, 0x00, 0x96, 0x59, 0x24, 0x70, 0x36, 0xa7, 0xdb, 0x64, 0xde, - 0x80, 0xc5, 0x27, 0xeb, 0x48, 0xd4, 0x6c, 0x01, 0x50, 0x66, 0x89, 0xdb, 0xdc, 0x99, 0x6e, 0x93, - 0x59, 0xf7, 0x56, 0xe3, 0x59, 0x66, 0xe8, 0xdf, 0x81, 0xc6, 0x04, 0xed, 0x78, 0xb4, 0xa4, 0xfb, - 0x77, 0xa7, 0x99, 0xd9, 0x98, 0xf8, 0x56, 0x7d, 0xa2, 0x9f, 0xf0, 0x23, 0xa4, 0x7b, 0xc8, 0x0c, - 0xe9, 0x7f, 0x34, 0xfd, 0x91, 0xcc, 0x0f, 0xb0, 0x48, 0x45, 0xb1, 0x4b, 0xf0, 0x29, 0x34, 0x79, - 0xd2, 0xb8, 0xd1, 0xd6, 0x34, 0x8f, 0xe4, 0x76, 0x8d, 0xc5, 0xb3, 0xcb, 0xcd, 0x6e, 0x42, 0xd5, - 0x89, 0xe3, 0xe0, 0xbc, 0xff, 0xf1, 0x34, 0x87, 0x6f, 0x23, 0xd8, 0x62, 0x2c, 0xb2, 0xd2, 0x78, - 0x12, 0xa4, 0xbe, 0x39, 0x0c, 0xf7, 0xc9, 0x34, 0x2b, 0x15, 0x4e, 0x17, 0x5b, 0xcd, 0x71, 0xe1, - 0xa8, 0xf1, 0x6d, 0xa8, 0xc7, 0x91, 0x4a, 0x6d, 0x6f, 0x1c, 0xf4, 0x3f, 0xbd, 0xa0, 0x46, 0xf8, - 0x30, 0x92, 0x55, 0x8b, 0xf5, 0x69, 0xae, 0xd2, 0x61, 0xfd, 0x9f, 0x4d, 0x1d, 0xd6, 0xdf, 0x82, - 0xd6, 0x38, 0x0a, 0x47, 0x91, 0x77, 0xcc, 0xb3, 0xff, 0xf3, 0x62, 0x3c, 0x60, 0x1f, 0x31, 0x1c, - 0x41, 0xd0, 0x44, 0x5a, 0x35, 0xf4, 0xb4, 0xed, 0xe6, 0x2b, 0xdb, 0x51, 0x24, 0xf6, 0x3f, 0xe3, - 0xe8, 0x0d, 0xc3, 0xf7, 0xd4, 0x36, 0x41, 0xcb, 0x99, 0x8d, 0xe3, 0x73, 0x9d, 0x86, 0xfe, 0x9c, - 0xd8, 0x33, 0xcf, 0x6c, 0xdc, 0x3b, 0xe7, 0x5c, 0xf4, 0x17, 0x70, 0xb9, 0x70, 0x7c, 0x36, 0x8a, - 0xed, 0xd0, 0x46, 0x11, 0x90, 0x48, 0x6f, 0xe2, 0xca, 0xfe, 0x17, 0x59, 0xa8, 0x41, 0x9f, 0xa1, - 0x8d, 0xe2, 0x47, 0x07, 0x89, 0xb4, 0x08, 0x2b, 0x1e, 0x15, 0x8f, 0xea, 0x3a, 0xc1, 0x28, 0x4a, - 0xfc, 0xf4, 0x64, 0xdc, 0xff, 0x85, 0x09, 0x34, 0x64, 0xfe, 0x40, 0x96, 0xfa, 0xdc, 0x36, 0x44, - 0x56, 0x3e, 0xc6, 0x0c, 0x26, 0xbe, 0x80, 0x4b, 0x5a, 0x17, 0x60, 0x87, 0xa5, 0x1b, 0x11, 0x54, - 0xff, 0x4f, 0xe8, 0x4a, 0x84, 0xf5, 0x9c, 0xe0, 0xeb, 0xc2, 0xe5, 0x08, 0x0a, 0xdd, 0xf6, 0x59, - 0x6d, 0xd1, 0x30, 0xe1, 0x09, 0xf8, 0x92, 0x26, 0xe0, 0xf2, 0xc5, 0xf6, 0x87, 0x52, 0x5f, 0x8f, - 0x74, 0x1b, 0x04, 0x4e, 0x00, 0x69, 0x2e, 0xe9, 0xd9, 0xd1, 0x24, 0x8d, 0x27, 0x69, 0xff, 0x97, - 0xec, 0x27, 0xa6, 0x51, 0xfc, 0x98, 0x11, 0x8f, 0x09, 0x2e, 0x3e, 0x87, 0x4b, 0x24, 0x34, 0xed, - 0x62, 0x1b, 0x0e, 0x14, 0xa9, 0xfe, 0x57, 0x3c, 0x6f, 0x44, 0x70, 0x94, 0xb5, 0xe4, 0xb0, 0x12, - 0x55, 0x58, 0x4e, 0xcf, 0xf9, 0x73, 0x3f, 0x3d, 0xb1, 0x29, 0xa2, 0xf2, 0xa7, 0x1c, 0x21, 0x28, - 0x4d, 0xf9, 0x77, 0x7e, 0x7a, 0x72, 0xe4, 0x4b, 0xc5, 0x1e, 0xfe, 0xaf, 0x16, 0xea, 0x4b, 0x3d, - 0xf1, 0xab, 0x85, 0xfa, 0x3b, 0xbd, 0x9b, 0x56, 0xb3, 0x10, 0xc8, 0x1a, 0x7c, 0x0a, 0xad, 0x6d, - 0xba, 0xba, 0xcb, 0x57, 0xa4, 0x4d, 0x6f, 0xc2, 0x42, 0x56, 0x00, 0x95, 0xa9, 0x69, 0xa2, 0x78, - 0x21, 0xf7, 0xc2, 0x61, 0x64, 0x11, 0x7a, 0xf0, 0xaf, 0x17, 0x60, 0x91, 0x8b, 0x52, 0x5e, 0x7f, - 0x3c, 0xf7, 0x4d, 0x23, 0x6b, 0xc2, 0xfc, 0xec, 0x10, 0x8b, 0x15, 0x42, 0x4f, 0x9f, 0x11, 0x68, - 0xe4, 0xb5, 0x55, 0x2b, 0x50, 0xe5, 0x00, 0x01, 0xa7, 0xcc, 0xf8, 0x85, 0xe4, 0xec, 0x44, 0x9d, - 0xd0, 0xfd, 0x5c, 0x3a, 0x09, 0xbc, 0x60, 0x81, 0x01, 0xed, 0x79, 0x14, 0x1b, 0x37, 0x04, 0x24, - 0xc8, 0x17, 0x75, 0xee, 0x4b, 0x03, 0x49, 0x9c, 0x9b, 0xba, 0xad, 0xda, 0x4b, 0xea, 0xb6, 0xae, - 0xc1, 0x42, 0x68, 0x8e, 0xb5, 0x65, 0x78, 0xba, 0x9f, 0x87, 0xe0, 0xe2, 0x16, 0x64, 0x67, 0x8a, - 0xb5, 0x61, 0xfa, 0xf2, 0x33, 0xc7, 0x5b, 0xd0, 0xc8, 0x2e, 0x7b, 0xd3, 0xb6, 0xe8, 0xca, 0x66, - 0x7e, 0xfd, 0xdb, 0x91, 0x79, 0xb2, 0x72, 0xb2, 0x57, 0x57, 0x1f, 0x35, 0x7f, 0x5a, 0xf5, 0x11, - 0x3b, 0xea, 0x6e, 0x14, 0xaa, 0x54, 0x47, 0xff, 0x6b, 0xbe, 0xda, 0xc1, 0x57, 0xf1, 0x39, 0xb4, - 0x13, 0xe9, 0x3e, 0xb3, 0xc7, 0x6a, 0xc4, 0x9f, 0x68, 0x17, 0xef, 0x35, 0x18, 0xab, 0xd1, 0x37, - 0x54, 0x19, 0xa5, 0xfd, 0xe6, 0x26, 0xd2, 0xee, 0xab, 0x11, 0xf5, 0xfa, 0x01, 0x2c, 0x8d, 0xe5, - 0xf8, 0x58, 0x26, 0xea, 0xc4, 0x8f, 0x8d, 0xc2, 0xed, 0x50, 0xc4, 0xac, 0x97, 0x23, 0x78, 0x2c, - 0x83, 0x7f, 0x50, 0x81, 0x3a, 0xce, 0x22, 0xf2, 0x92, 0x10, 0xb0, 0x30, 0x76, 0xe3, 0x89, 0x76, - 0x87, 0xe8, 0x59, 0x5f, 0x20, 0xc7, 0x5c, 0xa2, 0x2f, 0x90, 0xa3, 0x35, 0xe4, 0x9c, 0x36, 0x3d, - 0xf3, 0x9d, 0x40, 0xe7, 0x94, 0xf6, 0x60, 0xce, 0x30, 0xaf, 0x62, 0x15, 0x16, 0xdd, 0x90, 0x62, - 0x22, 0x5c, 0x1b, 0x50, 0x75, 0xc3, 0x9d, 0x30, 0xd5, 0xe0, 0xfc, 0x68, 0x54, 0xd5, 0x0d, 0xf7, - 0xbc, 0xb3, 0xc1, 0xbf, 0xab, 0xc0, 0xd2, 0x41, 0x12, 0xb9, 0x52, 0xa9, 0x87, 0x68, 0xce, 0x51, - 0x1e, 0x16, 0xbf, 0x48, 0x69, 0x2b, 0x4e, 0x79, 0xd2, 0x33, 0xf2, 0x30, 0x07, 0xac, 0x32, 0xa7, - 0x73, 0xde, 0x6a, 0x10, 0x84, 0x7c, 0xce, 0x0c, 0x5d, 0xa8, 0xef, 0x61, 0x34, 0x25, 0xbc, 0x6e, - 0x42, 0x27, 0xdf, 0xbd, 0x85, 0x52, 0xa4, 0xfc, 0xe6, 0x0e, 0xea, 0xe5, 0x3a, 0x34, 0x75, 0xc1, - 0x1a, 0x75, 0xc3, 0x39, 0x4c, 0x60, 0xd0, 0xa1, 0x1e, 0x05, 0xab, 0x08, 0xc2, 0x73, 0xd6, 0x92, - 0x95, 0x06, 0xa2, 0x07, 0x7f, 0x6f, 0x0e, 0x7a, 0x07, 0x89, 0x8c, 0x9d, 0x44, 0x52, 0x05, 0x1b, - 0xcd, 0xf1, 0x1a, 0x2c, 0x06, 0x32, 0x1c, 0xe9, 0xa2, 0xa5, 0x79, 0x4b, 0xbf, 0x65, 0x97, 0xff, - 0xcd, 0x15, 0x2e, 0xff, 0xc3, 0xb9, 0x4e, 0xa4, 0xa3, 0xef, 0x08, 0xa4, 0x67, 0xdc, 0x83, 0xe1, - 0x24, 0xd0, 0xa9, 0xb6, 0xba, 0xc5, 0x2f, 0xfa, 0x7e, 0x80, 0x63, 0x3f, 0xa4, 0x4a, 0x61, 0xba, - 0x1f, 0xe0, 0x9e, 0x4f, 0x2a, 0x87, 0xc1, 0x68, 0x02, 0xf2, 0x3d, 0x5b, 0x14, 0xf0, 0xaa, 0x5b, - 0x1d, 0x22, 0x70, 0x92, 0xf3, 0x43, 0x82, 0x92, 0x97, 0xce, 0xd7, 0x73, 0x71, 0x19, 0x1c, 0x67, - 0xd9, 0xda, 0xe6, 0x76, 0x2e, 0x96, 0x2d, 0x94, 0x45, 0x37, 0xbb, 0x40, 0x93, 0x7b, 0xd1, 0xd8, - 0xf1, 0x43, 0x45, 0x05, 0x78, 0x6d, 0x6b, 0x45, 0x63, 0xb9, 0xd7, 0xfb, 0x8c, 0x1b, 0xfc, 0x93, - 0x1a, 0x34, 0xf5, 0xba, 0xd2, 0x1c, 0x30, 0x4f, 0x55, 0x32, 0x9e, 0xea, 0xc1, 0xbc, 0x7a, 0x1a, - 0x68, 0x26, 0xc3, 0x47, 0xf1, 0x31, 0xcc, 0x07, 0xfe, 0x58, 0x3b, 0xdc, 0x57, 0x4a, 0xc6, 0x53, - 0x99, 0x3b, 0xf4, 0x06, 0x40, 0x6a, 0xd4, 0xd8, 0x74, 0xc5, 0x09, 0x0e, 0x40, 0xaf, 0x28, 0x1a, - 0x32, 0x67, 0xb8, 0x9f, 0x71, 0xad, 0x1c, 0x97, 0x6b, 0xdb, 0xb5, 0x90, 0x6a, 0x5b, 0x0d, 0x0d, - 0xd9, 0xf3, 0xc4, 0x27, 0x50, 0xcf, 0x32, 0x42, 0xc6, 0xc5, 0x4e, 0xcf, 0xc2, 0xcd, 0x9d, 0x47, - 0x47, 0x67, 0xa1, 0x49, 0xf9, 0xe8, 0x8f, 0x65, 0x94, 0xe2, 0x97, 0xd0, 0x52, 0x52, 0x29, 0xbe, - 0x83, 0x62, 0x18, 0x69, 0xe1, 0xb5, 0x5a, 0xf4, 0x9e, 0x09, 0x8b, 0xbf, 0xda, 0x6c, 0x55, 0x95, - 0x83, 0xc4, 0x37, 0xd0, 0x31, 0xed, 0x83, 0x68, 0x34, 0xca, 0x32, 0x96, 0x57, 0x2e, 0xf4, 0xf0, - 0x90, 0xd0, 0x85, 0x7e, 0xda, 0xaa, 0x88, 0x10, 0x5f, 0x43, 0x27, 0x66, 0x56, 0xb3, 0x75, 0xcd, - 0x26, 0x0b, 0xc1, 0xcb, 0x25, 0x5b, 0xbf, 0xc4, 0x8a, 0xf9, 0xf9, 0xe6, 0x1c, 0xae, 0x2e, 0xde, - 0xc7, 0xc2, 0x29, 0xec, 0xf2, 0x7d, 0x2c, 0x12, 0xd6, 0xf4, 0x05, 0x6a, 0xc3, 0xc4, 0xa1, 0x7b, - 0x25, 0x58, 0x45, 0x9b, 0x42, 0xec, 0x3b, 0x17, 0x56, 0x0c, 0x3f, 0xb8, 0xc9, 0xf7, 0x8e, 0xec, - 0xea, 0x26, 0xa4, 0xb2, 0x75, 0x55, 0xdb, 0x4a, 0x32, 0x03, 0x25, 0x36, 0x61, 0x59, 0x7f, 0x26, - 0xcf, 0x84, 0xf8, 0x1e, 0x89, 0xca, 0x96, 0xb5, 0xc4, 0xa8, 0x2c, 0x76, 0xbe, 0xe7, 0x89, 0xcf, - 0xa0, 0xaf, 0x52, 0x27, 0xe5, 0xe3, 0xfc, 0x86, 0x4f, 0xfd, 0x51, 0x18, 0x25, 0x52, 0x57, 0x82, - 0xad, 0x65, 0x78, 0x2d, 0xa5, 0xf7, 0x08, 0x2b, 0x7e, 0x09, 0x3d, 0xca, 0x10, 0x64, 0xe5, 0x28, - 0xa9, 0xd2, 0x51, 0x83, 0xd9, 0x8a, 0xa1, 0x83, 0xd4, 0x86, 0x2d, 0x8e, 0x68, 0x5f, 0x50, 0x7b, - 0x7d, 0x87, 0x00, 0xc9, 0x15, 0x39, 0x51, 0xd2, 0xd3, 0xb7, 0x26, 0xad, 0x20, 0xf6, 0xeb, 0x0c, - 0x69, 0x11, 0x0e, 0x2d, 0x1e, 0xb3, 0xe9, 0x74, 0x64, 0x98, 0x6c, 0xed, 0xdc, 0x0e, 0xeb, 0x71, - 0xa2, 0x42, 0xef, 0x41, 0xa6, 0x41, 0x93, 0x3b, 0x33, 0xb8, 0x2e, 0x7f, 0x0d, 0x97, 0x5e, 0x3a, - 0xab, 0xaf, 0x2b, 0x4d, 0x6b, 0x17, 0xef, 0xf8, 0xf8, 0xcf, 0x55, 0x68, 0x16, 0xb8, 0x95, 0x2e, - 0x0a, 0x55, 0x32, 0x31, 0x05, 0xa8, 0xf8, 0x8c, 0xb0, 0x93, 0x48, 0x99, 0x7a, 0x4a, 0x7a, 0x46, - 0x58, 0x12, 0x65, 0x75, 0x65, 0xf4, 0x8c, 0x3c, 0xa4, 0xc3, 0x61, 0x7a, 0xc5, 0x16, 0xf8, 0x6e, - 0x9a, 0x1c, 0xb8, 0xe7, 0xd1, 0x8d, 0xa2, 0x4e, 0xea, 0x1c, 0x3b, 0xca, 0x54, 0x12, 0x67, 0xef, - 0xa8, 0x50, 0x4c, 0xae, 0x46, 0xd7, 0xd1, 0xe8, 0x57, 0xdc, 0xe3, 0xb4, 0xaa, 0x2f, 0xa2, 0x90, - 0x6b, 0x68, 0x5a, 0x56, 0x1d, 0x01, 0xdf, 0x47, 0x21, 0x35, 0xd3, 0x3b, 0x5a, 0xd7, 0x82, 0x99, - 0x57, 0xd4, 0xb4, 0x4f, 0x27, 0x12, 0xfd, 0x60, 0x8f, 0xce, 0xcd, 0x36, 0xac, 0x1a, 0xbd, 0x73, - 0x79, 0x1a, 0x39, 0xec, 0xcf, 0x1d, 0x3f, 0x25, 0xd1, 0x11, 0x4d, 0x52, 0xcd, 0xf4, 0x5d, 0x44, - 0x7c, 0xe7, 0xf8, 0xe9, 0x11, 0x83, 0xc5, 0x47, 0xfa, 0x38, 0x7c, 0x91, 0x96, 0xee, 0xc6, 0xe2, - 0x30, 0xbb, 0x98, 0xa2, 0x3f, 0x94, 0x74, 0xfd, 0xe3, 0xd8, 0x49, 0x13, 0xff, 0x2c, 0x0a, 0xd1, - 0xe2, 0xa2, 0x6b, 0x30, 0xb2, 0x5b, 0x4c, 0xeb, 0xd6, 0x72, 0x86, 0x7c, 0x44, 0x38, 0x2a, 0x3a, - 0x78, 0x02, 0x1b, 0xf2, 0x2c, 0x0e, 0x7c, 0xd7, 0x9f, 0xba, 0xce, 0xc3, 0x76, 0x1d, 0x95, 0xda, - 0x89, 0x4c, 0x27, 0x49, 0xa8, 0x28, 0x82, 0xac, 0xf9, 0xfa, 0x6d, 0x43, 0x5f, 0xbc, 0xe2, 0x63, - 0xc7, 0x51, 0xa9, 0xc5, 0xb4, 0x8f, 0x26, 0x41, 0x80, 0x93, 0x90, 0x15, 0x3b, 0x70, 0x9d, 0x63, - 0x4d, 0xe9, 0x32, 0x87, 0xcf, 0xa0, 0x3f, 0x75, 0x1b, 0x46, 0xf6, 0x44, 0x1c, 0xbc, 0x60, 0xad, - 0x95, 0xae, 0xc1, 0xc8, 0x1e, 0xf0, 0xf7, 0x4d, 0xb5, 0xd4, 0x01, 0x18, 0xbe, 0x8b, 0x74, 0xb9, - 0xd4, 0x4c, 0x5f, 0xc5, 0xf2, 0x0e, 0x74, 0xb2, 0x45, 0x64, 0x2b, 0x74, 0x89, 0x2d, 0x42, 0xb3, - 0x92, 0xfa, 0x1c, 0x7b, 0x17, 0xdd, 0x73, 0x99, 0x24, 0x51, 0xa2, 0x3d, 0x00, 0x41, 0x5c, 0xdb, - 0x1e, 0x3b, 0x67, 0x0f, 0x10, 0xca, 0x46, 0xff, 0x87, 0xb0, 0x3c, 0x45, 0x47, 0x4b, 0xc2, 0x51, - 0xf5, 0x5e, 0x89, 0xf6, 0x50, 0xa6, 0x83, 0xff, 0x5a, 0x81, 0xa5, 0x0b, 0x42, 0x55, 0xac, 0x43, - 0x0d, 0x05, 0xaa, 0x29, 0x43, 0x6b, 0x59, 0x8b, 0xf8, 0xba, 0xe7, 0x11, 0x22, 0x1d, 0xa7, 0xa6, - 0x00, 0x0d, 0x11, 0xe9, 0x18, 0x35, 0xc6, 0x2a, 0x2c, 0xa6, 0x67, 0xc4, 0xdd, 0xac, 0x9e, 0xab, - 0xe9, 0x19, 0xb2, 0xf5, 0x36, 0x34, 0x82, 0x68, 0x64, 0x07, 0xf2, 0x99, 0xe4, 0xeb, 0xa4, 0x3a, - 0x5b, 0xef, 0xbc, 0x42, 0x9a, 0x6f, 0x3e, 0x8c, 0x46, 0x0f, 0x91, 0xd6, 0xaa, 0x07, 0xfa, 0x69, - 0xf0, 0x2b, 0xa8, 0x1b, 0xa8, 0x68, 0x40, 0xf5, 0xbe, 0x3c, 0x9e, 0x8c, 0x7a, 0x6f, 0x88, 0x3a, - 0x2c, 0x60, 0x8b, 0x5e, 0x05, 0x9f, 0xbe, 0x73, 0x92, 0xb0, 0x37, 0x87, 0x68, 0xfa, 0x75, 0xbd, - 0x79, 0x7c, 0x3c, 0x70, 0x42, 0xdf, 0xed, 0x2d, 0xe0, 0xe3, 0xae, 0x93, 0x3a, 0x41, 0xaf, 0x3a, - 0xf8, 0x0f, 0x55, 0xa8, 0x1f, 0xe8, 0xaf, 0x8b, 0xfb, 0xd0, 0xce, 0xae, 0x65, 0x9d, 0x1d, 0xef, - 0x3f, 0x98, 0x7e, 0xa0, 0x78, 0x7f, 0x2b, 0x2e, 0xbc, 0x4d, 0x5f, 0xee, 0x3a, 0x77, 0xe1, 0x72, - 0xd7, 0xab, 0x30, 0xff, 0x34, 0x39, 0x2f, 0x9f, 0xc0, 0x38, 0x08, 0x9c, 0xd0, 0x42, 0xb0, 0xf8, - 0x08, 0x9a, 0x54, 0x64, 0xc2, 0x76, 0x86, 0x8e, 0x91, 0x17, 0xef, 0x51, 0xe6, 0xda, 0x7a, 0x40, - 0x22, 0xed, 0xd2, 0x6c, 0x42, 0xdd, 0x3d, 0xf1, 0x03, 0x2f, 0x91, 0xa1, 0x3e, 0x09, 0x25, 0x2e, - 0x0e, 0xd9, 0xca, 0x68, 0xc4, 0x9f, 0x42, 0xcf, 0xcf, 0x63, 0xfc, 0x79, 0x4d, 0x51, 0x49, 0x35, - 0x17, 0xb2, 0x00, 0x56, 0xb7, 0x40, 0x4e, 0x36, 0x74, 0x7e, 0xdf, 0x52, 0xad, 0x78, 0xdf, 0x12, - 0x5f, 0xb7, 0x49, 0x86, 0x6e, 0x3d, 0x8b, 0x10, 0xa2, 0x9d, 0xfb, 0xae, 0xf6, 0x4e, 0x1a, 0xd3, - 0x21, 0x15, 0x63, 0x5b, 0x6b, 0x2f, 0xe5, 0x1d, 0xe8, 0xa0, 0xd7, 0x63, 0xb3, 0xb3, 0x84, 0x26, - 0x03, 0xe8, 0xeb, 0xe1, 0x26, 0xea, 0xe4, 0x3e, 0xba, 0x4b, 0xc8, 0x8c, 0x37, 0xa1, 0x63, 0x7e, - 0x8b, 0x66, 0xfc, 0xa6, 0xae, 0x39, 0xd2, 0x50, 0x66, 0xfc, 0x4d, 0x58, 0x76, 0x4f, 0x9c, 0x30, - 0x94, 0x81, 0x7d, 0x3c, 0x19, 0x0e, 0x8d, 0x9d, 0xca, 0x77, 0x06, 0x2e, 0x69, 0xd4, 0x3d, 0xc2, - 0x90, 0xb9, 0x3a, 0x80, 0x76, 0xe8, 0x07, 0xe6, 0xde, 0xe0, 0x90, 0x7d, 0x8a, 0xaa, 0xd5, 0x0c, - 0xfd, 0x80, 0x6f, 0x0a, 0xa6, 0x0b, 0x8e, 0x7b, 0x93, 0x89, 0xef, 0x29, 0x3b, 0x8d, 0xcc, 0x4d, - 0xa5, 0x3a, 0x5b, 0x57, 0x88, 0x7f, 0x3f, 0x99, 0xf8, 0xde, 0x51, 0xa4, 0xef, 0x2a, 0x6d, 0x13, - 0xbd, 0x79, 0xc5, 0xb1, 0x97, 0x5d, 0x69, 0x2a, 0xfc, 0xa9, 0x5b, 0xed, 0xa8, 0xe8, 0x41, 0x0f, - 0xbe, 0x82, 0x56, 0x91, 0xc5, 0x90, 0x65, 0x29, 0x8e, 0xd9, 0x7b, 0x43, 0x00, 0x2c, 0x3e, 0x8a, - 0x92, 0xb1, 0x13, 0xf4, 0x2a, 0xf8, 0xcc, 0xea, 0xad, 0x37, 0x27, 0x5a, 0x50, 0x37, 0x71, 0xb9, - 0xde, 0xbc, 0xce, 0x94, 0xff, 0x02, 0xea, 0xe6, 0x9e, 0x56, 0xba, 0xe3, 0x32, 0xf2, 0x24, 0xbb, - 0x98, 0xfa, 0x44, 0x00, 0x02, 0xc8, 0xbd, 0x34, 0xf7, 0x5e, 0xcf, 0xe5, 0xf7, 0x5e, 0x0f, 0x7e, - 0x0d, 0xad, 0xe2, 0x2f, 0x31, 0xb9, 0x9f, 0x4a, 0x9e, 0xfb, 0x99, 0xd1, 0x8a, 0x2a, 0xd5, 0x92, - 0x68, 0x6c, 0x17, 0xbc, 0xa0, 0x3a, 0x02, 0xf0, 0x33, 0x83, 0xbf, 0x3f, 0x07, 0x55, 0x8a, 0x56, - 0x91, 0xbd, 0x89, 0x0f, 0xf9, 0x46, 0xab, 0x5a, 0x0d, 0x82, 0xfc, 0x5f, 0x1c, 0xd2, 0xcf, 0x6a, - 0x01, 0x16, 0x5e, 0x5d, 0x0b, 0xb0, 0x0d, 0x4b, 0xcf, 0x0a, 0x77, 0x2b, 0x73, 0x8c, 0xaa, 0x6a, - 0x8c, 0x53, 0x6c, 0xf3, 0x9b, 0xfc, 0x8e, 0x65, 0x8a, 0x54, 0x75, 0x9f, 0x95, 0x01, 0xe2, 0x2d, - 0xd0, 0xe7, 0x9d, 0x6c, 0x2e, 0x33, 0xe4, 0x9a, 0xbc, 0x26, 0xc3, 0xb6, 0xa9, 0xa8, 0xf0, 0x4d, - 0x00, 0x94, 0x80, 0x5a, 0xde, 0x73, 0x19, 0x66, 0x23, 0x3d, 0x0b, 0x59, 0xca, 0x0f, 0xc6, 0xb0, - 0x3c, 0xe3, 0x70, 0xa6, 0xb8, 0x01, 0xad, 0x52, 0xc6, 0x94, 0x4f, 0x4c, 0x81, 0x9b, 0xa7, 0x48, - 0x3f, 0x81, 0x35, 0x19, 0xf8, 0x23, 0xff, 0xd8, 0xa7, 0x5a, 0xf4, 0xc2, 0x79, 0x51, 0x96, 0x35, - 0x2b, 0x05, 0x6c, 0x76, 0x5e, 0x74, 0xf0, 0x67, 0x15, 0x58, 0x9e, 0x51, 0xba, 0xf2, 0xba, 0xc3, - 0x2d, 0x6f, 0x02, 0x98, 0x1a, 0x91, 0xcc, 0xcf, 0x6d, 0x68, 0xc8, 0x1e, 0xb9, 0xbb, 0x32, 0x75, - 0x46, 0xc6, 0xbc, 0xc1, 0xe7, 0xcc, 0x21, 0x5d, 0x28, 0x38, 0xa4, 0x9f, 0xc2, 0x7a, 0x80, 0xba, - 0x79, 0x1c, 0x79, 0xfe, 0xd0, 0x97, 0x5e, 0xe1, 0xb2, 0x45, 0xf6, 0x1b, 0x57, 0x10, 0xbd, 0xaf, - 0xb1, 0xe6, 0xb2, 0xc5, 0xc1, 0x9f, 0xcd, 0xc1, 0xea, 0xcc, 0xb2, 0x98, 0xd7, 0x0d, 0xfb, 0x36, - 0x88, 0x72, 0xcd, 0x4d, 0xe1, 0x58, 0x4e, 0x2f, 0x29, 0x74, 0x46, 0xc7, 0x3a, 0x36, 0xa0, 0x57, - 0xa2, 0xce, 0x0f, 0xe7, 0x74, 0x0a, 0xb4, 0x28, 0xe1, 0xf6, 0xe0, 0x2d, 0x53, 0x2e, 0x6e, 0x7b, - 0x3e, 0x09, 0x44, 0xf4, 0x1d, 0x29, 0xd2, 0x83, 0x63, 0xf1, 0x5d, 0x69, 0xce, 0x04, 0x5c, 0x33, - 0x84, 0xf7, 0x33, 0x3a, 0x2a, 0x16, 0xd9, 0x63, 0x2a, 0x94, 0x04, 0xa5, 0x0b, 0xd7, 0xcc, 0xc1, - 0x9e, 0x76, 0xf1, 0xae, 0x35, 0x25, 0xee, 0xc2, 0x4a, 0x4e, 0xf6, 0xdc, 0x4f, 0x74, 0xcd, 0xad, - 0x76, 0xa7, 0x45, 0x86, 0xfb, 0xce, 0x4f, 0xb8, 0xe2, 0xf6, 0xd6, 0xbf, 0xa9, 0xc0, 0x22, 0xdf, - 0xa4, 0x2f, 0x96, 0xa0, 0xfd, 0x24, 0x3c, 0x0d, 0xa3, 0xe7, 0x21, 0x03, 0x7a, 0x6f, 0x88, 0x65, - 0xe8, 0x1a, 0xc9, 0xa2, 0xaf, 0xec, 0xef, 0x55, 0x44, 0x0f, 0x5a, 0xf4, 0x13, 0x0d, 0x64, 0x4e, - 0x5c, 0x85, 0xbe, 0x76, 0x88, 0xee, 0xa3, 0xc1, 0x11, 0xa5, 0xfe, 0xf0, 0xdc, 0x60, 0xe7, 0x45, - 0x17, 0x9a, 0x87, 0x69, 0x14, 0x1f, 0xca, 0xd0, 0xf3, 0xc3, 0x51, 0x6f, 0x41, 0xf4, 0x61, 0xc5, - 0xf4, 0xca, 0x12, 0x6c, 0xd7, 0x0f, 0x7d, 0x75, 0xd2, 0xab, 0x8a, 0x2b, 0xb0, 0x3e, 0x0b, 0xb3, - 0xed, 0x9e, 0xf6, 0x16, 0xc5, 0x0a, 0xf4, 0x0c, 0xf2, 0x9e, 0xbe, 0x37, 0xbd, 0x57, 0xbb, 0xf5, - 0x09, 0x88, 0x8b, 0x57, 0xd6, 0xe3, 0x37, 0x1f, 0xca, 0x91, 0xe3, 0x9e, 0xef, 0x04, 0x91, 0x42, - 0x41, 0xd8, 0x86, 0x46, 0xde, 0x57, 0xe5, 0xd6, 0x2e, 0x2c, 0xf2, 0x7f, 0x0c, 0x14, 0x7e, 0x35, - 0x03, 0x7a, 0x6f, 0x60, 0x63, 0x34, 0x3c, 0xfd, 0x70, 0xf4, 0x48, 0x9e, 0xa5, 0x6c, 0x23, 0x3c, - 0x74, 0x54, 0xda, 0x9b, 0x13, 0x1d, 0x00, 0xfd, 0xc3, 0x1e, 0x84, 0x5e, 0x6f, 0xfe, 0xd6, 0x37, - 0x7a, 0x9f, 0x4c, 0x95, 0x88, 0x5d, 0x83, 0xcb, 0xba, 0xd3, 0x19, 0xd8, 0xde, 0x1b, 0xf8, 0x51, - 0x42, 0xa0, 0xbe, 0xbb, 0xef, 0xa4, 0x4e, 0xaf, 0x72, 0x6f, 0xe7, 0x2f, 0xfe, 0x70, 0xad, 0xf2, - 0x97, 0x7f, 0xb8, 0x56, 0xf9, 0x4f, 0x7f, 0xb8, 0xf6, 0xc6, 0xef, 0xff, 0x78, 0xad, 0xf2, 0xfd, - 0x47, 0x85, 0xff, 0x62, 0xd0, 0x96, 0x2d, 0xd5, 0xe5, 0xde, 0xc9, 0xcc, 0xdc, 0x3b, 0xf1, 0xe9, - 0xe8, 0x4e, 0x7c, 0x7c, 0xc7, 0x28, 0x93, 0xe3, 0x45, 0xfa, 0x8b, 0x85, 0x8f, 0xff, 0x4f, 0x00, - 0x00, 0x00, 0xff, 0xff, 0x13, 0x7d, 0x98, 0xc8, 0xe1, 0x61, 0x00, 0x00, + // 9373 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0xbd, 0xcd, 0x6f, 0x1c, 0xc7, + 0xb6, 0x18, 0xee, 0x21, 0x39, 0x9c, 0x99, 0x33, 0x9f, 0x2c, 0x7e, 0x8d, 0x3e, 0x2c, 0xc9, 0x63, + 0xcb, 0xa6, 0x65, 0x99, 0x92, 0x69, 0xfb, 0x5e, 0xdb, 0xf7, 0xf9, 0xfa, 0x52, 0xa4, 0x64, 0xf3, + 0x5a, 0x94, 0x78, 0x9b, 0xd4, 0x35, 0x60, 0xe0, 0xf7, 0x6b, 0x34, 0xbb, 0x6b, 0x86, 0x7d, 0xd9, + 0xd3, 0xdd, 0xea, 0xea, 0x91, 0x48, 0xad, 0x92, 0x4d, 0x80, 0xe4, 0x1f, 0xc8, 0x45, 0x80, 0x00, + 0x2f, 0x48, 0x16, 0x09, 0xb2, 0x7b, 0xc1, 0xcb, 0x32, 0xeb, 0x87, 0x20, 0x8b, 0xb7, 0xca, 0x2a, + 0x08, 0x82, 0x7b, 0x81, 0xb7, 0x48, 0x80, 0x20, 0x08, 0x90, 0x20, 0x9b, 0x07, 0x04, 0xe7, 0x9c, + 0xaa, 0xfe, 0x18, 0x8e, 0x24, 0xdb, 0x09, 0xde, 0x26, 0x6f, 0xd7, 0x7d, 0xce, 0xa9, 0xea, 0x9a, + 0xaa, 0x53, 0xe7, 0xbb, 0x6a, 0xa0, 0x13, 0xfb, 0xb1, 0x0c, 0xfc, 0x50, 0x6e, 0xc6, 0x49, 0x94, + 0x46, 0xa2, 0x6e, 0xde, 0x2f, 0x7f, 0x38, 0xf2, 0xd3, 0x93, 0xc9, 0xf1, 0xa6, 0x1b, 0x8d, 0xef, + 0x8c, 0xa2, 0x51, 0x74, 0x87, 0x08, 0x8e, 0x27, 0x43, 0x7a, 0xa3, 0x17, 0x7a, 0xe2, 0x86, 0x97, + 0x21, 0x88, 0xdc, 0x53, 0xf3, 0x1c, 0x07, 0x4e, 0xa8, 0x9f, 0xbb, 0xa9, 0x3f, 0x96, 0x2a, 0x75, + 0xc6, 0xb1, 0x06, 0x34, 0xd2, 0x33, 0x8d, 0x1b, 0xfc, 0xd5, 0x22, 0xd4, 0xf6, 0xa5, 0x52, 0xce, + 0x48, 0x8a, 0x01, 0xcc, 0x2b, 0xdf, 0xeb, 0x57, 0x6e, 0x54, 0x36, 0x3a, 0x5b, 0xbd, 0xcd, 0x6c, + 0x58, 0x87, 0xa9, 0x93, 0x4e, 0x94, 0x85, 0x48, 0xa4, 0x71, 0xc7, 0x5e, 0x7f, 0x6e, 0x9a, 0x66, + 0x5f, 0xa6, 0x27, 0x91, 0x67, 0x21, 0x52, 0xf4, 0x60, 0x5e, 0x26, 0x49, 0x7f, 0xfe, 0x46, 0x65, + 0xa3, 0x65, 0xe1, 0xa3, 0x10, 0xb0, 0xe0, 0x39, 0xa9, 0xd3, 0x5f, 0x20, 0x10, 0x3d, 0x8b, 0x77, + 0xa0, 0x13, 0x27, 0x91, 0x6b, 0xfb, 0xe1, 0x30, 0xb2, 0x09, 0x5b, 0x25, 0x6c, 0x0b, 0xa1, 0x7b, + 0xe1, 0x30, 0xda, 0x45, 0xaa, 0x3e, 0xd4, 0x9c, 0xd0, 0x09, 0xce, 0x95, 0xec, 0x2f, 0x12, 0xda, + 0xbc, 0x8a, 0x0e, 0xcc, 0xf9, 0x5e, 0xbf, 0x76, 0xa3, 0xb2, 0xb1, 0x60, 0xcd, 0xf9, 0x1e, 0x7e, + 0x63, 0x32, 0xf1, 0xbd, 0x7e, 0x9d, 0xbf, 0x81, 0xcf, 0x62, 0x00, 0xad, 0x50, 0x4a, 0xef, 0x51, + 0x94, 0x5a, 0x32, 0x0e, 0xce, 0xfb, 0x8d, 0x1b, 0x95, 0x8d, 0xba, 0x55, 0x82, 0x89, 0xcb, 0x50, + 0xf7, 0xe4, 0xf1, 0x64, 0xb4, 0xaf, 0x46, 0x7d, 0xb8, 0x51, 0xd9, 0x68, 0x58, 0xd9, 0xbb, 0x38, + 0x82, 0xf5, 0x44, 0x3e, 0x9d, 0x48, 0x95, 0x4a, 0xcf, 0x4e, 0xa5, 0x93, 0x78, 0xd1, 0xf3, 0xd0, + 0x1e, 0x47, 0x9e, 0xec, 0x37, 0x69, 0x06, 0xae, 0x16, 0x67, 0x29, 0x91, 0xce, 0xf8, 0x48, 0x13, + 0xed, 0x47, 0x9e, 0xb4, 0x56, 0xb3, 0xc6, 0x45, 0xb0, 0xb0, 0x60, 0xcd, 0x71, 0x5d, 0x19, 0x5f, + 0xec, 0xb4, 0xf5, 0x03, 0x3a, 0x5d, 0x31, 0x6d, 0x4b, 0x7d, 0x7e, 0x05, 0x57, 0xf3, 0x91, 0x1e, + 0x3b, 0xa9, 0x7b, 0x62, 0xbb, 0x89, 0xf4, 0xfc, 0xd4, 0x76, 0xa3, 0x49, 0x98, 0xf6, 0xdb, 0x37, + 0x2a, 0x1b, 0x6d, 0xeb, 0x52, 0x46, 0x73, 0x0f, 0x49, 0x76, 0x88, 0x62, 0x07, 0x09, 0x5e, 0xd1, + 0xc1, 0xf1, 0x79, 0x2a, 0x55, 0xbf, 0x43, 0x13, 0x3d, 0xb3, 0x83, 0x7b, 0x48, 0x20, 0xbe, 0x84, + 0x2b, 0xd9, 0xaf, 0x9a, 0x31, 0x80, 0x2e, 0x0d, 0xa0, 0x6f, 0x48, 0x2e, 0x7c, 0xff, 0xa5, 0xcd, + 0xf9, 0xf3, 0x3d, 0xfa, 0xfc, 0xac, 0xe6, 0xfc, 0xf5, 0x9b, 0xd0, 0xe1, 0x56, 0x0a, 0x07, 0x18, + 0xba, 0xb2, 0xbf, 0x44, 0x2d, 0xda, 0x04, 0x3d, 0xd4, 0x40, 0x71, 0x1b, 0x04, 0x93, 0x39, 0xee, + 0x69, 0x4e, 0x2a, 0x88, 0xb4, 0x47, 0x98, 0x6d, 0xf7, 0x34, 0xa3, 0xde, 0x85, 0xeb, 0x05, 0xea, + 0x8f, 0xed, 0xe8, 0x79, 0x28, 0x13, 0x75, 0xe2, 0xc7, 0x76, 0x22, 0x53, 0xc7, 0x0f, 0xa5, 0xd7, + 0x5f, 0x26, 0x8e, 0xba, 0x92, 0x35, 0xfd, 0xf8, 0xb1, 0xa1, 0xb1, 0x34, 0xc9, 0x17, 0x0b, 0xbf, + 0xff, 0xd3, 0xeb, 0x6f, 0x0c, 0x9e, 0x40, 0x63, 0x27, 0x0a, 0x43, 0xe9, 0xa6, 0x51, 0x22, 0xae, + 0x43, 0xd3, 0x2c, 0xb1, 0xad, 0x77, 0x5c, 0xd5, 0x02, 0x03, 0xda, 0xf3, 0xc4, 0x7b, 0xd0, 0x75, + 0x0d, 0xb5, 0xed, 0x87, 0x9e, 0x3c, 0xa3, 0x2d, 0x57, 0xb5, 0x3a, 0x19, 0x78, 0x0f, 0xa1, 0x83, + 0xff, 0x3a, 0x0f, 0xb5, 0xc3, 0x93, 0xc9, 0x70, 0x18, 0x48, 0xf1, 0x0e, 0xb4, 0xf5, 0xe3, 0x4e, + 0x14, 0xec, 0x79, 0x67, 0xba, 0xdf, 0x32, 0x50, 0xdc, 0x80, 0xa6, 0x06, 0x1c, 0x9d, 0xc7, 0x52, + 0x77, 0x5b, 0x04, 0x95, 0xfb, 0xd9, 0xf7, 0x43, 0xda, 0xc9, 0xf3, 0x56, 0x19, 0x38, 0x45, 0xe5, + 0x9c, 0xd1, 0xe6, 0x2e, 0x53, 0x39, 0xf4, 0xb5, 0xed, 0xc0, 0x7f, 0x26, 0x2d, 0x39, 0xda, 0x09, + 0x53, 0xda, 0xe2, 0x55, 0xab, 0x08, 0x12, 0x5b, 0xb0, 0xaa, 0xb8, 0x89, 0x9d, 0x38, 0xe1, 0x48, + 0x2a, 0x7b, 0xe2, 0x87, 0xe9, 0xcf, 0x3e, 0xe9, 0x2f, 0xde, 0x98, 0xdf, 0x58, 0xb0, 0x96, 0x35, + 0xd2, 0x22, 0xdc, 0x13, 0x42, 0x89, 0xbb, 0xb0, 0x32, 0xd5, 0x86, 0x9b, 0xd4, 0x6e, 0xcc, 0x6f, + 0xcc, 0x5b, 0xa2, 0xd4, 0x64, 0x8f, 0x5a, 0xdc, 0x87, 0xa5, 0x64, 0x12, 0xa2, 0x20, 0x7c, 0xe0, + 0x07, 0xa9, 0x4c, 0x0e, 0x63, 0xe9, 0x92, 0xa8, 0x68, 0x6e, 0xad, 0x6f, 0x92, 0xac, 0xb4, 0xa6, + 0xd1, 0xd6, 0xc5, 0x16, 0xe2, 0x76, 0x36, 0x79, 0xf7, 0xcf, 0xe2, 0x84, 0xe4, 0x49, 0x73, 0x0b, + 0xb8, 0x03, 0x84, 0x58, 0x45, 0xb4, 0xb8, 0x05, 0x4b, 0x5e, 0xe2, 0xf8, 0xa1, 0xed, 0x04, 0x81, + 0x7d, 0x3c, 0x71, 0x4f, 0x65, 0xaa, 0x48, 0xc6, 0xd4, 0xad, 0x2e, 0x21, 0xb6, 0x83, 0xe0, 0x1e, + 0x83, 0xc5, 0xbb, 0xd0, 0x55, 0x69, 0xe2, 0x87, 0x23, 0xfb, 0xc4, 0x51, 0x27, 0xf6, 0xa9, 0x3c, + 0x27, 0x11, 0x53, 0xb7, 0xda, 0x0c, 0xfe, 0xc6, 0x51, 0x27, 0xdf, 0xca, 0xf3, 0xc1, 0xff, 0x9a, + 0x83, 0xfa, 0xae, 0xaf, 0x62, 0x64, 0x38, 0xb1, 0x0e, 0xb5, 0xe1, 0x24, 0x74, 0x73, 0x1e, 0x5a, + 0xc4, 0xd7, 0x3d, 0x4f, 0xfc, 0x09, 0x74, 0x83, 0xc8, 0x75, 0x02, 0x3b, 0x63, 0x97, 0xfe, 0xdc, + 0x8d, 0xf9, 0x8d, 0xe6, 0xd6, 0x72, 0x2e, 0x5b, 0x32, 0x76, 0xb4, 0x3a, 0x44, 0x9b, 0xb3, 0xe7, + 0x97, 0xd0, 0x4b, 0xe4, 0x38, 0x4a, 0x65, 0xa1, 0xf9, 0x3c, 0x35, 0x17, 0x79, 0xf3, 0xef, 0x12, + 0x27, 0x7e, 0x84, 0x02, 0xa9, 0xcb, 0xb4, 0x79, 0xf3, 0x8f, 0x0a, 0x2b, 0x2a, 0x47, 0xb6, 0xef, + 0x9d, 0xd9, 0xf4, 0x81, 0xfe, 0xc2, 0x8d, 0xf9, 0x8d, 0x6a, 0xbe, 0x3c, 0x72, 0xb4, 0xe7, 0x9d, + 0x3d, 0x44, 0x8c, 0xf8, 0x18, 0xd6, 0xa6, 0x9b, 0x70, 0xaf, 0xfd, 0x2a, 0xb5, 0x59, 0x2e, 0xb5, + 0xb1, 0x08, 0x25, 0xde, 0x82, 0x96, 0x69, 0x94, 0x22, 0x2b, 0x2f, 0x32, 0x73, 0xa9, 0x02, 0x2b, + 0xaf, 0x43, 0xcd, 0x57, 0xb6, 0xf2, 0xc3, 0x53, 0xd2, 0x14, 0x75, 0x6b, 0xd1, 0x57, 0x87, 0x7e, + 0x78, 0x2a, 0x2e, 0x41, 0x3d, 0x91, 0x2e, 0x63, 0xea, 0x84, 0xa9, 0x25, 0xd2, 0x25, 0xd4, 0x3a, + 0xe0, 0xa3, 0xed, 0xa6, 0x52, 0xeb, 0x8b, 0xc5, 0x44, 0xba, 0x3b, 0xa9, 0x1c, 0x28, 0xa8, 0xee, + 0xcb, 0x64, 0x24, 0x51, 0x65, 0x60, 0xc3, 0x43, 0xd7, 0x09, 0x69, 0xde, 0xeb, 0x56, 0xf6, 0x8e, + 0x0a, 0x2b, 0x76, 0x92, 0xd4, 0x77, 0x02, 0xda, 0x5a, 0x75, 0xcb, 0xbc, 0x8a, 0x2b, 0xd0, 0x50, + 0xa9, 0x93, 0xa4, 0xf8, 0xeb, 0x68, 0x4b, 0x55, 0xad, 0x3a, 0x01, 0x70, 0x57, 0xae, 0x43, 0x4d, + 0x86, 0x1e, 0xa1, 0x16, 0x78, 0x25, 0x65, 0xe8, 0xed, 0x79, 0x67, 0x83, 0x7f, 0x55, 0x81, 0xf6, + 0xfe, 0x24, 0x48, 0xfd, 0xed, 0x64, 0x34, 0x91, 0xe3, 0x30, 0x45, 0x45, 0xb7, 0xeb, 0xab, 0x54, + 0x7f, 0x99, 0x9e, 0xc5, 0x06, 0x34, 0xbe, 0x4e, 0xa2, 0x49, 0x4c, 0x5c, 0xc9, 0x2b, 0x5d, 0xe4, + 0xca, 0x1c, 0x89, 0x1c, 0xfc, 0x38, 0xf1, 0x64, 0x72, 0xef, 0x9c, 0x68, 0xe7, 0x2f, 0xd0, 0x16, + 0xd1, 0xe2, 0x2a, 0x34, 0x0e, 0x65, 0xec, 0x24, 0x0e, 0xb2, 0xc0, 0x02, 0x69, 0xc7, 0x1c, 0x80, + 0xbf, 0x95, 0x88, 0xf7, 0x3c, 0xbd, 0xb1, 0xcd, 0xeb, 0xe0, 0x9f, 0x56, 0xa0, 0xb1, 0x3d, 0x1a, + 0x25, 0x72, 0xe4, 0xa4, 0xa4, 0xaa, 0xa3, 0x98, 0xc6, 0x3b, 0x6f, 0xcd, 0x45, 0x31, 0x99, 0x03, + 0xf8, 0x0b, 0x78, 0x82, 0xe8, 0x59, 0x5c, 0x83, 0x05, 0x39, 0x7b, 0x40, 0x04, 0x17, 0x6b, 0xb0, + 0xe8, 0x46, 0xe1, 0xd0, 0x1f, 0x69, 0x23, 0x42, 0xbf, 0x89, 0x2f, 0xa0, 0xc9, 0x4f, 0xcc, 0x03, + 0x55, 0xd2, 0xa0, 0x97, 0xb8, 0x79, 0x36, 0x82, 0x1d, 0xa2, 0x40, 0x8e, 0xb0, 0xc0, 0xcd, 0x9e, + 0x07, 0x7f, 0x5c, 0x80, 0x2a, 0xcd, 0x0c, 0xae, 0x0d, 0x1a, 0x05, 0xb6, 0x7c, 0xe6, 0x04, 0x66, + 0x49, 0x11, 0x70, 0xff, 0x99, 0x13, 0x88, 0x1b, 0x50, 0xc5, 0x21, 0xa8, 0x19, 0x13, 0xcb, 0x08, + 0xf1, 0x2e, 0x54, 0xf1, 0xeb, 0xaa, 0x3c, 0x7a, 0xfc, 0xc6, 0xbd, 0x85, 0xbf, 0xf8, 0x8f, 0xd7, + 0xdf, 0xb0, 0x18, 0x2d, 0xde, 0x83, 0x05, 0x67, 0x34, 0x52, 0xb4, 0x11, 0x4a, 0x7b, 0x31, 0x1b, + 0xa9, 0x45, 0x04, 0xe2, 0x53, 0x68, 0xf0, 0xa2, 0x23, 0x75, 0x95, 0xa8, 0xd7, 0x0b, 0xc6, 0x56, + 0x91, 0x1f, 0xac, 0x9c, 0x12, 0x97, 0xcb, 0x57, 0x5a, 0x02, 0xd1, 0x76, 0xa8, 0x5b, 0x39, 0x00, + 0xad, 0xa1, 0x38, 0x91, 0xdb, 0x41, 0x10, 0xb9, 0x87, 0xfe, 0x0b, 0xa9, 0x6d, 0xa7, 0x12, 0x4c, + 0xbc, 0x0b, 0x9d, 0x03, 0xe6, 0x57, 0x4b, 0xaa, 0x49, 0x90, 0x2a, 0x6d, 0x4f, 0x4d, 0x41, 0xc5, + 0x26, 0x88, 0x12, 0xe4, 0x88, 0x7e, 0x7e, 0xe3, 0xc6, 0xfc, 0x46, 0xdb, 0x9a, 0x81, 0x11, 0x6f, + 0x43, 0x7b, 0x84, 0x33, 0x8d, 0x02, 0x6e, 0x18, 0x38, 0x68, 0x6a, 0xcd, 0xa3, 0x29, 0x66, 0x80, + 0x0f, 0x02, 0x67, 0x44, 0x3b, 0x24, 0xf6, 0x83, 0xc0, 0x1e, 0xcb, 0x31, 0x49, 0xbf, 0x79, 0xab, + 0x4e, 0x80, 0x7d, 0x39, 0x16, 0xef, 0xc3, 0x12, 0x11, 0xdb, 0xc7, 0xe7, 0xb9, 0x88, 0x6c, 0x91, + 0x74, 0xe8, 0x10, 0xe2, 0xde, 0xb9, 0x96, 0x91, 0xe2, 0x7d, 0xe8, 0x79, 0xe7, 0xa1, 0x33, 0xf6, + 0x5d, 0xdb, 0xf4, 0x4f, 0x06, 0x10, 0x8a, 0x5d, 0x86, 0x7f, 0xad, 0xc1, 0x28, 0x78, 0xe4, 0x38, + 0x4e, 0xcf, 0x33, 0x42, 0x5b, 0x49, 0xdc, 0xa1, 0x68, 0xf0, 0xa0, 0x2e, 0x59, 0x26, 0xac, 0x21, + 0x3f, 0x94, 0xe9, 0x9e, 0xa7, 0xd0, 0x8a, 0xb8, 0xd8, 0x88, 0x2c, 0x9c, 0xba, 0xd5, 0x9b, 0x6e, + 0x30, 0xf8, 0x97, 0x0b, 0xb0, 0xb8, 0x17, 0x2a, 0x99, 0xa4, 0x28, 0x38, 0x9c, 0xe1, 0x50, 0xba, + 0xa9, 0x64, 0x81, 0xbd, 0x60, 0x65, 0xef, 0xb8, 0x76, 0x47, 0xd1, 0x77, 0x89, 0x9f, 0xca, 0xc3, + 0x8f, 0xf5, 0xce, 0xc8, 0x01, 0xa8, 0x4a, 0x1c, 0xcf, 0xb3, 0x0d, 0xb5, 0x9d, 0x44, 0xcf, 0x15, + 0x09, 0x91, 0xba, 0xd5, 0x75, 0x3c, 0x6f, 0x5b, 0xc3, 0xad, 0xe8, 0xb9, 0x12, 0x6f, 0xc1, 0x7c, + 0x22, 0x87, 0xb4, 0x4f, 0x9a, 0x5b, 0x5d, 0xe6, 0xc5, 0xc7, 0xc7, 0xbf, 0x93, 0x6e, 0x6a, 0xc9, + 0xa1, 0x85, 0x38, 0xb1, 0x02, 0x55, 0x27, 0x4d, 0x13, 0xe6, 0xad, 0x86, 0xc5, 0x2f, 0x62, 0x13, + 0x96, 0x49, 0x58, 0xa5, 0x7e, 0x14, 0xda, 0xa9, 0x73, 0x1c, 0x48, 0x9a, 0x09, 0x56, 0xc4, 0x4b, + 0x19, 0xea, 0x08, 0x31, 0x38, 0x0f, 0x5b, 0xb0, 0x3a, 0x4d, 0x1f, 0x3a, 0x63, 0xa9, 0x48, 0x0f, + 0x37, 0xac, 0xe5, 0x72, 0x8b, 0x47, 0x88, 0x42, 0x46, 0xc8, 0xdb, 0xa0, 0xb8, 0xab, 0x93, 0xe4, + 0x68, 0x65, 0x40, 0x94, 0x86, 0xab, 0xb0, 0xe8, 0x2b, 0x5b, 0x86, 0x9e, 0x96, 0xc0, 0x55, 0x5f, + 0xdd, 0x0f, 0x3d, 0xf1, 0x01, 0x34, 0xf8, 0x2b, 0x9e, 0x1c, 0x92, 0x1e, 0x6d, 0x6e, 0x75, 0xf4, + 0x56, 0x43, 0xf0, 0xae, 0x1c, 0x5a, 0xf5, 0x54, 0x3f, 0xa1, 0x8d, 0x95, 0x46, 0xb6, 0x3c, 0x4b, + 0x65, 0x12, 0x3a, 0x81, 0x56, 0xa6, 0x90, 0x46, 0xf7, 0x35, 0x44, 0x7c, 0x0a, 0xeb, 0x06, 0x6b, + 0xab, 0x74, 0x9c, 0xda, 0x93, 0xd0, 0x3f, 0xb3, 0x43, 0x27, 0x8c, 0xc8, 0x0e, 0x9f, 0xb7, 0x56, + 0x0c, 0xfa, 0x30, 0x1d, 0xa7, 0x4f, 0x42, 0xff, 0xec, 0x91, 0x13, 0x46, 0x62, 0x03, 0x7a, 0x59, + 0xb3, 0xf4, 0x05, 0xfd, 0x60, 0x62, 0xae, 0x86, 0xd5, 0x31, 0xf0, 0xa3, 0x17, 0xf8, 0x5b, 0x89, + 0xb7, 0x0a, 0x94, 0xd1, 0x70, 0x88, 0xbc, 0xa5, 0xa4, 0x4b, 0xc6, 0x74, 0xd5, 0x5a, 0xce, 0xe9, + 0x1f, 0x13, 0xee, 0x50, 0xba, 0x83, 0x3f, 0xaf, 0x40, 0x93, 0x36, 0xf4, 0x93, 0xd8, 0x43, 0xd9, + 0xf9, 0x36, 0xb4, 0xcb, 0x8b, 0xce, 0x7c, 0xd3, 0x72, 0x8a, 0x2b, 0xbe, 0x06, 0x8b, 0xdb, 0x2e, + 0x4e, 0x1e, 0x31, 0x4e, 0xdb, 0xd2, 0x6f, 0xe2, 0xe7, 0xd0, 0x9d, 0x50, 0x37, 0xb6, 0x9b, 0x9e, + 0xd9, 0x01, 0xca, 0x5c, 0x96, 0x50, 0x9a, 0x2b, 0xf8, 0x1b, 0x3b, 0xe9, 0x99, 0xd5, 0x9e, 0x98, + 0xc7, 0x87, 0x28, 0x8d, 0xef, 0xc2, 0x4a, 0x22, 0x91, 0x63, 0xec, 0x17, 0x32, 0x89, 0xec, 0x54, + 0x8e, 0xe3, 0x28, 0x21, 0x0d, 0x8e, 0xb3, 0x28, 0x18, 0xf7, 0xbd, 0x4c, 0xa2, 0x23, 0x8d, 0x19, + 0xbc, 0x09, 0xd5, 0xed, 0x24, 0x71, 0xce, 0x89, 0xb5, 0xf0, 0xa1, 0x5f, 0xa1, 0xbd, 0xc9, 0x2f, + 0x03, 0x17, 0xe6, 0xf7, 0x9d, 0x58, 0xdc, 0x84, 0xb9, 0x71, 0x4c, 0x98, 0xe6, 0xd6, 0x6a, 0x41, + 0xa0, 0x39, 0xf1, 0xe6, 0x7e, 0x7c, 0x3f, 0x4c, 0x93, 0x73, 0x6b, 0x6e, 0x1c, 0x5f, 0xfe, 0x14, + 0x6a, 0xfa, 0x15, 0x9d, 0x49, 0xdc, 0xe8, 0x15, 0x9a, 0x61, 0x7c, 0xc4, 0x0f, 0x3c, 0x73, 0x82, + 0x89, 0x31, 0x5d, 0xf9, 0xe5, 0x8b, 0xb9, 0xcf, 0x2a, 0x83, 0xff, 0xb1, 0x00, 0xf5, 0x5d, 0x19, + 0x48, 0xfa, 0xed, 0x03, 0x68, 0x15, 0x77, 0x85, 0x99, 0xb7, 0xd2, 0x4e, 0x19, 0x40, 0x8b, 0x6d, + 0x09, 0x6a, 0x25, 0xf5, 0xb6, 0x2b, 0xc1, 0x50, 0xc9, 0xed, 0xb1, 0x91, 0x46, 0xfb, 0xad, 0x6d, + 0x99, 0x57, 0xc4, 0x3c, 0xd2, 0x98, 0x05, 0xc6, 0xe8, 0x57, 0x71, 0x15, 0x20, 0x89, 0x9e, 0xdb, + 0x3e, 0x2b, 0x74, 0xd6, 0x8d, 0xf5, 0x24, 0x7a, 0xbe, 0x87, 0x2a, 0xfd, 0x6f, 0x64, 0x9b, 0xfd, + 0x1c, 0xfa, 0x85, 0x6d, 0x86, 0xae, 0x82, 0xed, 0x87, 0xec, 0x58, 0xe9, 0x1d, 0x97, 0xf7, 0x49, + 0x9e, 0xc4, 0x5e, 0x48, 0x3e, 0x95, 0x11, 0x1e, 0x8d, 0x57, 0x08, 0x8f, 0x99, 0xb2, 0x08, 0x66, + 0xcb, 0xa2, 0x7b, 0x00, 0x87, 0x72, 0x34, 0x96, 0x61, 0xba, 0xef, 0xc4, 0xfd, 0x26, 0x2d, 0xfc, + 0x20, 0x5f, 0x78, 0xb3, 0x5a, 0x9b, 0x39, 0x11, 0x73, 0x41, 0xa1, 0x15, 0xda, 0x79, 0xae, 0x13, + 0xda, 0x69, 0x32, 0x09, 0x5d, 0x27, 0x65, 0x2f, 0xb9, 0x6e, 0x35, 0x5d, 0x27, 0x3c, 0xd2, 0xa0, + 0x82, 0xc0, 0x68, 0x17, 0x05, 0xc6, 0xbb, 0xd0, 0x8d, 0x13, 0x7f, 0xec, 0x24, 0xe7, 0xa8, 0x2d, + 0x68, 0x31, 0x78, 0xeb, 0xb5, 0x35, 0xf8, 0x5b, 0x79, 0xbe, 0xe7, 0x9d, 0x5d, 0xfe, 0x12, 0xba, + 0x53, 0x03, 0xf8, 0x51, 0x7c, 0xf7, 0xfb, 0x45, 0x68, 0x1c, 0x24, 0x52, 0x0b, 0xf9, 0xeb, 0xd0, + 0x54, 0xee, 0x89, 0x1c, 0x3b, 0x2c, 0x1b, 0xb8, 0x07, 0x60, 0x10, 0xc9, 0x85, 0x92, 0x18, 0x9b, + 0x7b, 0x8d, 0x18, 0xeb, 0xc1, 0x3c, 0xdb, 0x8b, 0xb8, 0x99, 0xf0, 0x31, 0x97, 0xdd, 0x0b, 0x45, + 0xd9, 0x7d, 0x03, 0x5a, 0x27, 0x8e, 0xb2, 0x9d, 0x49, 0x1a, 0xd9, 0x6e, 0x14, 0x10, 0xd3, 0xd5, + 0x2d, 0x38, 0x71, 0xd4, 0xf6, 0x24, 0x8d, 0x76, 0xa2, 0x40, 0xbc, 0x09, 0xe0, 0x46, 0x81, 0x16, + 0x43, 0xda, 0x58, 0x6e, 0xb8, 0x51, 0xc0, 0xb2, 0x07, 0xb9, 0x52, 0xaa, 0xd4, 0x1f, 0x3b, 0x7a, + 0x49, 0xb5, 0xdf, 0x5e, 0x23, 0x51, 0xb8, 0x94, 0xa1, 0xac, 0xe8, 0x39, 0x3b, 0xec, 0x77, 0xa1, + 0xe3, 0x46, 0xe3, 0xd8, 0x8e, 0x71, 0x66, 0xc9, 0x74, 0xab, 0x5f, 0xf0, 0x86, 0x5a, 0x48, 0x71, + 0x70, 0x2a, 0xd9, 0x98, 0xdc, 0x82, 0xae, 0x1b, 0x4c, 0x54, 0x2a, 0x13, 0xd4, 0xe1, 0x72, 0xb6, + 0x03, 0xd5, 0xd6, 0x24, 0xda, 0x00, 0x1d, 0x40, 0xdb, 0x57, 0x76, 0x14, 0x78, 0x36, 0x0b, 0x28, + 0xcd, 0x67, 0x4d, 0x5f, 0x3d, 0x0e, 0x3c, 0x2d, 0x22, 0x99, 0x26, 0x94, 0xcf, 0x0d, 0x4d, 0xd3, + 0xd0, 0x3c, 0x92, 0xcf, 0x35, 0xcd, 0xcb, 0x04, 0x5a, 0xeb, 0x65, 0x02, 0x0d, 0xe7, 0x03, 0x27, + 0x34, 0x75, 0x92, 0x11, 0x49, 0xed, 0x80, 0xfd, 0x20, 0xe6, 0xaf, 0xa5, 0x13, 0x47, 0x1d, 0x11, + 0xe6, 0x50, 0x23, 0xd0, 0xeb, 0xd1, 0xb4, 0x38, 0x79, 0xe1, 0x64, 0x7c, 0x2c, 0x13, 0x5a, 0x09, + 0xe6, 0x38, 0xc1, 0x48, 0x2b, 0x7a, 0xfe, 0x88, 0x50, 0xb8, 0x22, 0xb7, 0x60, 0x49, 0x37, 0x71, + 0xdc, 0xd4, 0x7f, 0x26, 0x89, 0xbc, 0x4b, 0xe4, 0x5d, 0x46, 0x6c, 0x13, 0x1c, 0x69, 0xdf, 0xcf, + 0x68, 0xb5, 0x64, 0x41, 0xda, 0x1e, 0xc7, 0x04, 0xb2, 0xae, 0xf7, 0x3c, 0x24, 0xdd, 0x81, 0x6b, + 0x69, 0xe2, 0xb8, 0xa7, 0xcc, 0x0c, 0x7e, 0xe8, 0x26, 0x12, 0x59, 0xdb, 0x1e, 0xc9, 0x50, 0x26, + 0xb8, 0x86, 0x14, 0x1b, 0xa9, 0x5b, 0x57, 0x88, 0x0a, 0xd9, 0x63, 0xcf, 0xd0, 0x7c, 0x6d, 0x48, + 0xc4, 0x7d, 0xb8, 0xfe, 0xb2, 0xe6, 0xf8, 0xe9, 0xc9, 0x38, 0xa4, 0xb0, 0x49, 0xd5, 0xba, 0xea, + 0xcc, 0xec, 0x60, 0x87, 0x68, 0x06, 0xff, 0x7e, 0x0e, 0x6a, 0x07, 0x91, 0x4a, 0x77, 0xc7, 0x81, + 0x11, 0x2d, 0x95, 0x1f, 0x2b, 0x5a, 0xe6, 0x66, 0x8b, 0x96, 0x19, 0x9b, 0x7b, 0x7e, 0xc6, 0xe6, + 0x46, 0x85, 0x5d, 0xa4, 0xa3, 0x4d, 0xc9, 0xae, 0x4c, 0x27, 0x27, 0xa4, 0x8d, 0x79, 0x05, 0xcd, + 0x67, 0xdb, 0x63, 0x5d, 0xc0, 0x1b, 0xa8, 0xee, 0x2b, 0xad, 0x07, 0x18, 0xe9, 0xd3, 0x1e, 0xd7, + 0xb6, 0x75, 0xdd, 0x57, 0x7a, 0xcf, 0x7f, 0x0e, 0x97, 0xb2, 0x96, 0xf6, 0x73, 0x3f, 0x3d, 0x89, + 0x26, 0xa9, 0x3d, 0xa4, 0xb8, 0x81, 0xd2, 0x9e, 0xe7, 0x9a, 0xe9, 0xe9, 0x3b, 0x46, 0x73, 0x54, + 0x81, 0x4c, 0xfd, 0xe1, 0x24, 0x08, 0xec, 0x54, 0x9e, 0xa5, 0x7a, 0x0b, 0xf5, 0x79, 0x6e, 0xf4, + 0xbc, 0x3d, 0x98, 0x04, 0xc1, 0x91, 0x3c, 0x4b, 0x51, 0x4d, 0xd7, 0x87, 0xfa, 0x65, 0xf0, 0x0f, + 0x17, 0x00, 0x1e, 0x46, 0xee, 0x29, 0x73, 0x21, 0xfa, 0xb3, 0x46, 0x93, 0x68, 0x4d, 0x57, 0x4b, + 0x59, 0x7f, 0x88, 0x2d, 0x58, 0x33, 0xbf, 0x1f, 0xf7, 0x3f, 0xfa, 0xd6, 0xac, 0x0a, 0xb4, 0x20, + 0x13, 0x1a, 0xcb, 0xf1, 0x21, 0xd2, 0x03, 0xe2, 0xb3, 0x7c, 0x6e, 0xb1, 0x4d, 0x7a, 0x1e, 0xd3, + 0xdc, 0xce, 0x72, 0x6d, 0xda, 0x79, 0xf3, 0xa3, 0xf3, 0x58, 0xdc, 0x85, 0xd5, 0x44, 0x0e, 0x13, + 0xa9, 0x4e, 0xec, 0x54, 0x15, 0x3f, 0xc6, 0x6e, 0xed, 0x92, 0x46, 0x1e, 0xa9, 0xec, 0x5b, 0x77, + 0x61, 0x95, 0x67, 0x6a, 0x7a, 0x78, 0xac, 0x37, 0x97, 0x18, 0x59, 0x1c, 0xdd, 0x9b, 0x40, 0xa1, + 0x6e, 0xd6, 0x85, 0xc6, 0xcf, 0x09, 0x68, 0x32, 0x8e, 0x03, 0x89, 0x96, 0xf4, 0xce, 0x89, 0x13, + 0x8e, 0x50, 0x7e, 0xea, 0xc9, 0xcf, 0x01, 0x62, 0x00, 0x0b, 0xfb, 0x91, 0x27, 0x69, 0xaa, 0x3b, + 0x5b, 0x9d, 0x4d, 0x0a, 0x9a, 0xe3, 0x4c, 0x52, 0x74, 0x95, 0x70, 0xe2, 0x3d, 0xa0, 0xee, 0x98, + 0xfd, 0x2e, 0xca, 0xa8, 0x3a, 0x22, 0x89, 0x07, 0xef, 0xc2, 0x6a, 0x3e, 0x12, 0xdb, 0x49, 0xed, + 0xf4, 0x44, 0x92, 0x1a, 0x62, 0x31, 0xb5, 0x94, 0x0d, 0x6a, 0x3b, 0x3d, 0x3a, 0x91, 0xa8, 0x92, + 0x36, 0xa0, 0x16, 0x1d, 0xff, 0xce, 0xc6, 0x8d, 0xd0, 0x9c, 0xbd, 0x11, 0x16, 0xa3, 0xe3, 0xdf, + 0x59, 0x72, 0x28, 0x7e, 0x56, 0x54, 0xe1, 0x53, 0x53, 0xd3, 0xa2, 0xa9, 0x59, 0xc9, 0xf0, 0x85, + 0xd9, 0x19, 0x7c, 0x06, 0x8b, 0xf8, 0x73, 0x1e, 0xc7, 0x62, 0x13, 0x6a, 0x2c, 0x1a, 0x94, 0x36, + 0xb9, 0x56, 0x72, 0xcd, 0x9b, 0xf3, 0x8e, 0x65, 0x88, 0x06, 0x16, 0x74, 0x33, 0x35, 0xf6, 0x24, + 0xf4, 0x9f, 0x4e, 0xa4, 0xf8, 0x0a, 0x96, 0xe2, 0x44, 0x6a, 0xb6, 0xb7, 0x27, 0xa7, 0x68, 0x48, + 0xea, 0x1d, 0xbc, 0xa2, 0xb9, 0x34, 0x6b, 0x71, 0x8a, 0x1c, 0xda, 0x89, 0x4b, 0xef, 0x83, 0xef, + 0x61, 0x3d, 0xa3, 0x38, 0x94, 0x6e, 0x14, 0x7a, 0x4e, 0x72, 0x4e, 0x16, 0xc7, 0x54, 0xdf, 0xea, + 0xc7, 0xf4, 0x7d, 0x48, 0x7d, 0xff, 0xb7, 0x0a, 0x34, 0x1f, 0x4c, 0x5e, 0xbc, 0x38, 0xe7, 0xbd, + 0x24, 0x5a, 0x50, 0x79, 0x44, 0x1d, 0xcc, 0x59, 0x95, 0x47, 0x68, 0x14, 0x1f, 0x9c, 0xe2, 0xbe, + 0x26, 0x3e, 0x6f, 0x58, 0xfa, 0x0d, 0x9d, 0xf5, 0x83, 0xd3, 0xa3, 0x57, 0x70, 0x34, 0xa3, 0xd1, + 0x59, 0xbb, 0x37, 0xf1, 0x03, 0x34, 0xd9, 0x34, 0xf3, 0x66, 0xef, 0xe8, 0xfe, 0xee, 0x0d, 0x79, + 0x28, 0x0f, 0x92, 0x68, 0xcc, 0x93, 0xa5, 0x45, 0xc6, 0x0c, 0x8c, 0xf8, 0x1a, 0x96, 0x75, 0x30, + 0x51, 0x4b, 0x05, 0x5b, 0xc5, 0xd2, 0x25, 0xd6, 0xfd, 0x51, 0x01, 0xc8, 0xc1, 0xdf, 0x5d, 0x84, + 0x3a, 0xba, 0xb9, 0xbf, 0x8e, 0xfc, 0x50, 0xdc, 0x85, 0xc6, 0xef, 0x22, 0x3f, 0xe4, 0xc8, 0x07, + 0xa7, 0x6d, 0x96, 0xb9, 0xaf, 0x47, 0x91, 0x27, 0x37, 0x91, 0x86, 0x62, 0x1e, 0xf5, 0xdf, 0xe9, + 0x27, 0xad, 0x2a, 0x13, 0x7f, 0x74, 0x92, 0xda, 0x08, 0xd4, 0xb2, 0xb5, 0xe9, 0x2b, 0x0b, 0x61, + 0xd4, 0xeb, 0x55, 0x00, 0xf2, 0xaf, 0xa3, 0xd0, 0x8e, 0x4f, 0xb5, 0x8f, 0x59, 0x47, 0xc8, 0xe3, + 0xf0, 0xe0, 0x14, 0xf7, 0x9e, 0xaf, 0x6c, 0x1d, 0x63, 0xd3, 0xfe, 0x40, 0x21, 0xc6, 0xf0, 0x0e, + 0x74, 0xd0, 0x56, 0x53, 0xa7, 0x7e, 0x6c, 0xc7, 0x49, 0x74, 0x6c, 0x26, 0x05, 0x2d, 0xb8, 0xc3, + 0x53, 0x3f, 0x3e, 0x40, 0x18, 0x99, 0x48, 0x3a, 0x72, 0x87, 0x62, 0x9b, 0x6d, 0x11, 0xd0, 0x20, + 0x9c, 0x5f, 0x0a, 0xcf, 0x05, 0xec, 0xb1, 0xd4, 0xc8, 0xf4, 0xa9, 0x25, 0x32, 0x20, 0xd7, 0xe4, + 0x12, 0xd4, 0x71, 0x33, 0x10, 0xaa, 0xce, 0x28, 0x37, 0x62, 0xd4, 0xfb, 0x00, 0x81, 0x1c, 0xa6, + 0x36, 0x72, 0x19, 0x07, 0x23, 0xa6, 0xc2, 0x60, 0x88, 0xdd, 0x41, 0xa4, 0xf8, 0x00, 0x9a, 0x3c, + 0x0b, 0x4c, 0x0b, 0x17, 0x68, 0x81, 0xd0, 0x4c, 0x7c, 0x0b, 0x9a, 0x61, 0x14, 0xda, 0xf2, 0x29, + 0x51, 0xeb, 0x7d, 0x5b, 0xea, 0x38, 0x8c, 0xc2, 0xfb, 0x4f, 0x91, 0x58, 0xdc, 0xd1, 0x63, 0xe0, + 0x78, 0x50, 0xeb, 0x25, 0xf1, 0x20, 0x1a, 0x09, 0x47, 0x46, 0x3e, 0x32, 0x23, 0xe1, 0x16, 0xed, + 0x97, 0xb4, 0xe0, 0xf1, 0x70, 0x93, 0x1b, 0xd0, 0xa2, 0x75, 0x1f, 0x3b, 0xb1, 0x9d, 0x3a, 0x23, + 0x6d, 0x61, 0x00, 0xc2, 0xf6, 0x9d, 0xf8, 0xc8, 0x19, 0x09, 0x0b, 0x2e, 0x4d, 0xf1, 0xdb, 0x31, + 0xb2, 0x2e, 0xcf, 0x5a, 0xd7, 0xc4, 0x93, 0x66, 0x73, 0xdd, 0x5a, 0x89, 0xeb, 0x88, 0xe5, 0x69, + 0x76, 0x3f, 0x87, 0x4b, 0x72, 0x4c, 0xf9, 0x9c, 0x71, 0x9c, 0x48, 0xa5, 0x4a, 0x66, 0x62, 0x8f, + 0x75, 0x1c, 0x12, 0xec, 0x64, 0xf8, 0xcc, 0x56, 0x7c, 0x07, 0x3a, 0x8e, 0x8a, 0x86, 0xb6, 0x99, + 0xf2, 0x80, 0x2c, 0x90, 0xaa, 0xd5, 0x42, 0xa8, 0xc5, 0x13, 0x1d, 0xa0, 0x42, 0x27, 0x2a, 0x3d, + 0x54, 0x39, 0x4c, 0xc9, 0xc4, 0xa8, 0x5b, 0x6d, 0x04, 0xf3, 0x40, 0xe4, 0x30, 0x1d, 0xfc, 0xb3, + 0x39, 0xa8, 0x3f, 0x8c, 0xa2, 0xf8, 0x27, 0xee, 0x81, 0x22, 0x6f, 0xcd, 0xbd, 0x9c, 0xb7, 0xe6, + 0xcb, 0xbc, 0x35, 0xc5, 0x03, 0x0b, 0x3f, 0x9c, 0x07, 0xaa, 0x3f, 0x9a, 0x07, 0x16, 0x7f, 0x02, + 0x0f, 0xd4, 0xa6, 0x79, 0x60, 0x70, 0x17, 0xaa, 0x87, 0x32, 0x7d, 0x1c, 0xa3, 0x36, 0x33, 0x36, + 0xba, 0x51, 0x04, 0x25, 0x6d, 0xa6, 0xed, 0x73, 0x35, 0xf8, 0xeb, 0x26, 0x34, 0x76, 0xa5, 0x37, + 0xe1, 0x99, 0x2d, 0xce, 0x53, 0xe5, 0xe5, 0xf3, 0x34, 0x57, 0x9e, 0x27, 0x54, 0x9d, 0x66, 0x0f, + 0xce, 0x08, 0xe6, 0xd6, 0xcd, 0x16, 0xc4, 0xcd, 0x9a, 0xef, 0x40, 0x1d, 0x11, 0x2d, 0xcd, 0x67, + 0xb6, 0x01, 0x5f, 0xcd, 0xcd, 0xd5, 0x9f, 0xc6, 0xcd, 0x65, 0x39, 0x76, 0x21, 0x56, 0xfa, 0xda, + 0xe9, 0x9d, 0x96, 0x61, 0xf5, 0x0b, 0x32, 0xec, 0x21, 0x2c, 0x47, 0xa1, 0xed, 0x4d, 0xe2, 0xc0, + 0x47, 0x1f, 0x95, 0x6c, 0xfc, 0x28, 0x24, 0x73, 0x82, 0x72, 0xbc, 0x19, 0x8f, 0x3e, 0x0e, 0x77, + 0x0d, 0x11, 0xc7, 0x6d, 0xac, 0xa5, 0x68, 0x1a, 0x84, 0x5b, 0xc8, 0xc3, 0xa5, 0x21, 0x4b, 0x80, + 0x6c, 0x58, 0x4e, 0x56, 0xb7, 0x08, 0xba, 0x13, 0x05, 0xa4, 0xdb, 0x3e, 0x83, 0x6e, 0x4e, 0xc5, + 0xcc, 0xd4, 0x7c, 0x09, 0x33, 0xb5, 0x4d, 0x43, 0xe6, 0xa7, 0xbf, 0x09, 0xb9, 0xf5, 0x21, 0x2c, + 0x9b, 0x70, 0x94, 0x36, 0x67, 0x68, 0x05, 0x3b, 0xc4, 0x41, 0x3d, 0x1d, 0x81, 0x22, 0x4b, 0x86, + 0x96, 0xe8, 0x17, 0xb0, 0x52, 0x20, 0x47, 0xf6, 0x2d, 0xca, 0xaf, 0x22, 0xaf, 0x2c, 0x65, 0x6d, + 0xf1, 0xf5, 0x21, 0xe7, 0x13, 0x9a, 0x9e, 0x0c, 0xcc, 0x87, 0xb4, 0xa7, 0xd4, 0xf0, 0x64, 0xa0, + 0xd3, 0xa0, 0xfb, 0xf0, 0x0e, 0x7a, 0x95, 0x88, 0x77, 0x9d, 0x38, 0x9d, 0x24, 0xd2, 0x8e, 0x03, + 0xc7, 0x95, 0x27, 0x51, 0xe0, 0xc9, 0x24, 0x1f, 0xdc, 0x12, 0x0d, 0xee, 0x7a, 0x14, 0xa0, 0x47, + 0xb3, 0xc3, 0x94, 0x07, 0x39, 0xa1, 0x19, 0xeb, 0x36, 0x5c, 0xbb, 0xd0, 0x1d, 0xaa, 0xba, 0xbc, + 0x23, 0x41, 0x1d, 0x5d, 0x2a, 0x77, 0x84, 0x24, 0xa6, 0x8b, 0x8f, 0x60, 0x95, 0xd7, 0x8e, 0x99, + 0xfb, 0x54, 0xca, 0xd8, 0x0e, 0x1c, 0x95, 0xea, 0x1c, 0xb3, 0x20, 0x24, 0x31, 0xf0, 0xb7, 0x52, + 0xc6, 0x0f, 0x1d, 0xfe, 0x2a, 0x37, 0xd1, 0x9e, 0x07, 0xb5, 0x29, 0xcd, 0xed, 0x0a, 0x7f, 0x95, + 0xa8, 0xd8, 0xfd, 0xc0, 0xc6, 0x85, 0x49, 0xfe, 0x13, 0xb8, 0x52, 0xea, 0x62, 0xec, 0x24, 0xa7, + 0xb9, 0x29, 0xde, 0x5f, 0xa5, 0x79, 0x5b, 0x2f, 0xb4, 0xdf, 0x27, 0x02, 0x3d, 0x8b, 0x1f, 0xc3, + 0x1a, 0x3a, 0xc9, 0x91, 0x77, 0x3a, 0x99, 0x72, 0xda, 0xd6, 0x68, 0xd0, 0xe8, 0x42, 0x3f, 0xf6, + 0x4e, 0x27, 0x25, 0xc7, 0xed, 0xe7, 0xd0, 0x2f, 0xd1, 0xda, 0x09, 0x25, 0x0a, 0xec, 0x38, 0x52, + 0xfd, 0x75, 0x8e, 0x4d, 0x15, 0xa3, 0x9b, 0x9c, 0x46, 0x38, 0x88, 0x94, 0x78, 0x00, 0x37, 0xe2, + 0x93, 0x73, 0xe5, 0x53, 0x62, 0x93, 0x0c, 0xfa, 0x8b, 0x1d, 0xf4, 0xd9, 0x29, 0x35, 0x74, 0x6c, + 0xf7, 0x4f, 0xf5, 0xf3, 0x19, 0x5c, 0x32, 0x8c, 0x75, 0x22, 0xdd, 0xd3, 0xf2, 0x8c, 0x5d, 0xa2, + 0x19, 0x5b, 0xd5, 0x1c, 0x85, 0xf8, 0xc2, 0x6c, 0x6d, 0x40, 0x8f, 0xf4, 0x9d, 0x3d, 0x8c, 0x26, + 0xa1, 0xfe, 0xa5, 0x97, 0xe9, 0x97, 0x76, 0x08, 0xfe, 0x00, 0xc1, 0xf4, 0x23, 0x37, 0xa0, 0x47, + 0xda, 0x92, 0x77, 0x3d, 0x53, 0x5e, 0x61, 0x4a, 0x84, 0xeb, 0x8d, 0x8e, 0x94, 0x9f, 0xc2, 0xba, + 0x26, 0x1a, 0xfa, 0xa1, 0x13, 0x14, 0x7f, 0xcc, 0x55, 0x36, 0xf3, 0x19, 0xfd, 0x00, 0xb1, 0xf9, + 0x8f, 0xf8, 0x0d, 0x88, 0x61, 0x94, 0x48, 0x7f, 0x14, 0x92, 0x5b, 0x4b, 0xbf, 0x44, 0xf5, 0xdf, + 0xa4, 0xbd, 0xf1, 0x66, 0x6e, 0xe7, 0x3f, 0xde, 0xfd, 0xf6, 0xc9, 0x03, 0xa6, 0xfb, 0x56, 0x9e, + 0xd3, 0xef, 0xd1, 0x3b, 0xb3, 0x37, 0x2c, 0x83, 0xd5, 0xe0, 0x1f, 0x2d, 0x42, 0x87, 0xb4, 0xf1, + 0xdf, 0x2a, 0x81, 0xbf, 0x55, 0x02, 0xff, 0x2f, 0x28, 0x81, 0x5b, 0xb0, 0xe4, 0x87, 0xf1, 0x24, + 0xc5, 0x1d, 0xa4, 0xec, 0x09, 0x7b, 0x71, 0x1c, 0x1c, 0xeb, 0x12, 0xe2, 0x5b, 0x79, 0xae, 0xd8, + 0x85, 0x1b, 0xfc, 0x9d, 0x0a, 0xd4, 0x0e, 0x92, 0xc8, 0x9b, 0xb8, 0xe9, 0x4f, 0xdc, 0x15, 0x65, + 0x6e, 0x9b, 0x7f, 0x1d, 0xb7, 0x2d, 0x5c, 0xb0, 0xe8, 0xfe, 0x45, 0x05, 0x1a, 0x7a, 0x08, 0x0f, + 0xb7, 0x7e, 0xe2, 0x20, 0xf2, 0x3c, 0x7b, 0x65, 0x66, 0x9e, 0xfd, 0xb5, 0xa3, 0x40, 0x26, 0x7c, + 0xc6, 0x85, 0x49, 0x51, 0x9c, 0x27, 0xdd, 0x1b, 0x56, 0x8b, 0xa1, 0x8f, 0x63, 0xca, 0xad, 0x3f, + 0x87, 0x06, 0x79, 0xf9, 0x24, 0x45, 0xd6, 0x60, 0x91, 0xa5, 0x9a, 0x1e, 0xa8, 0x7e, 0x7b, 0xf5, + 0x9e, 0x9e, 0xfb, 0x49, 0x7b, 0x7a, 0xf0, 0x6f, 0xe7, 0xa1, 0x4d, 0x21, 0x97, 0x07, 0x93, 0x90, + 0x77, 0x4d, 0x16, 0x30, 0xaf, 0x94, 0x03, 0xe6, 0x0b, 0x89, 0x4c, 0x4d, 0x52, 0xbf, 0xc5, 0x9f, + 0xd9, 0x89, 0x82, 0x5d, 0x39, 0xb4, 0x08, 0x83, 0x53, 0xe5, 0x24, 0x23, 0x35, 0xab, 0x24, 0x01, + 0xe1, 0xf8, 0xab, 0x62, 0x27, 0x71, 0xc6, 0xca, 0x94, 0x24, 0xf0, 0x9b, 0x10, 0xb0, 0x40, 0x7b, + 0x93, 0xa7, 0x85, 0x9e, 0x75, 0xf4, 0x50, 0xf9, 0xe1, 0x28, 0x13, 0x34, 0x75, 0x2a, 0x45, 0x19, + 0x05, 0x52, 0xec, 0x82, 0xe0, 0x14, 0x4d, 0x22, 0x1d, 0x34, 0x3e, 0xa8, 0x1f, 0x92, 0x36, 0xcd, + 0xad, 0x35, 0xfe, 0x2c, 0xcd, 0xa5, 0x45, 0xe8, 0x03, 0xc4, 0x5a, 0x3d, 0x7f, 0x0a, 0x32, 0x63, + 0x32, 0xd9, 0x02, 0xc9, 0x3c, 0xe5, 0x1f, 0x3c, 0x99, 0x64, 0x96, 0x10, 0xb7, 0x7c, 0x05, 0xcb, + 0xc3, 0x49, 0x10, 0xa4, 0xf2, 0x2c, 0xb5, 0x55, 0x34, 0x49, 0x5c, 0x69, 0xbf, 0x22, 0x3b, 0xb4, + 0x64, 0x68, 0x0f, 0x89, 0xd4, 0x92, 0x43, 0xf1, 0x25, 0x88, 0xac, 0x03, 0xf3, 0x1b, 0x4d, 0xee, + 0xf6, 0x42, 0xfb, 0x9e, 0x21, 0xd5, 0xbf, 0x76, 0x38, 0xd8, 0x86, 0x55, 0x93, 0xaf, 0x45, 0xd1, + 0xb6, 0x85, 0xfb, 0x96, 0x62, 0x47, 0x66, 0x8e, 0x2b, 0x85, 0x39, 0x5e, 0x81, 0x6a, 0xb1, 0x54, + 0x8e, 0x5f, 0x06, 0x37, 0xa1, 0x39, 0xf4, 0x03, 0xa9, 0xf3, 0x1e, 0xb8, 0x68, 0x3a, 0x03, 0x52, + 0xa1, 0x04, 0xbf, 0x7e, 0x1b, 0xfc, 0x79, 0x05, 0xd6, 0x63, 0x27, 0x79, 0x3a, 0xd1, 0x11, 0x76, + 0xae, 0x35, 0x50, 0x27, 0x4e, 0xe2, 0xe1, 0xc6, 0xa5, 0x2e, 0xb8, 0x77, 0xae, 0xb4, 0x6a, 0x20, + 0x84, 0xc7, 0xf2, 0x2e, 0x74, 0x0b, 0x2d, 0x52, 0x27, 0x31, 0x91, 0xd5, 0x76, 0x12, 0x3d, 0xa7, + 0x4a, 0x80, 0x43, 0x04, 0x8a, 0x01, 0xb4, 0x73, 0x3a, 0x49, 0x9a, 0x91, 0x0a, 0x96, 0x0c, 0xd5, + 0xfd, 0xd0, 0xc3, 0x9d, 0x1b, 0x4e, 0xc6, 0x6c, 0x2e, 0x70, 0x41, 0x5d, 0x2d, 0x9c, 0x8c, 0xc9, + 0x4e, 0x58, 0x81, 0x2a, 0xd7, 0x42, 0x56, 0x09, 0xce, 0x2f, 0x83, 0xbf, 0xac, 0xc2, 0xf2, 0x9e, + 0x2b, 0x8f, 0x65, 0x32, 0xda, 0x75, 0x52, 0xe7, 0x81, 0x1f, 0xc8, 0x23, 0x47, 0x9d, 0x22, 0xc3, + 0xd1, 0x98, 0x63, 0x27, 0x3d, 0xd1, 0xb3, 0x54, 0x47, 0xc0, 0x81, 0x93, 0x9e, 0xa0, 0xda, 0x22, + 0xe4, 0x30, 0x4a, 0xc6, 0x3a, 0x0e, 0xdc, 0xb0, 0xe8, 0x37, 0x3e, 0x20, 0x48, 0xd6, 0x5a, 0xf9, + 0x2f, 0xa4, 0x2e, 0xff, 0xa3, 0xd6, 0x54, 0x23, 0xf2, 0x16, 0xb4, 0x12, 0xe9, 0x46, 0x89, 0xa7, + 0x7d, 0x7f, 0x1e, 0x67, 0x93, 0x61, 0xec, 0xf0, 0xdf, 0x82, 0x3c, 0x8f, 0x49, 0xa1, 0x2e, 0xdb, + 0x37, 0x35, 0x42, 0xdd, 0x0c, 0x81, 0x9c, 0xb7, 0xe7, 0x89, 0xff, 0x0f, 0x7a, 0x39, 0x2d, 0x25, + 0xd5, 0x8c, 0x07, 0xbc, 0x95, 0x9b, 0x31, 0x33, 0x7e, 0xe2, 0xe6, 0x81, 0x69, 0xf5, 0x5b, 0x6a, + 0xc4, 0x89, 0xc3, 0xbc, 0x7b, 0x86, 0x8a, 0xb7, 0xa1, 0xad, 0xe2, 0xc0, 0x4f, 0x35, 0x03, 0x28, + 0x5d, 0x24, 0xd8, 0x22, 0x20, 0xe7, 0xbe, 0xd4, 0xac, 0x25, 0xac, 0xff, 0xa0, 0x25, 0x6c, 0x5c, + 0x5c, 0xc2, 0xf7, 0xa1, 0xe7, 0x26, 0xd2, 0x93, 0x61, 0xea, 0x3b, 0x81, 0xad, 0xdc, 0x28, 0x36, + 0x6a, 0xba, 0x9b, 0xc3, 0x0f, 0x11, 0x2c, 0x7e, 0x06, 0xeb, 0x6e, 0x14, 0xa6, 0x32, 0x4c, 0xb3, + 0x62, 0x54, 0x9d, 0x39, 0xd2, 0xe5, 0x2f, 0xab, 0x1a, 0x6d, 0x4a, 0x52, 0x39, 0x77, 0x24, 0xee, + 0xc2, 0x0a, 0x2f, 0xcf, 0x54, 0x23, 0xae, 0x5b, 0x10, 0xb4, 0x52, 0xe5, 0x16, 0x3a, 0x9b, 0x95, + 0x48, 0xe5, 0x7b, 0x13, 0x27, 0xd0, 0x12, 0xa2, 0x90, 0xcd, 0xb2, 0x34, 0x46, 0x87, 0x52, 0x29, + 0x5f, 0x56, 0xa2, 0xa5, 0xa2, 0x1b, 0x0a, 0x35, 0x35, 0x2c, 0x91, 0x94, 0xa8, 0xbf, 0x71, 0xd4, + 0xc9, 0xe5, 0x7b, 0xb0, 0x32, 0x6b, 0x41, 0x5e, 0x97, 0x48, 0x6d, 0x14, 0x12, 0xa9, 0xba, 0x54, + 0xf6, 0xbf, 0xcf, 0xc1, 0xaa, 0x59, 0x6f, 0x72, 0x39, 0x32, 0xa6, 0xbe, 0x4e, 0xfa, 0x1c, 0xdd, + 0x94, 0x2c, 0xdc, 0xd3, 0xb0, 0x80, 0x41, 0x14, 0xdb, 0xd9, 0x80, 0x9e, 0x26, 0xc8, 0x99, 0x9f, + 0xbf, 0xd2, 0xf1, 0xb2, 0xae, 0x68, 0x0b, 0xd0, 0x0f, 0x1c, 0xca, 0x04, 0xe7, 0xc8, 0xa3, 0xfa, + 0x73, 0x6a, 0x42, 0xcc, 0x4e, 0x3f, 0xd0, 0xe0, 0x0c, 0xcb, 0x51, 0xd5, 0xcf, 0xd3, 0x89, 0x13, + 0xf8, 0xe9, 0xb9, 0x3d, 0xf4, 0x65, 0xe0, 0x51, 0xd6, 0x9e, 0x6b, 0x1a, 0x7b, 0x06, 0xf3, 0x00, + 0x11, 0x7b, 0x9e, 0x2a, 0x8c, 0x44, 0x27, 0x83, 0xb3, 0x0d, 0xa0, 0x47, 0x72, 0x48, 0xe0, 0x3d, + 0x6f, 0xf6, 0x5e, 0x59, 0x9c, 0xbd, 0x57, 0xde, 0x83, 0xee, 0xf4, 0x9a, 0x73, 0x82, 0xb6, 0xa3, + 0xca, 0xeb, 0x3d, 0x8b, 0x09, 0xeb, 0x33, 0x99, 0x50, 0x4f, 0xfa, 0xff, 0x9c, 0x83, 0x15, 0x3d, + 0xe9, 0x9c, 0xba, 0xdb, 0x77, 0x62, 0xaa, 0x90, 0xba, 0x01, 0xad, 0x71, 0xc4, 0x26, 0x54, 0x41, + 0xfc, 0xc1, 0x38, 0xca, 0x64, 0xf1, 0x06, 0xf4, 0x7c, 0x6e, 0x99, 0xcd, 0x8b, 0xa9, 0x56, 0xd6, + 0x70, 0x3d, 0x2b, 0xc8, 0x85, 0x2a, 0x74, 0x62, 0x75, 0x12, 0xa5, 0x9a, 0x94, 0x84, 0x38, 0xcf, + 0xf9, 0x92, 0x41, 0x11, 0x35, 0x59, 0xb2, 0xb7, 0x41, 0xb8, 0x93, 0x24, 0xc1, 0xfd, 0x51, 0x20, + 0xe7, 0xe4, 0x5d, 0x4f, 0x63, 0x72, 0xea, 0xb7, 0xa1, 0x36, 0x8e, 0x72, 0x8b, 0xa4, 0x64, 0x88, + 0x5a, 0x8b, 0xe3, 0x88, 0x38, 0xe4, 0x32, 0x5a, 0x4d, 0x4f, 0x27, 0x7e, 0x22, 0x3d, 0xa3, 0x87, + 0xcd, 0xbb, 0x56, 0xd2, 0x27, 0xbe, 0xe7, 0xc9, 0x50, 0x27, 0x8e, 0xea, 0xbe, 0xfa, 0x86, 0xde, + 0xa9, 0x98, 0x57, 0x0e, 0x1d, 0x74, 0xcd, 0xc2, 0x49, 0x40, 0xbb, 0x22, 0xd0, 0xa5, 0xa3, 0x5d, + 0x8d, 0x78, 0x34, 0x09, 0x70, 0x47, 0x04, 0x7a, 0x49, 0x49, 0x97, 0x20, 0x0b, 0xda, 0x27, 0x7e, + 0x98, 0x92, 0xa8, 0x68, 0xd0, 0x92, 0x22, 0x02, 0x99, 0xf0, 0x1b, 0x3f, 0x4c, 0x07, 0xff, 0x7c, + 0x0e, 0xd6, 0xf4, 0xc4, 0x1f, 0xea, 0x09, 0xd0, 0xfa, 0x99, 0xbc, 0x0b, 0x33, 0x5d, 0x3a, 0xaf, + 0x37, 0x6f, 0x81, 0x01, 0xed, 0xd1, 0x80, 0x73, 0xee, 0x9a, 0xd3, 0x25, 0xa5, 0x86, 0xaf, 0x6e, + 0x83, 0xb8, 0xc0, 0x57, 0x4a, 0x87, 0x35, 0x7b, 0x53, 0x8c, 0xa5, 0xc4, 0x27, 0xb0, 0x36, 0x96, + 0xa9, 0x43, 0x1b, 0x21, 0x88, 0x5c, 0x87, 0x5a, 0xd1, 0x96, 0xe7, 0xe9, 0x5e, 0x31, 0xd8, 0x87, + 0x1a, 0x89, 0x9b, 0x1e, 0xbf, 0x31, 0x76, 0x42, 0x7f, 0x28, 0x55, 0x4a, 0x76, 0x06, 0xb7, 0x60, + 0xc3, 0xa7, 0x67, 0x30, 0x68, 0x49, 0x10, 0x35, 0x59, 0xac, 0x43, 0x5e, 0xc4, 0x45, 0xa2, 0xa9, + 0x25, 0x72, 0xa8, 0xd7, 0xae, 0x8d, 0x6b, 0x15, 0xfa, 0xe1, 0x88, 0x8f, 0x42, 0xd4, 0xd8, 0xa6, + 0x34, 0xc0, 0xfd, 0xc8, 0x93, 0x83, 0xdf, 0x2f, 0x64, 0x3c, 0x7a, 0xa0, 0xe1, 0x87, 0xa9, 0x93, + 0x52, 0xf5, 0x7f, 0x36, 0x78, 0xd6, 0x91, 0x3c, 0x57, 0x6d, 0x03, 0xe5, 0x43, 0x02, 0x9b, 0xb0, + 0x5c, 0x1e, 0x2d, 0xd3, 0xce, 0x71, 0x89, 0x43, 0x71, 0xb8, 0xd9, 0xa1, 0x82, 0x8c, 0x9e, 0x49, + 0x75, 0x25, 0xbc, 0x81, 0x32, 0xd9, 0x87, 0xf9, 0x24, 0x28, 0x5d, 0x28, 0x20, 0x3d, 0xad, 0x15, + 0xb3, 0x5e, 0xd5, 0xa1, 0x46, 0xe0, 0xd6, 0xcc, 0xc9, 0xe3, 0x64, 0x12, 0x4a, 0x4f, 0xab, 0xf4, + 0x6e, 0x06, 0x3f, 0x20, 0x30, 0x0e, 0x38, 0x93, 0x4c, 0x85, 0xae, 0x17, 0xb9, 0x6b, 0x4f, 0x4b, + 0xa6, 0xbc, 0x6b, 0xe4, 0xd1, 0x9c, 0x5e, 0xf7, 0xcd, 0x02, 0xa2, 0x9b, 0x51, 0xeb, 0xbe, 0x7f, + 0x0e, 0xfd, 0x8c, 0x96, 0x7f, 0x5d, 0xfe, 0x81, 0x3a, 0x2b, 0x1f, 0xd3, 0x84, 0x7e, 0x66, 0xf6, + 0x91, 0x8f, 0x61, 0x6d, 0xba, 0xa1, 0xfe, 0x52, 0x83, 0x9a, 0x2d, 0x97, 0x9a, 0xe5, 0xbf, 0x24, + 0x5b, 0x5f, 0xd7, 0x71, 0x4f, 0xa4, 0x7d, 0xe2, 0xeb, 0x62, 0xf8, 0x79, 0x6b, 0xc9, 0xa0, 0x76, + 0x10, 0xf3, 0x8d, 0x9f, 0xaa, 0x19, 0xf4, 0x63, 0x5f, 0x29, 0xad, 0x15, 0xcb, 0xf4, 0xfb, 0xbe, + 0x52, 0x83, 0xff, 0xdc, 0x85, 0x96, 0xb1, 0x14, 0xa9, 0x0e, 0xfb, 0x76, 0xd1, 0xe8, 0x6f, 0x6e, + 0xf5, 0x8c, 0xf5, 0x8e, 0x24, 0xdb, 0x69, 0x9a, 0x98, 0x5c, 0x1f, 0x3b, 0x03, 0x25, 0x7b, 0x67, + 0x8e, 0x0c, 0x84, 0xdc, 0xde, 0xd9, 0x86, 0xa5, 0x82, 0x05, 0x69, 0xa7, 0x51, 0xea, 0x04, 0xda, + 0x29, 0x28, 0xd4, 0xb0, 0x15, 0x48, 0xac, 0x2e, 0xbe, 0xb0, 0x6d, 0x71, 0x84, 0xd4, 0xe8, 0x6c, + 0xb8, 0x51, 0x60, 0x0a, 0x7f, 0xa7, 0x9c, 0x0d, 0xc4, 0x50, 0x75, 0x4e, 0x22, 0xd1, 0xcf, 0x55, + 0x4f, 0x03, 0xbd, 0x83, 0x1a, 0x0c, 0x39, 0x7c, 0x1a, 0x64, 0x03, 0x24, 0x63, 0x7e, 0x91, 0xfc, + 0x18, 0x1a, 0x20, 0x59, 0xe9, 0x1f, 0x42, 0x33, 0x4a, 0xfc, 0x91, 0x4f, 0x69, 0x62, 0x36, 0x70, + 0xa6, 0x3f, 0x02, 0x4c, 0xb0, 0x83, 0x9f, 0x1a, 0xc0, 0xa2, 0x56, 0xff, 0x17, 0x2b, 0x76, 0x34, + 0xc6, 0x1c, 0x47, 0x70, 0x53, 0x1c, 0x0e, 0xef, 0xc8, 0x46, 0x7e, 0x1c, 0xc1, 0x4d, 0x0f, 0x9f, + 0x06, 0x94, 0x29, 0x7f, 0x17, 0xba, 0x5c, 0x0d, 0xc2, 0x1b, 0x2a, 0x90, 0x21, 0xad, 0x69, 0xd5, + 0x6a, 0x33, 0x18, 0xc7, 0xf7, 0x50, 0x86, 0xba, 0xec, 0xd3, 0x09, 0x02, 0xf4, 0x58, 0x23, 0xc7, + 0xd3, 0x35, 0x3a, 0x2d, 0x03, 0x7c, 0x18, 0x39, 0x9e, 0xf8, 0x02, 0x2e, 0x23, 0xce, 0xe6, 0xe2, + 0xda, 0x70, 0x32, 0x96, 0x89, 0xef, 0xda, 0x8e, 0xa2, 0x9a, 0x1d, 0x5d, 0xaa, 0xb3, 0x86, 0x14, + 0xf7, 0x91, 0xe0, 0x11, 0xe3, 0xb7, 0xd5, 0xf7, 0x32, 0x89, 0xc4, 0xf7, 0x94, 0x2d, 0x9f, 0x65, + 0xbe, 0x9b, 0xb0, 0xc4, 0x5b, 0xf9, 0x5a, 0xbd, 0x84, 0x92, 0x6a, 0xe2, 0x10, 0x61, 0x19, 0xa3, + 0x8f, 0xda, 0x8b, 0x6f, 0x41, 0x18, 0x05, 0x47, 0x9c, 0x9f, 0x3a, 0xea, 0x94, 0x0b, 0x84, 0x4b, + 0xa1, 0xb6, 0x19, 0x36, 0xaa, 0x65, 0x34, 0x23, 0x02, 0x11, 0xa0, 0xc4, 0x6f, 0x60, 0x25, 0xeb, + 0x4c, 0xdb, 0x32, 0xd4, 0x1d, 0x07, 0x34, 0xae, 0x5f, 0xec, 0xae, 0x64, 0x02, 0x59, 0x66, 0x24, + 0x0c, 0xe6, 0x2e, 0xbf, 0x86, 0xae, 0xe9, 0x92, 0x67, 0x5d, 0xf5, 0x7b, 0xd4, 0xdb, 0xb5, 0x0b, + 0xbd, 0x95, 0x74, 0x7b, 0xa6, 0x9f, 0x19, 0x8a, 0x3f, 0x34, 0xd3, 0xe4, 0x46, 0xcb, 0x50, 0x38, + 0xa4, 0xb9, 0x75, 0xe3, 0x42, 0x4f, 0x53, 0xca, 0xca, 0x32, 0x43, 0x30, 0x70, 0xf1, 0x11, 0xac, + 0x9a, 0xce, 0x22, 0x72, 0xf1, 0x6c, 0x3f, 0x22, 0xef, 0x4f, 0xb0, 0x89, 0xa5, 0x91, 0xec, 0xfe, + 0xed, 0x45, 0xec, 0x2d, 0x5e, 0x31, 0x4d, 0x58, 0x0b, 0x93, 0x47, 0x9c, 0xfd, 0xa8, 0x65, 0xd2, + 0x5d, 0x7d, 0x4d, 0xc2, 0x7a, 0x19, 0x3d, 0x60, 0x33, 0xfc, 0x0d, 0xe8, 0x51, 0x15, 0x3f, 0x2e, + 0x6b, 0x94, 0x78, 0x7e, 0xe8, 0x04, 0xfd, 0x15, 0x8e, 0xb9, 0x22, 0xdc, 0x8a, 0x9e, 0x3f, 0x66, + 0xa8, 0x38, 0x82, 0x35, 0xf3, 0xa1, 0x4c, 0xcc, 0x28, 0x54, 0x25, 0x14, 0xf0, 0x9e, 0x35, 0x71, + 0x25, 0x85, 0x63, 0x99, 0x25, 0x2c, 0xab, 0xa1, 0x5d, 0xb8, 0x3e, 0xb5, 0xb4, 0x63, 0xe7, 0xcc, + 0x1e, 0xcb, 0x71, 0x94, 0x9c, 0x6b, 0x05, 0xb2, 0x46, 0x02, 0xec, 0x4a, 0x69, 0x11, 0xf7, 0x9d, + 0xb3, 0x7d, 0xa2, 0x61, 0x75, 0xf2, 0x15, 0x5c, 0x9d, 0xea, 0x85, 0x8b, 0xe2, 0x65, 0xe8, 0x1c, + 0x07, 0xd2, 0xa3, 0x10, 0x79, 0xdd, 0xba, 0x54, 0xea, 0xe2, 0x10, 0x29, 0xee, 0x33, 0x81, 0xf8, + 0x12, 0x48, 0xd8, 0x2b, 0x3a, 0x3b, 0x68, 0x2b, 0xd7, 0x09, 0x29, 0x2a, 0x9e, 0x55, 0x55, 0x20, + 0x2f, 0xf2, 0xc1, 0x42, 0x94, 0x94, 0x56, 0x27, 0x27, 0x26, 0xc9, 0xf9, 0x09, 0xb4, 0x4c, 0x60, + 0x99, 0xda, 0x5e, 0xa2, 0xb6, 0x4b, 0xdc, 0x56, 0x87, 0x92, 0xa9, 0x61, 0x73, 0x98, 0xbf, 0x88, + 0x4d, 0x80, 0x53, 0x67, 0x78, 0xea, 0x70, 0x9b, 0xcb, 0x45, 0x07, 0xff, 0x5b, 0x84, 0x53, 0x8b, + 0xc6, 0xa9, 0x79, 0x14, 0x9f, 0xc3, 0x25, 0xb3, 0x0b, 0x9f, 0x9f, 0x44, 0x81, 0x36, 0xd8, 0x87, + 0x4e, 0x18, 0x4d, 0x52, 0x1d, 0x28, 0x5f, 0xd3, 0x04, 0xdf, 0x21, 0x1e, 0x37, 0xc0, 0x03, 0xc2, + 0x8a, 0xdf, 0xc0, 0xaa, 0x93, 0xe0, 0x1a, 0xcb, 0x33, 0xe9, 0x4e, 0xd8, 0xbc, 0x21, 0x03, 0xf7, + 0x2a, 0x45, 0x55, 0x0b, 0x3b, 0x72, 0x1b, 0xc9, 0xee, 0x1b, 0x2a, 0x32, 0x77, 0xad, 0x65, 0xe7, + 0x22, 0x50, 0x3c, 0x81, 0x75, 0xee, 0xd2, 0x70, 0x2a, 0x59, 0xc8, 0xa9, 0x2f, 0x67, 0x44, 0xd4, + 0xa9, 0x53, 0xcd, 0xb5, 0x4c, 0x76, 0x6e, 0xf1, 0x80, 0x4a, 0x40, 0x5f, 0x2a, 0xf1, 0x3d, 0x5c, + 0xe2, 0x6e, 0xb5, 0xbf, 0xac, 0x8f, 0x28, 0xb2, 0x54, 0xba, 0x36, 0xbd, 0xe1, 0xa9, 0x63, 0x8b, + 0x28, 0xa9, 0x16, 0x97, 0xa4, 0x8f, 0xb5, 0xe6, 0xcc, 0x02, 0x2b, 0xf1, 0x19, 0xf4, 0xb9, 0x6f, + 0x6d, 0x01, 0x0e, 0xfd, 0x70, 0x24, 0x93, 0x38, 0x41, 0x53, 0xf3, 0x3a, 0xc5, 0xa3, 0xb8, 0x25, + 0xfb, 0x19, 0x0f, 0x72, 0xac, 0xd8, 0x86, 0x37, 0xb9, 0xa5, 0x1b, 0x85, 0xcf, 0x64, 0xa2, 0x70, + 0x02, 0x71, 0xa1, 0x6c, 0xfd, 0xd2, 0xbf, 0x41, 0xd5, 0xcc, 0x97, 0x89, 0x68, 0x27, 0xa3, 0x41, + 0x5e, 0xff, 0x2d, 0x3f, 0xa2, 0xe3, 0xca, 0x5d, 0x0c, 0xa3, 0xc4, 0x45, 0x36, 0x4f, 0x65, 0xe2, + 0x3b, 0x01, 0x6a, 0xcf, 0xb7, 0x68, 0xed, 0x78, 0x42, 0x1e, 0x20, 0x76, 0x3f, 0x47, 0x8a, 0x5f, + 0xc2, 0x15, 0x6e, 0xe7, 0xf9, 0xa8, 0x47, 0x8e, 0x27, 0xa9, 0xf4, 0xf2, 0x65, 0xec, 0x0f, 0x98, + 0xb5, 0x89, 0x64, 0x37, 0xa7, 0xc8, 0x16, 0x4b, 0xfb, 0x2a, 0xc7, 0xd0, 0xa0, 0x08, 0x1f, 0x31, + 0x52, 0x76, 0xf6, 0xa6, 0xf2, 0xea, 0xb3, 0x37, 0x1f, 0x42, 0x4b, 0x3b, 0xb2, 0x2f, 0x3b, 0xcc, + 0xd3, 0x64, 0x3c, 0xe7, 0xc2, 0x6f, 0x43, 0x83, 0xbc, 0x58, 0xfa, 0xc6, 0x75, 0x68, 0xf2, 0xd2, + 0x1d, 0x07, 0x91, 0x7b, 0x6a, 0xfc, 0x4e, 0x02, 0xdd, 0x43, 0xc8, 0x00, 0xa0, 0xfe, 0x24, 0xf4, + 0xa3, 0x70, 0x3b, 0x08, 0x06, 0x7f, 0x5c, 0x84, 0x06, 0x9a, 0xbb, 0x14, 0x92, 0x14, 0x03, 0x68, + 0x93, 0x4c, 0xa2, 0x92, 0x9a, 0xb1, 0x13, 0xeb, 0xd3, 0x45, 0x4d, 0x04, 0x22, 0xd5, 0xbe, 0x13, + 0x4f, 0x55, 0xdc, 0xcc, 0x4d, 0x55, 0xdc, 0xbc, 0xc5, 0x87, 0x98, 0x99, 0x79, 0xa4, 0x39, 0xf5, + 0x41, 0x1d, 0xdc, 0x63, 0x10, 0x9a, 0xe1, 0x44, 0xe2, 0x04, 0x64, 0xba, 0x4b, 0xb4, 0xe4, 0x94, + 0x2e, 0xce, 0x21, 0x91, 0xb8, 0xad, 0x11, 0x87, 0x92, 0x4d, 0x8d, 0x42, 0x1c, 0xba, 0x3a, 0x1d, + 0x87, 0xbe, 0x05, 0xe0, 0x46, 0xa1, 0x47, 0xde, 0xc1, 0x54, 0x2d, 0x02, 0x57, 0xc6, 0xe4, 0xd8, + 0x1f, 0x90, 0x21, 0x79, 0x0f, 0x7a, 0x19, 0x05, 0x1a, 0xff, 0x6e, 0x98, 0x85, 0x56, 0x34, 0x95, + 0x25, 0x87, 0x3b, 0x61, 0x3a, 0x9d, 0x4a, 0x69, 0x5c, 0x48, 0xa5, 0xbc, 0xa4, 0x86, 0x0a, 0x7e, + 0xf4, 0x21, 0xce, 0x4b, 0x50, 0xa7, 0x62, 0x4d, 0x6f, 0x12, 0x6b, 0x33, 0xa4, 0xe6, 0x2b, 0x4a, + 0x79, 0xbd, 0x2c, 0x5d, 0xd3, 0xfa, 0xbf, 0x95, 0xae, 0x69, 0xff, 0xb0, 0x74, 0x4d, 0xe7, 0x87, + 0xa5, 0x6b, 0xa6, 0xd2, 0x1b, 0xdd, 0xe9, 0xf4, 0xc6, 0x4b, 0x33, 0xca, 0xbd, 0x97, 0x66, 0x94, + 0x5f, 0x93, 0x0e, 0x5e, 0x7a, 0x75, 0x3a, 0xf8, 0xf5, 0xf9, 0x68, 0xf1, 0xba, 0x7c, 0xf4, 0xbb, + 0xd0, 0xe5, 0xe2, 0x65, 0x72, 0xb2, 0x4f, 0xe5, 0xb9, 0xd2, 0xf9, 0xef, 0x36, 0x81, 0xd1, 0xc5, + 0xfe, 0x56, 0x9e, 0xab, 0xc1, 0x13, 0x00, 0x8a, 0x3e, 0xd0, 0x4f, 0x7b, 0x19, 0x6f, 0x54, 0x7e, + 0x74, 0x7d, 0xdd, 0x5f, 0x57, 0x00, 0x0e, 0x9d, 0x71, 0xcc, 0xe9, 0x03, 0xf1, 0x2b, 0x68, 0x2a, + 0x7a, 0x2b, 0xd6, 0x17, 0x15, 0x44, 0x76, 0x4e, 0xaa, 0x1f, 0xf9, 0x8c, 0xa1, 0xca, 0x9e, 0x89, + 0xad, 0xb9, 0x87, 0xac, 0x96, 0xb9, 0x6a, 0x08, 0x28, 0xac, 0x7b, 0x13, 0x3a, 0x9a, 0x20, 0x96, + 0x89, 0x2b, 0x43, 0x3e, 0x66, 0x52, 0xb1, 0xda, 0x0c, 0x3d, 0x60, 0xa0, 0xf8, 0x28, 0x23, 0x33, + 0xd6, 0xd0, 0xc5, 0x6c, 0xa9, 0x6e, 0xa2, 0xcd, 0xa1, 0xc1, 0x96, 0xf9, 0x29, 0x34, 0x90, 0x3a, + 0x2c, 0xe0, 0xf7, 0x7a, 0x6f, 0x88, 0x26, 0xd4, 0x74, 0xaf, 0xbd, 0x8a, 0x68, 0x43, 0x83, 0x0e, + 0xb5, 0x12, 0x6e, 0x6e, 0xf0, 0x8f, 0x57, 0xa1, 0xb9, 0x17, 0xaa, 0x34, 0x99, 0x30, 0x0b, 0xe7, + 0x47, 0x37, 0xab, 0x74, 0x74, 0x53, 0x1f, 0x47, 0xe0, 0x9f, 0x41, 0xc7, 0x11, 0x3e, 0x84, 0x9a, + 0x3e, 0x25, 0xac, 0x73, 0x4a, 0x33, 0x8f, 0x18, 0x1b, 0x1a, 0xb1, 0x09, 0x75, 0x4f, 0x1f, 0x5f, + 0xd6, 0x45, 0x54, 0x85, 0x33, 0xc5, 0xe6, 0x60, 0xb3, 0x95, 0xd1, 0x88, 0xb7, 0x60, 0xde, 0x19, + 0x8d, 0x74, 0x40, 0xa7, 0x9b, 0x93, 0x92, 0x7d, 0x6e, 0x21, 0x4e, 0xdc, 0x81, 0x06, 0x89, 0x4f, + 0x2a, 0x68, 0x5c, 0x9c, 0xee, 0xd3, 0x54, 0x4b, 0xb2, 0x44, 0xa5, 0x74, 0xd4, 0x1d, 0x68, 0x04, + 0x51, 0x14, 0x73, 0x83, 0xda, 0x74, 0x03, 0x53, 0x5a, 0x66, 0xd5, 0x03, 0x53, 0x64, 0xf6, 0x2e, + 0x2c, 0xa2, 0xe7, 0x17, 0xc5, 0xda, 0x63, 0x2a, 0x8c, 0x83, 0x4a, 0xac, 0xac, 0xaa, 0xa2, 0x4a, + 0xab, 0x2d, 0x00, 0xe6, 0x7f, 0xea, 0xb9, 0x31, 0x3d, 0x1d, 0x59, 0x5a, 0x1d, 0x37, 0xa9, 0xc9, + 0xb0, 0xdf, 0x83, 0x1e, 0xa7, 0x50, 0x0b, 0x2d, 0xc1, 0x94, 0x81, 0x9b, 0x96, 0xe5, 0xac, 0xbc, + 0xd5, 0x49, 0xca, 0x59, 0xfa, 0x0f, 0xa0, 0x16, 0x73, 0x5e, 0x90, 0x24, 0x0c, 0x59, 0x6b, 0xa6, + 0xa9, 0x4e, 0x18, 0x5a, 0x86, 0x42, 0xfc, 0x12, 0x3a, 0x5c, 0xae, 0x3c, 0xd4, 0x09, 0x32, 0x0a, + 0xea, 0x96, 0x0e, 0x98, 0x96, 0xf2, 0x67, 0x56, 0x3b, 0x2d, 0xa5, 0xd3, 0x7e, 0x01, 0xed, 0xfc, + 0xdc, 0x1c, 0x1a, 0x7b, 0x5d, 0x93, 0xa8, 0x32, 0xcd, 0x8b, 0x8e, 0xb8, 0xd5, 0x92, 0x45, 0xb7, + 0x7c, 0x03, 0x16, 0x75, 0x09, 0x7d, 0x8f, 0x5a, 0x15, 0xae, 0x10, 0xe1, 0xa2, 0x59, 0x4b, 0xe3, + 0x71, 0x2e, 0xf3, 0xea, 0x60, 0xf2, 0x19, 0x4a, 0x73, 0x99, 0x95, 0x06, 0x5b, 0x8d, 0xac, 0x2a, + 0x58, 0xdc, 0x2f, 0x57, 0x2b, 0x73, 0x3e, 0x77, 0x99, 0x9a, 0x5e, 0x9a, 0xd1, 0x94, 0x33, 0xbb, + 0x56, 0x37, 0x9e, 0x2a, 0x7a, 0xbe, 0x0d, 0xf5, 0x28, 0xf1, 0xe8, 0x98, 0x0a, 0x15, 0xd0, 0x64, + 0xd6, 0xaf, 0x3e, 0x1a, 0x4d, 0xc2, 0xa3, 0x16, 0xf1, 0x0b, 0x1a, 0x16, 0x71, 0x12, 0x91, 0xd9, + 0x48, 0x22, 0x6e, 0xf5, 0xa2, 0x61, 0xa1, 0xf1, 0x24, 0xe0, 0xde, 0x81, 0x9a, 0x39, 0x18, 0xb0, + 0x76, 0x81, 0xd2, 0xa0, 0xc4, 0xc7, 0xd0, 0x2d, 0x0b, 0x34, 0xd5, 0x5f, 0xbf, 0x40, 0xdd, 0x29, + 0xc9, 0x2f, 0xd4, 0xc6, 0xd5, 0xc0, 0x1f, 0xfb, 0xa9, 0x36, 0xf7, 0x4b, 0x07, 0x95, 0x09, 0x81, + 0xae, 0xbf, 0xce, 0x7e, 0x5d, 0xba, 0xe8, 0xfa, 0xeb, 0x0c, 0x59, 0x1f, 0x6a, 0xbe, 0x7a, 0xe0, + 0x27, 0x2a, 0xd5, 0xa5, 0x2d, 0xe6, 0x55, 0xac, 0xc1, 0xa2, 0xaf, 0x50, 0x4d, 0x68, 0x03, 0x5d, + 0xbf, 0x89, 0x5b, 0xb0, 0xa8, 0x0f, 0x4d, 0xdc, 0xb8, 0xb0, 0xa3, 0xf5, 0x01, 0x2f, 0x4b, 0x53, + 0x88, 0xf7, 0xa1, 0x46, 0x15, 0xf3, 0x51, 0x4c, 0x96, 0x62, 0x89, 0x03, 0xb8, 0x6c, 0xdd, 0x5a, + 0x0c, 0xb8, 0x7c, 0xfd, 0x03, 0xa8, 0x19, 0x23, 0x65, 0x30, 0xcd, 0xd5, 0xda, 0x58, 0xb1, 0x0c, + 0x85, 0xb8, 0x09, 0xd5, 0x31, 0xca, 0xb1, 0xfe, 0xdb, 0xd3, 0x3b, 0x94, 0xc5, 0x1b, 0x63, 0xc5, + 0xff, 0x0f, 0x97, 0x8b, 0x35, 0xe7, 0xa6, 0x20, 0x5d, 0xc7, 0xb6, 0x6f, 0x52, 0xdb, 0xb7, 0x66, + 0xb0, 0x4a, 0xb9, 0x74, 0xdd, 0x5a, 0x8f, 0x5f, 0x52, 0xd3, 0xfe, 0x69, 0x26, 0xee, 0x71, 0x77, + 0xf5, 0xdf, 0x35, 0x7e, 0xd7, 0x45, 0x85, 0x61, 0x94, 0x00, 0xe9, 0x99, 0xcf, 0xa0, 0x35, 0x9c, + 0xbc, 0x78, 0x71, 0x6e, 0xf2, 0x32, 0xef, 0x51, 0xbb, 0x42, 0x74, 0xa9, 0x50, 0xe6, 0x6e, 0x35, + 0x87, 0x85, 0x9a, 0xf7, 0x75, 0xa8, 0xb9, 0xa1, 0xed, 0x78, 0x5e, 0xd2, 0xdf, 0xe0, 0x32, 0x77, + 0x37, 0xdc, 0xf6, 0x3c, 0xba, 0x63, 0x24, 0x8a, 0x25, 0x1d, 0xd4, 0xb7, 0x7d, 0xaf, 0xff, 0x3e, + 0x2b, 0x1e, 0x03, 0xda, 0xf3, 0xe8, 0x12, 0x12, 0x13, 0x92, 0xf1, 0xbd, 0xfe, 0x2d, 0x7d, 0x09, + 0x89, 0x06, 0xed, 0x79, 0x68, 0x78, 0xa2, 0xff, 0x6a, 0x20, 0xfd, 0x0f, 0x38, 0xd7, 0x35, 0x76, + 0xce, 0x0e, 0x34, 0x08, 0x37, 0x29, 0x67, 0x75, 0x49, 0x6c, 0xdd, 0x9e, 0xde, 0xa4, 0x59, 0x05, + 0x80, 0xd5, 0xf0, 0xb3, 0x62, 0x00, 0xda, 0xd8, 0x24, 0x8a, 0xec, 0x60, 0xab, 0xff, 0xe1, 0xc5, + 0x8d, 0xad, 0x0b, 0x1c, 0x70, 0x63, 0x9b, 0x5a, 0x87, 0x2d, 0x00, 0x96, 0x59, 0x24, 0x70, 0x36, + 0xa7, 0xdb, 0x64, 0xde, 0x80, 0xc5, 0x27, 0xeb, 0x48, 0xd4, 0x6c, 0x01, 0x50, 0x66, 0x89, 0xdb, + 0xdc, 0x99, 0x6e, 0x93, 0x59, 0xf7, 0x56, 0xe3, 0x59, 0x66, 0xe8, 0xdf, 0x81, 0xc6, 0x04, 0xed, + 0x78, 0xb4, 0xa4, 0xfb, 0x77, 0xa7, 0x99, 0xd9, 0x98, 0xf8, 0x56, 0x7d, 0xa2, 0x9f, 0xf0, 0x23, + 0xa4, 0x7b, 0xc8, 0x0c, 0xe9, 0x7f, 0x34, 0xfd, 0x91, 0xcc, 0x0f, 0xb0, 0x48, 0x45, 0xb1, 0x4b, + 0xf0, 0x29, 0x34, 0x79, 0xd2, 0xb8, 0xd1, 0xd6, 0x34, 0x8f, 0xe4, 0x76, 0x8d, 0xc5, 0xb3, 0xcb, + 0xcd, 0x6e, 0x42, 0xd5, 0x89, 0xe3, 0xe0, 0xbc, 0xff, 0xf1, 0x34, 0x87, 0x6f, 0x23, 0xd8, 0x62, + 0x2c, 0xb2, 0xd2, 0x78, 0x12, 0xa4, 0xbe, 0x39, 0x0c, 0xf7, 0xc9, 0x34, 0x2b, 0x15, 0x4e, 0x17, + 0x5b, 0xcd, 0x71, 0xe1, 0xa8, 0xf1, 0x6d, 0xa8, 0xc7, 0x91, 0x4a, 0x6d, 0x6f, 0x1c, 0xf4, 0x3f, + 0xbd, 0xa0, 0x46, 0xf8, 0x30, 0x92, 0x55, 0x8b, 0xf5, 0x69, 0xae, 0xd2, 0x61, 0xfd, 0x9f, 0x4d, + 0x1d, 0xd6, 0xdf, 0x82, 0xd6, 0x38, 0x0a, 0x47, 0x91, 0x77, 0xcc, 0xb3, 0xff, 0xf3, 0x62, 0x3c, + 0x60, 0x1f, 0x31, 0x1c, 0x41, 0xd0, 0x44, 0x5a, 0x35, 0xf4, 0xb4, 0xed, 0xe6, 0x2b, 0xdb, 0x51, + 0x24, 0xf6, 0x3f, 0xe3, 0xe8, 0x0d, 0xc3, 0xf7, 0xd4, 0x36, 0x41, 0xcb, 0x99, 0x8d, 0xe3, 0x73, + 0x9d, 0x86, 0xfe, 0x9c, 0xd8, 0x33, 0xcf, 0x6c, 0xdc, 0x3b, 0xe7, 0x5c, 0xf4, 0x17, 0x70, 0xb9, + 0x70, 0x7c, 0x36, 0x8a, 0xed, 0xd0, 0x46, 0x11, 0x90, 0x48, 0x6f, 0xe2, 0xca, 0xfe, 0x17, 0x59, + 0xa8, 0x41, 0x9f, 0xa1, 0x8d, 0xe2, 0x47, 0x07, 0x89, 0xb4, 0x08, 0x2b, 0x1e, 0x15, 0x8f, 0xea, + 0x3a, 0xc1, 0x28, 0x4a, 0xfc, 0xf4, 0x64, 0xdc, 0xff, 0x85, 0x09, 0x34, 0x64, 0xfe, 0x40, 0x96, + 0xfa, 0xdc, 0x36, 0x44, 0x56, 0x3e, 0xc6, 0x0c, 0x26, 0xbe, 0x80, 0x4b, 0x5a, 0x17, 0x60, 0x87, + 0xa5, 0x1b, 0x11, 0x54, 0xff, 0x4f, 0xe8, 0x4a, 0x84, 0xf5, 0x9c, 0xe0, 0xeb, 0xc2, 0xe5, 0x08, + 0x0a, 0xdd, 0xf6, 0x59, 0x6d, 0xd1, 0x30, 0xe1, 0x09, 0xf8, 0x92, 0x26, 0xe0, 0xf2, 0xc5, 0xf6, + 0x87, 0x52, 0x5f, 0xb2, 0x74, 0x1b, 0x04, 0x4e, 0x00, 0x69, 0x2e, 0xe9, 0xd9, 0xd1, 0x24, 0x8d, + 0x27, 0x69, 0xff, 0x97, 0xec, 0x27, 0xa6, 0x51, 0xfc, 0x98, 0x11, 0x8f, 0x09, 0x2e, 0x3e, 0x87, + 0x4b, 0x24, 0x34, 0xed, 0x62, 0x1b, 0x0e, 0x14, 0xa9, 0xfe, 0x57, 0x3c, 0x6f, 0x44, 0x70, 0x94, + 0xb5, 0xe4, 0xb0, 0x12, 0x55, 0x58, 0x4e, 0xcf, 0xf9, 0x73, 0x3f, 0x3d, 0xb1, 0x29, 0xa2, 0xf2, + 0x2b, 0x8e, 0x10, 0x94, 0xa6, 0xfc, 0x3b, 0x3f, 0x3d, 0x39, 0xf2, 0xa5, 0x62, 0x0f, 0xff, 0xd7, + 0x0b, 0xf5, 0xa5, 0x9e, 0xf8, 0xf5, 0x42, 0xfd, 0x9d, 0xde, 0x4d, 0xab, 0x59, 0x08, 0x64, 0x0d, + 0x3e, 0x85, 0xd6, 0x36, 0x5d, 0x00, 0xe6, 0x2b, 0xd2, 0xa6, 0x37, 0x61, 0x21, 0x2b, 0x80, 0xca, + 0xd4, 0x34, 0x51, 0xbc, 0x90, 0x7b, 0xe1, 0x30, 0xb2, 0x08, 0x3d, 0xf8, 0xd7, 0x0b, 0xb0, 0xc8, + 0x45, 0x29, 0xaf, 0x3f, 0x9e, 0xfb, 0xa6, 0x91, 0x35, 0x61, 0x7e, 0x76, 0x88, 0xc5, 0x0a, 0xa1, + 0xa7, 0xcf, 0x08, 0x34, 0xf2, 0xda, 0xaa, 0x15, 0xa8, 0x72, 0x80, 0x80, 0x53, 0x66, 0xfc, 0x42, + 0x72, 0x76, 0xa2, 0x4e, 0xe8, 0x96, 0x2f, 0x9d, 0x04, 0x5e, 0xb0, 0xc0, 0x80, 0xf6, 0x3c, 0x8a, + 0x8d, 0x1b, 0x02, 0x12, 0xe4, 0x8b, 0x3a, 0xf7, 0xa5, 0x81, 0x24, 0xce, 0x4d, 0xdd, 0x56, 0xed, + 0x25, 0x75, 0x5b, 0xd7, 0x60, 0x21, 0x34, 0xc7, 0xda, 0x32, 0x3c, 0xdd, 0xcf, 0x43, 0x70, 0x71, + 0x0b, 0xb2, 0x33, 0xc5, 0xda, 0x30, 0x7d, 0xf9, 0x99, 0xe3, 0x2d, 0x68, 0x64, 0x57, 0xc6, 0x69, + 0x5b, 0x74, 0x65, 0x33, 0xbf, 0x44, 0xee, 0xc8, 0x3c, 0x59, 0x39, 0xd9, 0xab, 0xab, 0x8f, 0x9a, + 0x3f, 0xad, 0xfa, 0x88, 0x1d, 0x75, 0x37, 0x0a, 0x55, 0xaa, 0xa3, 0xff, 0x35, 0x5f, 0xed, 0xe0, + 0xab, 0xf8, 0x1c, 0xda, 0x89, 0x74, 0x9f, 0xd9, 0x63, 0x35, 0xe2, 0x4f, 0xb4, 0x8b, 0xf7, 0x1a, + 0x8c, 0xd5, 0xe8, 0x1b, 0xaa, 0x8c, 0xd2, 0x7e, 0x73, 0x13, 0x69, 0xf7, 0xd5, 0x88, 0x7a, 0xfd, + 0x00, 0x96, 0xc6, 0x72, 0x7c, 0xac, 0x6f, 0xf2, 0xd2, 0x0a, 0xb7, 0x43, 0x11, 0xb3, 0x5e, 0x8e, + 0xe0, 0xb1, 0x0c, 0xfe, 0x41, 0x05, 0xea, 0x38, 0x8b, 0xc8, 0x4b, 0x42, 0xc0, 0xc2, 0xd8, 0x8d, + 0x27, 0xda, 0x1d, 0xa2, 0x67, 0x7d, 0x0d, 0x1d, 0x73, 0x89, 0xbe, 0x86, 0x8e, 0xd6, 0x90, 0x73, + 0xda, 0xf4, 0xcc, 0x77, 0x02, 0x9d, 0x53, 0xda, 0x83, 0x39, 0xc3, 0xbc, 0x8a, 0x55, 0x58, 0x74, + 0x43, 0x8a, 0x89, 0x70, 0x6d, 0x40, 0xd5, 0x0d, 0x77, 0xc2, 0x54, 0x83, 0xf3, 0xa3, 0x51, 0x55, + 0x37, 0xdc, 0xf3, 0xce, 0x06, 0xff, 0xae, 0x02, 0x4b, 0x07, 0x49, 0xe4, 0x4a, 0xa5, 0x1e, 0xa2, + 0x39, 0x47, 0x79, 0x58, 0xfc, 0x22, 0xa5, 0xad, 0x38, 0xe5, 0x49, 0xcf, 0xc8, 0xc3, 0x1c, 0xb0, + 0xca, 0x9c, 0xce, 0x79, 0xab, 0x41, 0x10, 0xf2, 0x39, 0x33, 0x74, 0xa1, 0xbe, 0x87, 0xd1, 0x94, + 0xf0, 0xba, 0x09, 0x9d, 0x7c, 0xf7, 0x16, 0x4a, 0x91, 0xf2, 0x9b, 0x3b, 0xa8, 0x97, 0xeb, 0xd0, + 0xd4, 0x05, 0x6b, 0xd4, 0x0d, 0xe7, 0x30, 0x81, 0x41, 0x87, 0x7a, 0x14, 0xac, 0x22, 0x08, 0xcf, + 0x59, 0x4b, 0x56, 0x1a, 0x88, 0x1e, 0xfc, 0xbd, 0x39, 0xe8, 0x1d, 0x24, 0x32, 0x76, 0x12, 0x49, + 0x15, 0x6c, 0x34, 0xc7, 0x6b, 0xb0, 0x18, 0xc8, 0x70, 0xa4, 0x8b, 0x96, 0xe6, 0x2d, 0xfd, 0x96, + 0x5d, 0x21, 0x38, 0x57, 0xb8, 0x42, 0x10, 0xe7, 0x3a, 0x91, 0x8e, 0xbe, 0x69, 0x90, 0x9e, 0x71, + 0x0f, 0x86, 0x93, 0x40, 0xa7, 0xda, 0xea, 0x16, 0xbf, 0xe8, 0xfb, 0x01, 0x8e, 0xfd, 0x90, 0x2a, + 0x85, 0xe9, 0x7e, 0x80, 0x7b, 0x3e, 0xa9, 0x1c, 0x06, 0xa3, 0x09, 0xc8, 0xf7, 0x6c, 0x51, 0xc0, + 0xab, 0x6e, 0x75, 0x88, 0xc0, 0x49, 0xce, 0x0f, 0x09, 0x4a, 0x5e, 0x3a, 0x5f, 0xcf, 0xc5, 0x65, + 0x70, 0x9c, 0x65, 0x6b, 0x9b, 0xdb, 0xb9, 0x58, 0xb6, 0x50, 0x16, 0xdd, 0xec, 0x02, 0x4d, 0xee, + 0x45, 0x63, 0xc7, 0x0f, 0x15, 0x15, 0xe0, 0xb5, 0xad, 0x15, 0x8d, 0xe5, 0x5e, 0x77, 0x19, 0x37, + 0xf8, 0x27, 0x35, 0x68, 0xea, 0x75, 0xa5, 0x39, 0x60, 0x9e, 0xaa, 0x64, 0x3c, 0xd5, 0x83, 0x79, + 0xf5, 0x34, 0xd0, 0x4c, 0x86, 0x8f, 0xe2, 0x63, 0x98, 0x0f, 0xfc, 0xb1, 0x76, 0xb8, 0xaf, 0x94, + 0x8c, 0xa7, 0x32, 0x77, 0xe8, 0x0d, 0x80, 0xd4, 0xa8, 0xb1, 0xe9, 0x8a, 0x13, 0x1c, 0x80, 0x5e, + 0x51, 0x34, 0x64, 0xce, 0x70, 0x3f, 0xe3, 0x5a, 0x39, 0x2e, 0xd7, 0xb6, 0x6b, 0x21, 0xd5, 0xb6, + 0x1a, 0x1a, 0xb2, 0xe7, 0x89, 0x4f, 0xa0, 0x9e, 0x65, 0x84, 0x8c, 0x8b, 0x9d, 0x9e, 0x85, 0x9b, + 0x3b, 0x8f, 0x8e, 0xce, 0x42, 0x93, 0xf2, 0xd1, 0x1f, 0xcb, 0x28, 0xc5, 0x2f, 0xa1, 0xa5, 0xa4, + 0x52, 0x7c, 0x07, 0xc5, 0x30, 0xd2, 0xc2, 0x6b, 0xb5, 0xe8, 0x3d, 0x13, 0x16, 0x7f, 0xb5, 0xd9, + 0xaa, 0x2a, 0x07, 0x89, 0x6f, 0xa0, 0x63, 0xda, 0x07, 0xd1, 0x68, 0x94, 0x65, 0x2c, 0xaf, 0x5c, + 0xe8, 0xe1, 0x21, 0xa1, 0x0b, 0xfd, 0xb4, 0x55, 0x11, 0x21, 0xbe, 0x86, 0x4e, 0xcc, 0xac, 0x66, + 0xeb, 0x9a, 0x4d, 0x16, 0x82, 0x97, 0x4b, 0xb6, 0x7e, 0x89, 0x15, 0xf3, 0xf3, 0xcd, 0x39, 0x5c, + 0x5d, 0xbc, 0x8f, 0x85, 0x53, 0xd8, 0xe5, 0xfb, 0x58, 0x24, 0xac, 0xe9, 0x0b, 0xd4, 0x86, 0x89, + 0x43, 0xf7, 0x4a, 0xb0, 0x8a, 0x36, 0x85, 0xd8, 0x77, 0x2e, 0xac, 0x18, 0x7e, 0x70, 0x93, 0xef, + 0x1d, 0x79, 0xa0, 0x9b, 0x90, 0xca, 0xd6, 0x55, 0x6d, 0x2b, 0xc9, 0x0c, 0x94, 0xd8, 0x84, 0x65, + 0xfd, 0x99, 0x3c, 0x13, 0xe2, 0x7b, 0x24, 0x2a, 0x5b, 0xd6, 0x12, 0xa3, 0xb2, 0xd8, 0xf9, 0x9e, + 0x27, 0x3e, 0x83, 0xbe, 0x4a, 0x9d, 0x94, 0x8f, 0xf3, 0x1b, 0x3e, 0xf5, 0x47, 0x61, 0x94, 0x48, + 0x5d, 0x09, 0xb6, 0x96, 0xe1, 0xb5, 0x94, 0xde, 0x23, 0xac, 0xf8, 0x25, 0xf4, 0x28, 0x43, 0x90, + 0x95, 0xa3, 0xa4, 0x4a, 0x47, 0x0d, 0x66, 0x2b, 0x86, 0x0e, 0x52, 0x1b, 0xb6, 0x38, 0xa2, 0x7d, + 0x41, 0xed, 0xf5, 0x1d, 0x02, 0x24, 0x57, 0xe4, 0x44, 0x49, 0x4f, 0xdf, 0x9a, 0xb4, 0x82, 0xd8, + 0xaf, 0x33, 0xa4, 0x45, 0x38, 0xb4, 0x78, 0xcc, 0xa6, 0xd3, 0x91, 0x61, 0xb2, 0xb5, 0x73, 0x3b, + 0xac, 0xc7, 0x89, 0x0a, 0xbd, 0x07, 0x99, 0x06, 0x4d, 0xee, 0xcc, 0xe0, 0xba, 0xfc, 0x35, 0x5c, + 0x7a, 0xe9, 0xac, 0xbe, 0xae, 0x34, 0xad, 0x5d, 0xbc, 0xe3, 0xe3, 0xaf, 0xaa, 0xd0, 0x2c, 0x70, + 0x2b, 0x5d, 0x37, 0xaa, 0x64, 0x62, 0x0a, 0x50, 0xf1, 0x19, 0x61, 0x27, 0x91, 0x32, 0xf5, 0x94, + 0xf4, 0x8c, 0xb0, 0x24, 0xca, 0xea, 0xca, 0xe8, 0x19, 0x79, 0x48, 0x87, 0xc3, 0xf4, 0x8a, 0x2d, + 0xf0, 0xdd, 0x34, 0x39, 0x70, 0xcf, 0xa3, 0x7b, 0x49, 0x9d, 0xd4, 0x39, 0x76, 0x94, 0xa9, 0x24, + 0xce, 0xde, 0x51, 0xa1, 0x98, 0x5c, 0x8d, 0xae, 0xa3, 0xd1, 0xaf, 0xb8, 0xc7, 0x69, 0x55, 0x5f, + 0x44, 0x21, 0xd7, 0xd0, 0xb4, 0xac, 0x3a, 0x02, 0xbe, 0x8f, 0x42, 0x6a, 0xa6, 0x77, 0xb4, 0xae, + 0x05, 0x33, 0xaf, 0xa8, 0x69, 0x9f, 0x4e, 0x24, 0xfa, 0xc1, 0x1e, 0x9d, 0x9b, 0x6d, 0x58, 0x35, + 0x7a, 0xe7, 0xf2, 0x34, 0x72, 0xd8, 0x9f, 0x3b, 0x7e, 0x4a, 0xa2, 0x23, 0x9a, 0xa4, 0x9a, 0xe9, + 0xbb, 0x88, 0xf8, 0xce, 0xf1, 0xd3, 0x23, 0x06, 0x8b, 0x8f, 0xf4, 0x71, 0xf8, 0x22, 0x2d, 0xdd, + 0x8d, 0xc5, 0x61, 0x76, 0x31, 0x45, 0x7f, 0x28, 0xe9, 0xfa, 0xc7, 0xb1, 0x93, 0x26, 0xfe, 0x59, + 0x14, 0xa2, 0xc5, 0x45, 0xd7, 0x60, 0x64, 0x77, 0xa1, 0xd6, 0xad, 0xe5, 0x0c, 0xf9, 0x88, 0x70, + 0x54, 0x74, 0xf0, 0x04, 0x36, 0xe4, 0x59, 0x1c, 0xf8, 0xae, 0x3f, 0x75, 0x9d, 0x87, 0xed, 0x3a, + 0x2a, 0xb5, 0x13, 0x99, 0x4e, 0x92, 0x50, 0x51, 0x04, 0x59, 0xf3, 0xf5, 0xdb, 0x86, 0xbe, 0x78, + 0xc5, 0xc7, 0x8e, 0xa3, 0x52, 0x8b, 0x69, 0x1f, 0x4d, 0x82, 0x00, 0x27, 0x21, 0x2b, 0x76, 0xe0, + 0x3a, 0xc7, 0x9a, 0xd2, 0x65, 0x0e, 0x9f, 0x41, 0x7f, 0xea, 0x36, 0x8c, 0xec, 0x89, 0x38, 0x78, + 0xc1, 0x5a, 0x2b, 0x5d, 0x83, 0x91, 0x3d, 0xe0, 0xef, 0x9b, 0x6a, 0xa9, 0x03, 0x30, 0x7c, 0xa3, + 0xe9, 0x72, 0xa9, 0x99, 0xbe, 0x8a, 0xe5, 0x1d, 0xe8, 0x64, 0x8b, 0xc8, 0x56, 0xe8, 0x12, 0x5b, + 0x84, 0x66, 0x25, 0xf5, 0x39, 0xf6, 0x2e, 0xba, 0xe7, 0x32, 0x49, 0xa2, 0x44, 0x7b, 0x00, 0x82, + 0xb8, 0xb6, 0x3d, 0x76, 0xce, 0xee, 0x23, 0x94, 0x8d, 0xfe, 0x0f, 0x61, 0x79, 0x8a, 0x8e, 0x96, + 0x84, 0xa3, 0xea, 0xbd, 0x12, 0xed, 0xa1, 0x4c, 0x07, 0xff, 0xa5, 0x02, 0x4b, 0x17, 0x84, 0xaa, + 0x58, 0x87, 0x1a, 0x0a, 0x54, 0x53, 0x86, 0xd6, 0xb2, 0x16, 0xf1, 0x75, 0xcf, 0x23, 0x44, 0x3a, + 0x4e, 0x4d, 0x01, 0x1a, 0x22, 0xd2, 0x31, 0x6a, 0x8c, 0x55, 0x58, 0x4c, 0xcf, 0x88, 0xbb, 0x59, + 0x3d, 0x57, 0xd3, 0x33, 0x64, 0xeb, 0x6d, 0x68, 0x04, 0xd1, 0xc8, 0x0e, 0xe4, 0x33, 0xc9, 0xd7, + 0x49, 0x75, 0xb6, 0xde, 0x79, 0x85, 0x34, 0xdf, 0x7c, 0x18, 0x8d, 0x1e, 0x22, 0xad, 0x55, 0x0f, + 0xf4, 0xd3, 0xe0, 0xd7, 0x50, 0x37, 0x50, 0xd1, 0x80, 0xea, 0xae, 0x3c, 0x9e, 0x8c, 0x7a, 0x6f, + 0x88, 0x3a, 0x2c, 0x60, 0x8b, 0x5e, 0x05, 0x9f, 0xbe, 0x73, 0x92, 0xb0, 0x37, 0x87, 0x68, 0xfa, + 0x75, 0xbd, 0x79, 0x7c, 0x3c, 0x70, 0x42, 0xdf, 0xed, 0x2d, 0xe0, 0xe3, 0x03, 0x27, 0x75, 0x82, + 0x5e, 0x75, 0xf0, 0x1f, 0xaa, 0x50, 0x3f, 0xd0, 0x5f, 0x17, 0xbb, 0xd0, 0xce, 0xae, 0x65, 0x9d, + 0x1d, 0xef, 0x3f, 0x98, 0x7e, 0xa0, 0x78, 0x7f, 0x2b, 0x2e, 0xbc, 0x4d, 0x5f, 0xee, 0x3a, 0x77, + 0xe1, 0x72, 0xd7, 0xab, 0x30, 0xff, 0x34, 0x39, 0x2f, 0x9f, 0xc0, 0x38, 0x08, 0x9c, 0xd0, 0x42, + 0xb0, 0xf8, 0x08, 0x9a, 0x54, 0x64, 0xc2, 0x76, 0x86, 0x8e, 0x91, 0x17, 0x6f, 0x63, 0xe6, 0xda, + 0x7a, 0x40, 0x22, 0xed, 0xd2, 0x6c, 0x42, 0xdd, 0x3d, 0xf1, 0x03, 0x2f, 0x91, 0xa1, 0x3e, 0x09, + 0x25, 0x2e, 0x0e, 0xd9, 0xca, 0x68, 0xc4, 0xaf, 0xa0, 0xe7, 0xe7, 0x31, 0xfe, 0xbc, 0xa6, 0xa8, + 0xa4, 0x9a, 0x0b, 0x59, 0x00, 0xab, 0x5b, 0x20, 0x27, 0x1b, 0x3a, 0xbf, 0x6f, 0xa9, 0x56, 0xbc, + 0x6f, 0x89, 0xaf, 0xdb, 0x24, 0x43, 0xb7, 0x9e, 0x45, 0x08, 0xd1, 0xce, 0x7d, 0x57, 0x7b, 0x27, + 0x8d, 0xe9, 0x90, 0x8a, 0xb1, 0xad, 0xb5, 0x97, 0xf2, 0x0e, 0x74, 0xd0, 0xeb, 0xb1, 0xd9, 0x59, + 0x42, 0x93, 0x01, 0xf4, 0xf5, 0x70, 0x13, 0x75, 0xb2, 0x8b, 0xee, 0x12, 0x32, 0xe3, 0x4d, 0xe8, + 0x98, 0xdf, 0xa2, 0x19, 0xbf, 0xa9, 0x6b, 0x8e, 0x34, 0x94, 0x19, 0x7f, 0x13, 0x96, 0xdd, 0x13, + 0x27, 0x0c, 0x65, 0x60, 0x1f, 0x4f, 0x86, 0x43, 0x63, 0xa7, 0xf2, 0x9d, 0x81, 0x4b, 0x1a, 0x75, + 0x8f, 0x30, 0x64, 0xae, 0x0e, 0xa0, 0x1d, 0xfa, 0x81, 0xb9, 0x7d, 0x38, 0x64, 0x9f, 0xa2, 0x6a, + 0x35, 0x43, 0x3f, 0xe0, 0xfb, 0x86, 0xe9, 0x9a, 0xe4, 0xde, 0x64, 0xe2, 0x7b, 0xca, 0x4e, 0x23, + 0x73, 0x53, 0xa9, 0xce, 0xd6, 0x15, 0xe2, 0xdf, 0x4f, 0x26, 0xbe, 0x77, 0x14, 0xe9, 0xbb, 0x4a, + 0xdb, 0x44, 0x6f, 0x5e, 0x71, 0xec, 0x65, 0x57, 0x9a, 0x0a, 0x7f, 0xea, 0x56, 0x3b, 0x2a, 0x7a, + 0xd0, 0x83, 0xaf, 0xa0, 0x55, 0x64, 0x31, 0x64, 0x59, 0x8a, 0x63, 0xf6, 0xde, 0x10, 0x00, 0x8b, + 0x8f, 0xa2, 0x64, 0xec, 0x04, 0xbd, 0x0a, 0x3e, 0xb3, 0x7a, 0xeb, 0xcd, 0x89, 0x16, 0xd4, 0x4d, + 0x5c, 0xae, 0x37, 0xaf, 0x33, 0xe5, 0xbf, 0x80, 0xba, 0xb9, 0xa7, 0x95, 0xee, 0xb8, 0x8c, 0x3c, + 0xc9, 0x2e, 0xa6, 0x3e, 0x11, 0x80, 0x00, 0x72, 0x2f, 0xcd, 0xed, 0xd9, 0x73, 0xf9, 0xed, 0xd9, + 0x83, 0xdf, 0x40, 0xab, 0xf8, 0x4b, 0x4c, 0xee, 0xa7, 0x92, 0xe7, 0x7e, 0x66, 0xb4, 0xa2, 0x4a, + 0xb5, 0x24, 0x1a, 0xdb, 0x05, 0x2f, 0xa8, 0x8e, 0x00, 0xfc, 0xcc, 0xe0, 0xef, 0xcf, 0x41, 0x95, + 0xa2, 0x55, 0x64, 0x6f, 0xe2, 0x43, 0xbe, 0xd1, 0xaa, 0x56, 0x83, 0x20, 0xff, 0x07, 0x87, 0xf4, + 0xb3, 0x5a, 0x80, 0x85, 0x57, 0xd7, 0x02, 0x6c, 0xc3, 0xd2, 0xb3, 0xc2, 0xdd, 0xca, 0x1c, 0xa3, + 0xaa, 0x1a, 0xe3, 0x14, 0xdb, 0xfc, 0x36, 0xbf, 0x63, 0x99, 0x22, 0x55, 0xdd, 0x67, 0x65, 0x80, + 0x78, 0x0b, 0xf4, 0x79, 0x27, 0x9b, 0xcb, 0x0c, 0xb9, 0x26, 0xaf, 0xc9, 0xb0, 0x6d, 0x2a, 0x2a, + 0x7c, 0x13, 0x00, 0x25, 0xa0, 0x96, 0xf7, 0x5c, 0x86, 0xd9, 0x48, 0xcf, 0x42, 0x96, 0xf2, 0x83, + 0x31, 0x2c, 0xcf, 0x38, 0x9c, 0x29, 0x6e, 0x40, 0xab, 0x94, 0x31, 0xe5, 0x13, 0x53, 0xe0, 0xe6, + 0x29, 0xd2, 0x4f, 0x60, 0x4d, 0x06, 0xfe, 0xc8, 0x3f, 0xf6, 0xa9, 0x16, 0xbd, 0x70, 0x5e, 0x94, + 0x65, 0xcd, 0x4a, 0x01, 0x9b, 0x9d, 0x17, 0x1d, 0xfc, 0x59, 0x05, 0x96, 0x67, 0x94, 0xae, 0xbc, + 0xee, 0x70, 0xcb, 0x9b, 0x00, 0xa6, 0x46, 0x24, 0xf3, 0x73, 0x1b, 0x1a, 0xb2, 0x47, 0xee, 0xae, + 0x4c, 0x9d, 0x91, 0x31, 0x6f, 0xf0, 0x39, 0x73, 0x48, 0x17, 0x0a, 0x0e, 0xe9, 0xa7, 0xb0, 0x1e, + 0xa0, 0x6e, 0x1e, 0x47, 0x9e, 0x3f, 0xf4, 0xa5, 0x57, 0xb8, 0x6c, 0x91, 0xfd, 0xc6, 0x15, 0x44, + 0xef, 0x6b, 0xac, 0xb9, 0x6c, 0x71, 0xf0, 0x67, 0x73, 0xb0, 0x3a, 0xb3, 0x2c, 0xe6, 0x75, 0xc3, + 0xbe, 0x0d, 0xa2, 0x5c, 0x73, 0x53, 0x38, 0x96, 0xd3, 0x4b, 0x0a, 0x9d, 0xd1, 0xb1, 0x8e, 0x0d, + 0xe8, 0x95, 0xa8, 0xf3, 0xc3, 0x39, 0x9d, 0x02, 0x2d, 0x4a, 0xb8, 0x3d, 0x78, 0xcb, 0x94, 0x8b, + 0xdb, 0x9e, 0x4f, 0x02, 0x11, 0x7d, 0x47, 0x8a, 0xf4, 0xe0, 0x58, 0x7c, 0x57, 0x9a, 0x33, 0x01, + 0xd7, 0x0c, 0xe1, 0x6e, 0x46, 0x47, 0xc5, 0x22, 0x7b, 0x4c, 0x85, 0x92, 0xa0, 0x74, 0xe1, 0x9a, + 0x39, 0xd8, 0xd3, 0x2e, 0xde, 0xb5, 0xa6, 0xc4, 0x5d, 0x58, 0xc9, 0xc9, 0x9e, 0xfb, 0x89, 0xae, + 0xb9, 0xd5, 0xee, 0xb4, 0xc8, 0x70, 0xdf, 0xf9, 0x09, 0x57, 0xdc, 0xde, 0xfa, 0x37, 0x15, 0x58, + 0xe4, 0xfb, 0xf8, 0xc5, 0x12, 0xb4, 0x9f, 0x84, 0xa7, 0x61, 0xf4, 0x3c, 0x64, 0x40, 0xef, 0x0d, + 0xb1, 0x0c, 0x5d, 0x23, 0x59, 0xf4, 0xc5, 0xff, 0xbd, 0x8a, 0xe8, 0x41, 0x8b, 0x7e, 0xa2, 0x81, + 0xcc, 0x89, 0xab, 0xd0, 0xd7, 0x0e, 0xd1, 0x2e, 0x1a, 0x1c, 0x51, 0xea, 0x0f, 0xcf, 0x0d, 0x76, + 0x5e, 0x74, 0xa1, 0x79, 0x98, 0x46, 0xf1, 0xa1, 0x0c, 0x3d, 0x3f, 0x1c, 0xf5, 0x16, 0x44, 0x1f, + 0x56, 0x4c, 0xaf, 0x2c, 0xc1, 0x1e, 0xf8, 0xa1, 0xaf, 0x4e, 0x7a, 0x55, 0x71, 0x05, 0xd6, 0x67, + 0x61, 0xb6, 0xdd, 0xd3, 0xde, 0xa2, 0x58, 0x81, 0x9e, 0x41, 0xde, 0xd3, 0x57, 0xa8, 0xf7, 0x6a, + 0xb7, 0x3e, 0x01, 0x71, 0xf1, 0xe2, 0x7b, 0xfc, 0xe6, 0x43, 0x39, 0x72, 0xdc, 0xf3, 0x9d, 0x20, + 0x52, 0x28, 0x08, 0xdb, 0xd0, 0xc8, 0xfb, 0xaa, 0xdc, 0x7a, 0x00, 0x8b, 0xfc, 0x4f, 0x05, 0x85, + 0x5f, 0xcd, 0x80, 0xde, 0x1b, 0xd8, 0x18, 0x0d, 0x4f, 0x3f, 0x1c, 0x3d, 0x92, 0x67, 0x29, 0xdb, + 0x08, 0x0f, 0x1d, 0x95, 0xf6, 0xe6, 0x44, 0x07, 0x40, 0xff, 0xb0, 0xfb, 0xa1, 0xd7, 0x9b, 0xbf, + 0xf5, 0x8d, 0xde, 0x27, 0x53, 0x25, 0x62, 0xd7, 0xe0, 0xb2, 0xee, 0x74, 0x06, 0xb6, 0xf7, 0x06, + 0x7e, 0x94, 0x10, 0xa8, 0xef, 0x76, 0x9d, 0xd4, 0xe9, 0x55, 0xee, 0xed, 0xfc, 0xc5, 0x1f, 0xae, + 0x55, 0xfe, 0xf2, 0x0f, 0xd7, 0x2a, 0xff, 0xe9, 0x0f, 0xd7, 0xde, 0xf8, 0xd3, 0x3f, 0x5e, 0xab, + 0x7c, 0xff, 0x51, 0xe1, 0x1f, 0x1d, 0xb4, 0x65, 0x4b, 0x75, 0xb9, 0x77, 0x32, 0x33, 0xf7, 0x4e, + 0x7c, 0x3a, 0xba, 0x13, 0x1f, 0xdf, 0x31, 0xca, 0xe4, 0x78, 0x91, 0xfe, 0xa8, 0xe1, 0xe3, 0xff, + 0x1d, 0x00, 0x00, 0xff, 0xff, 0x44, 0x34, 0xcc, 0x1e, 0x27, 0x62, 0x00, 0x00, } func (m *Message) Marshal() (dAtA []byte, err error) { @@ -7597,6 +7609,18 @@ func (m *Message) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.BatchAckS3OwnershipRetained { + i-- + if m.BatchAckS3OwnershipRetained { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x1 + i-- + dAtA[i] = 0x98 + } if m.BatchAckSequence != 0 { i = encodeVarintPipeline(dAtA, i, uint64(m.BatchAckSequence)) i-- @@ -14581,6 +14605,9 @@ func (m *Message) ProtoSize() (n int) { if m.BatchAckSequence != 0 { n += 2 + sovPipeline(uint64(m.BatchAckSequence)) } + if m.BatchAckS3OwnershipRetained { + n += 3 + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -17850,6 +17877,26 @@ func (m *Message) Unmarshal(dAtA []byte) error { break } } + case 19: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field BatchAckS3OwnershipRetained", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPipeline + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.BatchAckS3OwnershipRetained = bool(v != 0) default: iNdEx = preIndex skippy, err := skipPipeline(dAtA[iNdEx:]) diff --git a/pkg/pb/pipeline/pipeline_s3_ack_test.go b/pkg/pb/pipeline/pipeline_s3_ack_test.go new file mode 100644 index 0000000000000..1b011c193848a --- /dev/null +++ b/pkg/pb/pipeline/pipeline_s3_ack_test.go @@ -0,0 +1,56 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pipeline + +import ( + "testing" + + "github.com/gogo/protobuf/proto" + "github.com/stretchr/testify/require" +) + +// The pre-field-19 ACK shape exercises gogo's unknown-field wire handling. +type legacyS3Ack struct { + Cmd uint32 `protobuf:"varint,2,opt,name=cmd,proto3"` + Sequence uint64 `protobuf:"varint,18,opt,name=batch_ack_sequence,proto3"` +} + +func (m *legacyS3Ack) Reset() { *m = legacyS3Ack{} } +func (m *legacyS3Ack) String() string { return proto.CompactTextString(m) } +func (*legacyS3Ack) ProtoMessage() {} + +func TestS3OwnershipAckWire(t *testing.T) { + for _, retained := range []bool{false, true} { + msg := &Message{Cmd: Method_PipelineBatchAck, BatchAckSequence: 7, BatchAckS3OwnershipRetained: retained} + data, err := msg.Marshal() + require.NoError(t, err) + require.Equal(t, msg.ProtoSize(), len(data)) + var decoded Message + require.NoError(t, decoded.Unmarshal(data)) + require.Equal(t, retained, decoded.GetBatchAckS3OwnershipRetained()) + require.Equal(t, uint64(7), decoded.GetBatchAckSequence()) + var oldDecoder legacyS3Ack + require.NoError(t, proto.Unmarshal(data, &oldDecoder)) + require.Equal(t, uint32(Method_PipelineBatchAck), oldDecoder.Cmd) + require.Equal(t, uint64(7), oldDecoder.Sequence) + decoded.Reset() + require.False(t, decoded.GetBatchAckS3OwnershipRetained()) + } + // Legacy ACK: field 2 (command 7), field 18 (sequence 7); no field 19. + var legacy Message + require.NoError(t, legacy.Unmarshal([]byte{0x10, 7, 0x90, 1, 7})) + require.Equal(t, uint64(7), legacy.GetBatchAckSequence()) + require.False(t, legacy.GetBatchAckS3OwnershipRetained()) +} diff --git a/pkg/sql/colexec/deletion/deletion.go b/pkg/sql/colexec/deletion/deletion.go index 809a1e0bae3d3..c4742aea601e2 100644 --- a/pkg/sql/colexec/deletion/deletion.go +++ b/pkg/sql/colexec/deletion/deletion.go @@ -50,6 +50,10 @@ func (deletion *Deletion) OpType() vm.OpType { } func (deletion *Deletion) Prepare(proc *process.Process) error { + if err := deletion.retryPendingS3Cleanup(proc); err != nil { + return err + } + if deletion.OpAnalyzer == nil { deletion.OpAnalyzer = process.NewAnalyzer(deletion.GetIdx(), deletion.IsFirst, deletion.IsLast, "deletion") } else { diff --git a/pkg/sql/colexec/deletion/deletion_test.go b/pkg/sql/colexec/deletion/deletion_test.go index 6cfd5c24be07e..e197d4237c46c 100644 --- a/pkg/sql/colexec/deletion/deletion_test.go +++ b/pkg/sql/colexec/deletion/deletion_test.go @@ -18,21 +18,26 @@ import ( "bytes" "context" "errors" + "fmt" "testing" "time" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" + "github.com/matrixorigin/matrixone/pkg/common/moerr" "github.com/matrixorigin/matrixone/pkg/common/mpool" "github.com/matrixorigin/matrixone/pkg/container/batch" "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/matrixorigin/matrixone/pkg/container/vector" + "github.com/matrixorigin/matrixone/pkg/fileservice" mock_frontend "github.com/matrixorigin/matrixone/pkg/frontend/test" + "github.com/matrixorigin/matrixone/pkg/objectio" "github.com/matrixorigin/matrixone/pkg/pb/txn" "github.com/matrixorigin/matrixone/pkg/sql/colexec" "github.com/matrixorigin/matrixone/pkg/sql/plan" "github.com/matrixorigin/matrixone/pkg/testutil" + "github.com/matrixorigin/matrixone/pkg/txn/client" "github.com/matrixorigin/matrixone/pkg/vm" "github.com/matrixorigin/matrixone/pkg/vm/engine" "github.com/matrixorigin/matrixone/pkg/vm/process" @@ -207,9 +212,165 @@ func TestNewDeletionTombstoneWriterUsesProcessService(t *testing.T) { require.NoError(t, writer.Close()) } -func TestFlushCreatesTombstoneWriterForFirstBlock(t *testing.T) { +func TestRemoteDeleteFlushTransfersOwnershipBeforeReset(t *testing.T) { + for _, accepted := range []bool{false, true} { + t.Run(fmt.Sprintf("accepted=%v", accepted), func(t *testing.T) { + proc := testutil.NewProc(t) + defer proc.Free() + ctr, objectName := flushTombstoneObjectForTest(t, proc) + workspace := proc.GetTxnOperator().GetWorkspace().(*deletionS3CleanupWorkspace) + require.Len(t, workspace.owners, 1) + require.Empty(t, ctr.s3Writers, "completed writer buffers must be released") + if accepted { + workspace.AcceptUnpublishedS3ObjectNames(objectName) + } + arg := &Deletion{RemoteDelete: true, ctr: *ctr} + // Both success and later failure teardown must leave transferred + // objects alone. Only the workspace knows which names registered. + arg.Reset(proc, false, nil) + arg.Free(proc, true, errors.New("consumer failed")) + _, err := ctr.fs.StatFile(proc.Ctx, objectName) + require.NoError(t, err) + require.NoError(t, workspace.Cleanup(proc.Ctx)) + _, err = ctr.fs.StatFile(proc.Ctx, objectName) + if accepted { + require.NoError(t, err) + } else { + require.True(t, moerr.IsMoErrCode(err, moerr.ErrFileNotFound), "%v", err) + } + }) + } +} + +type deletionS3CleanupWorkspace struct { + client.Workspace + owners []*colexec.UnpublishedS3ObjectOwner + cleanups []func(context.Context) error +} + +func (w *deletionS3CleanupWorkspace) RetainUnpublishedS3Cleanup(cleanup func(context.Context) error) { + w.cleanups = append(w.cleanups, cleanup) +} + +func TestRemoteDeleteFreeRetainsFailedWriterCleanup(t *testing.T) { + t.Run("Free", func(t *testing.T) { checkRemoteDeleteFreeRetainsFailedWriterCleanup(t, false) }) + t.Run("prepared Reset", func(t *testing.T) { checkRemoteDeleteFreeRetainsFailedWriterCleanup(t, true) }) +} + +func checkRemoteDeleteFreeRetainsFailedWriterCleanup(t *testing.T, prepared bool) { + proc := testutil.NewProc(t) + defer proc.Free() + baseFS, err := colexec.GetSharedFSFromProc(proc) + require.NoError(t, err) + deleteErr := errors.New("injected remote delete cleanup failure") + fs := &failOnceDeleteFileService{FileService: baseFS, failErr: deleteErr, failCount: 2} + proc.Base.FileService = fs + ctr, name := flushTombstoneObjectForTest(t, proc, true) + require.Len(t, ctr.s3Writers, 1, "failed producer handoff must keep the writer") + + workspace := &deletionS3CleanupWorkspace{} + txnOp := mock_frontend.NewMockTxnOperator(gomock.NewController(t)) + txnOp.EXPECT().GetWorkspace().Return(workspace).AnyTimes() + proc.Base.TxnOperator = txnOp + arg := &Deletion{RemoteDelete: true, ctr: *ctr} + if prepared { + arg.Reset(proc, true, deleteErr) + } else { + arg.Free(proc, true, deleteErr) + } + require.Empty(t, arg.ctr.s3Writers) + require.Len(t, workspace.cleanups, 1, "transaction must own the failed cleanup") + _, err = baseFS.StatFile(proc.Ctx, name) + require.NoError(t, err) + require.NoError(t, workspace.cleanups[0](proc.Ctx)) + _, err = baseFS.StatFile(proc.Ctx, name) + require.True(t, moerr.IsMoErrCode(err, moerr.ErrFileNotFound), "%v", err) +} + +func TestRemoteDeleteWorkspaceRetriesCleanupAfterProducerReuse(t *testing.T) { + proc := testutil.NewProc(t) + defer proc.Free() + baseFS, err := colexec.GetSharedFSFromProc(proc) + require.NoError(t, err) + deleteErr := errors.New("injected tombstone cleanup failure") + fs := &failOnceDeleteFileService{FileService: baseFS, failErr: deleteErr, failCount: 1} + proc.Base.FileService = fs + ctr, name := flushTombstoneObjectForTest(t, proc) + workspace := proc.GetTxnOperator().GetWorkspace().(*deletionS3CleanupWorkspace) + arg := &Deletion{RemoteDelete: true, ctr: *ctr} + arg.Reset(proc, true, errors.New("consumer failed")) + require.ErrorIs(t, workspace.Cleanup(proc.Ctx), deleteErr) + require.True(t, workspace.HasUnpublishedS3ObjectOwners()) + require.Equal(t, []string{name}, workspace.owners[0].Names()) + _, err = fs.StatFile(proc.Ctx, name) + require.NoError(t, err) + // Reuse is safe now: the workspace outlives the reusable operator and keeps + // the retry ledger, without retaining old stats batches or writer buffers. + require.NoError(t, arg.retryPendingS3Cleanup(proc)) + arg.Free(proc, false, nil) + require.True(t, workspace.HasUnpublishedS3ObjectOwners()) + require.NoError(t, workspace.Cleanup(proc.Ctx)) + require.False(t, workspace.HasUnpublishedS3ObjectOwners()) + _, err = fs.StatFile(proc.Ctx, name) + require.True(t, moerr.IsMoErrCode(err, moerr.ErrFileNotFound), "%v", err) +} + +func TestRemoteDeleteMissingWorkspacePreservesWriterCleanup(t *testing.T) { proc := testutil.NewProc(t) defer proc.Free() + baseFS, err := colexec.GetSharedFSFromProc(proc) + require.NoError(t, err) + deleteErr := errors.New("injected writer cleanup failure") + fs := &failOnceDeleteFileService{FileService: baseFS, failErr: deleteErr, failCount: 1} + proc.Base.FileService = fs + ctr, name := flushTombstoneObjectForTest(t, proc, true) + require.Len(t, ctr.s3Writers, 1, "failed retention and deletion must keep the writer") + arg := &Deletion{RemoteDelete: true, ctr: *ctr} + arg.Reset(proc, false, nil) + require.Empty(t, arg.ctr.s3Writers) + _, err = fs.StatFile(proc.Ctx, name) + require.True(t, moerr.IsMoErrCode(err, moerr.ErrFileNotFound), "%v", err) + arg.Free(proc, false, nil) +} + +func (w *deletionS3CleanupWorkspace) RetainUnpublishedS3ObjectOwner(owner *colexec.UnpublishedS3ObjectOwner) { + w.owners = append(w.owners, owner) +} +func (w *deletionS3CleanupWorkspace) AcceptUnpublishedS3ObjectNames(names ...string) { + for _, owner := range w.owners { + owner.Accept(names...) + } +} +func (w *deletionS3CleanupWorkspace) HasUnpublishedS3ObjectOwners() bool { + for _, owner := range w.owners { + if owner.Pending() { + return true + } + } + return false +} +func (w *deletionS3CleanupWorkspace) AcceptAllUnpublishedS3ObjectOwners() { + for _, owner := range w.owners { + owner.AcceptAll() + } +} +func (w *deletionS3CleanupWorkspace) Cleanup(ctx context.Context) error { + for _, owner := range w.owners { + if err := owner.Cleanup(ctx); err != nil { + return err + } + } + return nil +} + +func flushTombstoneObjectForTest(t *testing.T, proc *process.Process, missingWorkspace ...bool) (*container, string) { + t.Helper() + if len(missingWorkspace) == 0 { + ctrl := gomock.NewController(t) + txnOp := mock_frontend.NewMockTxnOperator(ctrl) + txnOp.EXPECT().GetWorkspace().Return(&deletionS3CleanupWorkspace{}).AnyTimes() + proc.Base.TxnOperator = txnOp + } blockID := types.BuildTestBlockid(1, 1) rowID := types.NewRowid(&blockID, 0) bat := batch.NewWithSize(2) @@ -219,7 +380,7 @@ func TestFlushCreatesTombstoneWriterForFirstBlock(t *testing.T) { require.NoError(t, vector.AppendFixed(bat.Vecs[1], int64(7), false, proc.Mp())) bat.SetRowCount(1) - ctr := container{ + ctr := &container{ partitionId_blockId_rowIdBatch: map[int]map[types.Blockid]*batch.Batch{ 0: {blockID: bat}, }, @@ -228,14 +389,33 @@ func TestFlushCreatesTombstoneWriterForFirstBlock(t *testing.T) { pool: &BatchPool{}, } size, err := ctr.flush(proc, process.NewAnalyzer(0, false, false, "deletion-flush")) - require.NoError(t, err) - require.NotZero(t, size) + if len(missingWorkspace) == 0 { + require.NoError(t, err) + require.NotZero(t, size) + } else { + require.ErrorContains(t, err, "transaction workspace cannot retain") + } require.Empty(t, ctr.partitionId_blockId_rowIdBatch[0]) require.Len(t, ctr.partitionId_tombstoneObjectStatsBats[0], 1) - for _, statsBat := range ctr.partitionId_tombstoneObjectStatsBats[0] { - statsBat.Clean(proc.Mp()) - } - for _, pooled := range ctr.pool.pools { - pooled.Clean(proc.Mp()) + statsBat := ctr.partitionId_tombstoneObjectStatsBats[0][0] + data, area := vector.MustVarlenaRawData(statsBat.Vecs[0]) + stats := objectio.ObjectStats(data[0].GetByteSlice(area)) + objectName := stats.ObjectName().String() + _, err = ctr.fs.StatFile(proc.Ctx, objectName) + require.NoError(t, err) + return ctr, objectName +} + +type failOnceDeleteFileService struct { + fileservice.FileService + failErr error + failCount int +} + +func (fs *failOnceDeleteFileService) Delete(ctx context.Context, names ...string) error { + if fs.failCount > 0 { + fs.failCount-- + return fs.failErr } + return fs.FileService.Delete(ctx, names...) } diff --git a/pkg/sql/colexec/deletion/types.go b/pkg/sql/colexec/deletion/types.go index 13d5c1b2469e2..0217165092448 100644 --- a/pkg/sql/colexec/deletion/types.go +++ b/pkg/sql/colexec/deletion/types.go @@ -15,6 +15,8 @@ package deletion import ( + "context" + "errors" "slices" "github.com/matrixorigin/matrixone/pkg/catalog" @@ -25,6 +27,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/matrixorigin/matrixone/pkg/container/vector" "github.com/matrixorigin/matrixone/pkg/fileservice" + "github.com/matrixorigin/matrixone/pkg/logutil" "github.com/matrixorigin/matrixone/pkg/objectio" "github.com/matrixorigin/matrixone/pkg/perfcounter" "github.com/matrixorigin/matrixone/pkg/sql/colexec" @@ -33,6 +36,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/vm/engine" "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/options" "github.com/matrixorigin/matrixone/pkg/vm/process" + "go.uber.org/zap" ) var _ vm.Operator = new(Deletion) @@ -69,6 +73,8 @@ type container struct { blockId_bitmap map[types.Blockid]*nulls.Nulls partitionId_blockId_rowIdBatch map[int]map[types.Blockid]*batch.Batch // PartitionId -> blockId -> RowIdBatch partitionId_tombstoneObjectStatsBats map[int][]*batch.Batch // PartitionId -> tombstone object stats + fs fileservice.FileService + s3Writers []*colexec.CNS3Writer // don't flush cn block rowId and rawBatch // we just do compaction for cn block in the // future @@ -136,6 +142,10 @@ func NewArgument() *Deletion { func (deletion *Deletion) Release() { if deletion != nil { + if len(deletion.ctr.s3Writers) != 0 || + len(deletion.ctr.partitionId_tombstoneObjectStatsBats) != 0 { + return + } reuse.Free[Deletion](deletion, nil) } } @@ -153,6 +163,12 @@ type DeleteCtx struct { func (deletion *Deletion) Reset(proc *process.Process, pipelineFailed bool, err error) { ctr := &deletion.ctr ctr.state = vm.Build + var cleanupErr error + ctr.s3Writers, cleanupErr = closeS3Writers(proc.Ctx, ctr.s3Writers, pipelineFailed) + if cleanupErr != nil { + logutil.Warn("failed to clean remote delete S3 writers", zap.Error(cleanupErr)) + deletion.retainPendingS3Writers(proc) + } if deletion.RemoteDelete { for k := range ctr.blockId_bitmap { delete(ctr.blockId_bitmap, k) @@ -167,14 +183,7 @@ func (deletion *Deletion) Reset(proc *process.Process, pipelineFailed bool, err delete(ctr.partitionId_blockId_rowIdBatch, pidx) } - for pIdx, bats := range ctr.partitionId_tombstoneObjectStatsBats { - for _, bat := range bats { - if bat != nil { - bat.Clean(proc.GetMPool()) - } - } - delete(ctr.partitionId_tombstoneObjectStatsBats, pIdx) - } + ctr.cleanTombstoneStats(proc.GetMPool()) for blkid := range ctr.blockId_type { delete(ctr.blockId_type, blkid) @@ -196,11 +205,20 @@ func (deletion *Deletion) Reset(proc *process.Process, pipelineFailed bool, err // delete from t1 using t1 join t2 on t1.a = t2.a; func (deletion *Deletion) Free(proc *process.Process, pipelineFailed bool, err error) { ctr := &deletion.ctr + var cleanupErr error + ctr.s3Writers, cleanupErr = closeS3Writers(proc.Ctx, ctr.s3Writers, pipelineFailed) + if cleanupErr != nil { + logutil.Warn("failed to clean remote delete S3 writers", zap.Error(cleanupErr)) + deletion.retainPendingS3Writers(proc) + } if deletion.RemoteDelete { + ctr.cleanTombstoneStats(proc.GetMPool()) deletion.SegmentMap = nil ctr.blockId_bitmap = nil ctr.partitionId_blockId_rowIdBatch = nil - ctr.partitionId_tombstoneObjectStatsBats = nil + if len(ctr.partitionId_tombstoneObjectStatsBats) == 0 { + ctr.partitionId_tombstoneObjectStatsBats = nil + } ctr.blockId_type = nil ctr.pool = nil } @@ -213,6 +231,65 @@ func (deletion *Deletion) Free(proc *process.Process, pipelineFailed bool, err e ctr.source = nil } +func closeS3Writers( + ctx context.Context, + writers []*colexec.CNS3Writer, + pipelineFailed bool, +) ([]*colexec.CNS3Writer, error) { + pending := make([]*colexec.CNS3Writer, 0, len(writers)) + var cleanupErrs []error + for _, writer := range writers { + if writer == nil { + continue + } + if err := writer.CloseWithCleanup(ctx, pipelineFailed); err != nil { + pending = append(pending, writer) + cleanupErrs = append(cleanupErrs, err) + } + } + return pending, errors.Join(cleanupErrs...) +} + +func (deletion *Deletion) retainPendingS3Writers(proc *process.Process) { + pending := make([]*colexec.CNS3Writer, 0, len(deletion.ctr.s3Writers)) + for _, writer := range deletion.ctr.s3Writers { + if writer == nil { + continue + } + if colexec.RetainUnpublishedS3Cleanup(proc, func(ctx context.Context) error { + return writer.CloseWithCleanup(ctx, true) + }) { + continue + } + pending = append(pending, writer) + } + deletion.ctr.s3Writers = pending +} + +func (ctr *container) cleanTombstoneStats(mp *mpool.MPool) { + for pIdx, bats := range ctr.partitionId_tombstoneObjectStatsBats { + for _, bat := range bats { + if bat != nil { + bat.Clean(mp) + } + } + delete(ctr.partitionId_tombstoneObjectStatsBats, pIdx) + } +} + +func (deletion *Deletion) retryPendingS3Cleanup(proc *process.Process) error { + var cleanupErrs []error + var err error + deletion.ctr.s3Writers, err = closeS3Writers(proc.Ctx, deletion.ctr.s3Writers, false) + if err != nil { + cleanupErrs = append(cleanupErrs, err) + } + if deletion.RemoteDelete { + deletion.ctr.cleanTombstoneStats(proc.GetMPool()) + } + return errors.Join(cleanupErrs...) +} + func (deletion *Deletion) ExecProjection(proc *process.Process, input *batch.Batch) (*batch.Batch, error) { return input, nil } @@ -241,6 +318,7 @@ func (ctr *container) flush(proc *process.Process, analyzer process.Analyzer) (u if err != nil { return 0, err } + ctr.fs = fs resSize := uint32(0) for pidx, blockId_rowIdBatch := range ctr.partitionId_blockId_rowIdBatch { @@ -258,12 +336,18 @@ func (ctr *container) flush(proc *process.Process, analyzer process.Analyzer) (u statsList []objectio.ObjectStats s3writer *colexec.CNS3Writer ) - - defer func() { - if s3writer != nil { - s3writer.Close() + abortWriter := func(cause error) error { + if s3writer == nil { + return cause + } + if cleanupErr := s3writer.CloseWithCleanup(proc.Ctx, true); cleanupErr != nil { + ctr.s3Writers = append(ctr.s3Writers, s3writer) + logutil.Warn("remote deletion cleanup retained for retry", zap.Error(cleanupErr)) + return cause } - }() + s3writer = nil + return cause + } slices.SortFunc(blkids, func(a, b types.Blockid) int { return a.Compare(&b) @@ -277,7 +361,7 @@ func (ctr *container) flush(proc *process.Process, analyzer process.Analyzer) (u } if err = s3writer.Write(proc.Ctx, bat); err != nil { - return 0, err + return 0, abortWriter(err) } resSize += uint32(bat.Size()) @@ -295,7 +379,7 @@ func (ctr *container) flush(proc *process.Process, analyzer process.Analyzer) (u if statsList, err = process.MeasureFilesystemWait(analyzer, func() ([]objectio.ObjectStats, error) { return s3writer.Sync(newCtx) }); err != nil { - return 0, err + return 0, abortWriter(err) } for _, stats := range statsList { @@ -303,13 +387,22 @@ func (ctr *container) flush(proc *process.Process, analyzer process.Analyzer) (u bat.SetVector(0, vector.NewVec(types.T_text.ToType())) if err = vector.AppendBytes( bat.GetVector(0), stats.Marshal(), false, proc.GetMPool()); err != nil { - return 0, err + bat.Clean(proc.GetMPool()) + return 0, abortWriter(err) } bat.SetRowCount(bat.Vecs[0].Length()) ctr.partitionId_tombstoneObjectStatsBats[pidx] = append(ctr.partitionId_tombstoneObjectStatsBats[pidx], bat) } + if err = s3writer.TransferPersistedObjects(proc); err != nil { + return 0, abortWriter(err) + } + if err = s3writer.Close(); err != nil { + ctr.s3Writers = append(ctr.s3Writers, s3writer) + return 0, err + } + s3writer = nil } return resSize, nil } diff --git a/pkg/sql/colexec/insert/insert.go b/pkg/sql/colexec/insert/insert.go index eea48874e6a22..7bbf578a2f2a4 100644 --- a/pkg/sql/colexec/insert/insert.go +++ b/pkg/sql/colexec/insert/insert.go @@ -118,6 +118,10 @@ func (insert *Insert) OpType() vm.OpType { } func (insert *Insert) Prepare(proc *process.Process) error { + if err := insert.retryPendingS3Writers(proc); err != nil { + return err + } + if insert.OpAnalyzer == nil { insert.OpAnalyzer = process.NewAnalyzer(insert.GetIdx(), insert.IsFirst, insert.IsLast, "insert") } else { @@ -499,6 +503,9 @@ func (insert *Insert) flushS3WriterOnMemoryPressure(proc *process.Process, analy insert.ctr.s3Writer.ResetBlockInfoBat() } + if err = insert.ctr.s3Writer.TransferPersistedObjects(proc); err != nil { + return err + } insert.refreshAndReleaseS3MemGrant() return nil } @@ -616,6 +623,9 @@ func flushTailBatch( return err } + if err = writer.TransferPersistedObjects(proc); err != nil { + return err + } writer.ResetBlockInfoBat() return nil diff --git a/pkg/sql/colexec/insert/insert_test.go b/pkg/sql/colexec/insert/insert_test.go index 3765385bccf7f..4c2cb1fa6624e 100644 --- a/pkg/sql/colexec/insert/insert_test.go +++ b/pkg/sql/colexec/insert/insert_test.go @@ -17,21 +17,26 @@ package insert import ( "context" "errors" + "fmt" goruntime "runtime" "testing" "time" "github.com/golang/mock/gomock" "github.com/matrixorigin/matrixone/pkg/catalog" + "github.com/matrixorigin/matrixone/pkg/common/moerr" "github.com/matrixorigin/matrixone/pkg/common/runtime" "github.com/matrixorigin/matrixone/pkg/container/batch" "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/matrixorigin/matrixone/pkg/container/vector" + "github.com/matrixorigin/matrixone/pkg/fileservice" mock_frontend "github.com/matrixorigin/matrixone/pkg/frontend/test" + "github.com/matrixorigin/matrixone/pkg/objectio" "github.com/matrixorigin/matrixone/pkg/objectio/ioutil" "github.com/matrixorigin/matrixone/pkg/pb/plan" "github.com/matrixorigin/matrixone/pkg/sql/colexec" "github.com/matrixorigin/matrixone/pkg/testutil" + "github.com/matrixorigin/matrixone/pkg/txn/client" "github.com/matrixorigin/matrixone/pkg/vm" "github.com/matrixorigin/matrixone/pkg/vm/engine" "github.com/matrixorigin/matrixone/pkg/vm/process" @@ -44,6 +49,62 @@ type relationHandleFactory struct { newHandleCalls int } +type insertS3CleanupWorkspace struct { + client.Workspace + cleanups []func(context.Context) error + owners []*colexec.UnpublishedS3ObjectOwner +} + +func (w *insertS3CleanupWorkspace) RetainUnpublishedS3ObjectOwner(owner *colexec.UnpublishedS3ObjectOwner) { + w.owners = append(w.owners, owner) +} +func (w *insertS3CleanupWorkspace) AcceptUnpublishedS3ObjectNames(names ...string) { + for _, owner := range w.owners { + owner.Accept(names...) + } +} +func (w *insertS3CleanupWorkspace) HasUnpublishedS3ObjectOwners() bool { + for _, owner := range w.owners { + if owner.Pending() { + return true + } + } + return false +} +func (w *insertS3CleanupWorkspace) AcceptAllUnpublishedS3ObjectOwners() { + for _, owner := range w.owners { + owner.AcceptAll() + } +} +func installInsertOwnershipWorkspace(t *testing.T, proc *process.Process) *insertS3CleanupWorkspace { + t.Helper() + w := &insertS3CleanupWorkspace{} + txnOp := mock_frontend.NewMockTxnOperator(gomock.NewController(t)) + txnOp.EXPECT().GetWorkspace().Return(w).AnyTimes() + proc.Base.TxnOperator = txnOp + return w +} + +func (w *insertS3CleanupWorkspace) RetainUnpublishedS3Cleanup( + cleanup func(context.Context) error, +) { + w.cleanups = append(w.cleanups, cleanup) +} + +type failOnceInsertDeleteFileService struct { + fileservice.FileService + failErr error +} + +func (fs *failOnceInsertDeleteFileService) Delete(ctx context.Context, paths ...string) error { + if fs.failErr != nil { + err := fs.failErr + fs.failErr = nil + return err + } + return fs.FileService.Delete(ctx, paths...) +} + func (f *relationHandleFactory) NewRelationHandle() engine.Relation { f.newHandleCalls++ return f.handle @@ -408,6 +469,7 @@ func TestInsertPrepareToWriteS3WithPipelineFlush(t *testing.T) { func TestInsertFlushS3WriterOnMemoryPressureAppendsBlockInfo(t *testing.T) { proc := testutil.NewProc(t) defer proc.Free() + installInsertOwnershipWorkspace(t, proc) fs, err := colexec.GetSharedFSFromProc(proc) require.NoError(t, err) @@ -448,6 +510,233 @@ func TestInsertFlushS3WriterOnMemoryPressureAppendsBlockInfo(t *testing.T) { require.Greater(t, insert.ctr.buf.RowCount(), 0) } +func TestInsertProducerTransfersPersistedOwnership(t *testing.T) { + for _, pressure := range []bool{false, true} { + for _, missingWorkspace := range []string{"present", "missing", "retry"} { + t.Run(fmt.Sprintf("pressure=%v/missingWorkspace=%v", pressure, missingWorkspace), func(t *testing.T) { + proc := testutil.NewProc(t) + defer proc.Free() + var workspace *insertS3CleanupWorkspace + if missingWorkspace == "present" { + workspace = installInsertOwnershipWorkspace(t, proc) + } + fs, err := colexec.GetSharedFSFromProc(proc) + require.NoError(t, err) + arg := &Insert{ctr: container{s3Writer: colexec.NewCNS3DataWriter(proc.Mp(), fs, testInsertS3TableDef(), 1, false)}} + arg.initBufForS3() + defer arg.Free(proc, true, nil) + input := &batch.Batch{Attrs: []string{"a", "b"}, Vecs: []*vector.Vector{ + testutil.MakeInt64Vector([]int64{1}, nil, proc.Mp()), + testutil.MakeVarcharVector([]string{"x"}, nil, proc.Mp()), + }} + input.SetRowCount(1) + defer input.Clean(proc.Mp()) + analyzer := process.NewAnalyzer(0, false, false, "insert-owner") + var names []string + for i := 0; i < 2; i++ { + require.NoError(t, arg.ctr.s3Writer.Write(proc.Ctx, input)) + if pressure { + err = arg.flushS3WriterOnMemoryPressure(proc, analyzer) + } else { + result := vm.CallResult{Batch: arg.ctr.buf} + err = flushTailBatch(proc, arg.ctr.s3Writer, &result, analyzer) + arg.ctr.buf = result.Batch + } + // Each flush appends exactly one new object; old results must not replay. + require.Equal(t, i+1, arg.ctr.buf.Vecs[1].Length()) + stats := objectio.ObjectStats(arg.ctr.buf.Vecs[1].GetBytesAt(i)) + names = append(names, stats.ObjectName().String()) + if missingWorkspace != "present" { + require.Error(t, err) + _, statErr := fs.StatFile(proc.Ctx, names[0]) + require.NoError(t, statErr, "failed retention must leave writer ownership intact") + if missingWorkspace == "retry" { + workspace = installInsertOwnershipWorkspace(t, proc) + require.NoError(t, arg.ctr.s3Writer.TransferPersistedObjects(proc)) + require.Equal(t, names, workspace.owners[0].Names()) + workspace.AcceptUnpublishedS3ObjectNames(names...) + arg.Reset(proc, true, nil) + require.Nil(t, arg.ctr.s3Writer, "successful retry clears pending-cleanup state") + require.NoError(t, workspace.owners[0].Cleanup(proc.Ctx)) + _, statErr = fs.StatFile(proc.Ctx, names[0]) + require.NoError(t, statErr) + return + } + // Even a success-shaped teardown must retry this failed handoff. + arg.Reset(proc, false, nil) + _, statErr = fs.StatFile(proc.Ctx, names[0]) + require.True(t, moerr.IsMoErrCode(statErr, moerr.ErrFileNotFound), "%v", statErr) + return + } + require.NoError(t, err) + require.Equal(t, []string{names[i]}, workspace.owners[i].Names()) + } + require.NotEqual(t, names[0], names[1]) + workspace.AcceptUnpublishedS3ObjectNames(names[0]) + arg.Reset(proc, false, nil) + arg.Free(proc, true, errors.New("consumer failed after first registration")) + for _, name := range names { + _, err = fs.StatFile(proc.Ctx, name) + require.NoError(t, err) + } + for _, owner := range workspace.owners { + require.NoError(t, owner.Cleanup(proc.Ctx)) + } + _, err = fs.StatFile(proc.Ctx, names[0]) + require.NoError(t, err, "accepted object must survive both cleanup paths") + _, err = fs.StatFile(proc.Ctx, names[1]) + require.True(t, moerr.IsMoErrCode(err, moerr.ErrFileNotFound), "%v", err) + }) + } + } +} + +func TestInsertResetUsesPipelineOutcomeForSpilledObjectOwnership(t *testing.T) { + for _, tc := range []struct { + name string + pipelineFail bool + }{ + {name: "failed pipeline deletes unaccepted object", pipelineFail: true}, + {name: "successful pipeline preserves transferred object", pipelineFail: false}, + } { + t.Run(tc.name, func(t *testing.T) { + proc := testutil.NewProc(t) + defer proc.Free() + + fs, err := colexec.GetSharedFSFromProc(proc) + require.NoError(t, err) + writer := colexec.NewCNS3DataWriter(proc.Mp(), fs, testInsertS3TableDef(), 1, false) + bat := &batch.Batch{ + Attrs: []string{"a", "b"}, + Vecs: []*vector.Vector{ + testutil.MakeInt64Vector([]int64{1}, nil, proc.Mp()), + testutil.MakeVarcharVector([]string{"x"}, nil, proc.Mp()), + }, + } + bat.SetRowCount(1) + defer bat.Clean(proc.Mp()) + require.NoError(t, writer.Write(proc.Ctx, bat)) + blockInfo, err := writer.SyncAndFillBlockInfoBat(proc.Ctx) + require.NoError(t, err) + require.Equal(t, 1, blockInfo.Vecs[1].Length()) + data, area := vector.MustVarlenaRawData(blockInfo.Vecs[1]) + stats := objectio.ObjectStats(data[0].GetByteSlice(area)) + objectName := stats.ObjectName().String() + + insert := &Insert{ctr: container{s3Writer: writer}} + insert.Reset(proc, tc.pipelineFail, errors.New("test pipeline outcome")) + require.Nil(t, insert.ctr.s3Writer) + _, err = fs.StatFile(proc.Ctx, objectName) + if tc.pipelineFail { + require.True(t, moerr.IsMoErrCode(err, moerr.ErrFileNotFound), "failed pipeline should delete object, got %v", err) + } else { + require.NoError(t, err, "successful pipeline must preserve transferred object") + } + insert.Free(proc, false, nil) + }) + } +} + +func TestInsertFreeTransfersFailedCleanupToTransaction(t *testing.T) { + t.Run("Free", func(t *testing.T) { checkInsertFreeTransfersFailedCleanupToTransaction(t, false) }) + t.Run("prepared Reset", func(t *testing.T) { checkInsertFreeTransfersFailedCleanupToTransaction(t, true) }) +} + +func checkInsertFreeTransfersFailedCleanupToTransaction(t *testing.T, prepared bool) { + proc := testutil.NewProc(t) + defer proc.Free() + baseFS, err := colexec.GetSharedFSFromProc(proc) + require.NoError(t, err) + deleteErr := errors.New("injected insert cleanup failure") + fs := &failOnceInsertDeleteFileService{FileService: baseFS, failErr: deleteErr} + writer := colexec.NewCNS3DataWriter(proc.Mp(), fs, testInsertS3TableDef(), 1, false) + bat := &batch.Batch{ + Attrs: []string{"a", "b"}, + Vecs: []*vector.Vector{ + testutil.MakeInt64Vector([]int64{1}, nil, proc.Mp()), + testutil.MakeVarcharVector([]string{"x"}, nil, proc.Mp()), + }, + } + bat.SetRowCount(1) + defer bat.Clean(proc.Mp()) + require.NoError(t, writer.Write(proc.Ctx, bat)) + blockInfo, err := writer.SyncAndFillBlockInfoBat(proc.Ctx) + require.NoError(t, err) + data, area := vector.MustVarlenaRawData(blockInfo.Vecs[1]) + stats := objectio.ObjectStats(data[0].GetByteSlice(area)) + objectName := stats.ObjectName().String() + blockInfo.Clean(proc.Mp()) + + ctrl := gomock.NewController(t) + txnOp := mock_frontend.NewMockTxnOperator(ctrl) + workspace := &insertS3CleanupWorkspace{} + txnOp.EXPECT().GetWorkspace().Return(workspace).AnyTimes() + proc.Base.TxnOperator = txnOp + + insert := NewArgument() + insert.ctr.s3Writer = writer + if prepared { + insert.Reset(proc, true, deleteErr) + } else { + insert.Free(proc, true, deleteErr) + } + require.Nil(t, insert.ctr.s3Writer) + require.Len(t, workspace.cleanups, 1, "transaction must take ownership before operator release") + insert.Release() + + _, err = baseFS.StatFile(proc.Ctx, objectName) + require.NoError(t, err, "failed immediate deletion should keep the object for retry") + require.NoError(t, workspace.cleanups[0](proc.Ctx)) + _, err = baseFS.StatFile(proc.Ctx, objectName) + require.True(t, moerr.IsMoErrCode(err, moerr.ErrFileNotFound), "transaction retry should delete the object, got %v", err) +} + +func TestPartitionInsertFreeKeepsFailedCleanupOwner(t *testing.T) { + for _, failDelete := range []bool{false, true} { + t.Run(fmt.Sprintf("failDelete=%v", failDelete), func(t *testing.T) { + proc := testutil.NewProc(t) + defer proc.Free() + baseFS, err := colexec.GetSharedFSFromProc(proc) + require.NoError(t, err) + deleteErr := errors.New("injected partition cleanup failure") + fs := &failOnceInsertDeleteFileService{FileService: baseFS} + if failDelete { + fs.failErr = deleteErr + } + writer := colexec.NewCNS3DataWriter(proc.Mp(), fs, testInsertS3TableDef(), 1, false) + bat := &batch.Batch{Attrs: []string{"a", "b"}, Vecs: []*vector.Vector{ + testutil.MakeInt64Vector([]int64{1}, nil, proc.Mp()), + testutil.MakeVarcharVector([]string{"x"}, nil, proc.Mp()), + }} + bat.SetRowCount(1) + defer bat.Clean(proc.Mp()) + require.NoError(t, writer.Write(proc.Ctx, bat)) + blockInfo, err := writer.SyncAndFillBlockInfoBat(proc.Ctx) + require.NoError(t, err) + stats := objectio.ObjectStats(blockInfo.Vecs[1].GetBytesAt(0)) + name := stats.ObjectName().String() + blockInfo.Clean(proc.Mp()) + + workspace := installInsertOwnershipWorkspace(t, proc) + arg := NewArgument() + arg.ctr.partitionS3Writers = []*colexec.CNS3Writer{nil, writer} + arg.Free(proc, true, deleteErr) + require.Nil(t, arg.ctr.partitionS3Writers) + arg.Release() + if failDelete { + require.Len(t, workspace.cleanups, 1) + _, err = baseFS.StatFile(proc.Ctx, name) + require.NoError(t, err, "failed deletion must preserve the object for retry") + require.NoError(t, workspace.cleanups[0](proc.Ctx)) + } else { + require.Empty(t, workspace.cleanups) + } + _, err = baseFS.StatFile(proc.Ctx, name) + require.True(t, moerr.IsMoErrCode(err, moerr.ErrFileNotFound), "%v", err) + }) + } +} + func TestInsertFlushS3WriterOnMemoryPressureRefreshesBeforeReleaseOnAppendError(t *testing.T) { proc := testutil.NewProc(t) defer proc.Free() @@ -493,6 +782,7 @@ func TestInsertFlushS3WriterOnMemoryPressureRefreshesBeforeReleaseOnAppendError( func TestInsertS3FinalFlushRefreshesBeforeReleaseOnSuccess(t *testing.T) { proc := testutil.NewProc(t) defer proc.Free() + installInsertOwnershipWorkspace(t, proc) fs, err := colexec.GetSharedFSFromProc(proc) require.NoError(t, err) diff --git a/pkg/sql/colexec/insert/types.go b/pkg/sql/colexec/insert/types.go index 6512f029e0d17..b539a19db8903 100644 --- a/pkg/sql/colexec/insert/types.go +++ b/pkg/sql/colexec/insert/types.go @@ -15,9 +15,13 @@ package insert import ( + "context" + "errors" + "github.com/matrixorigin/matrixone/pkg/common/reuse" "github.com/matrixorigin/matrixone/pkg/common/rscthrottler" "github.com/matrixorigin/matrixone/pkg/container/batch" + "github.com/matrixorigin/matrixone/pkg/logutil" "github.com/matrixorigin/matrixone/pkg/pb/plan" "github.com/matrixorigin/matrixone/pkg/perfcounter" "github.com/matrixorigin/matrixone/pkg/sql/colexec" @@ -25,6 +29,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/vm" "github.com/matrixorigin/matrixone/pkg/vm/engine" "github.com/matrixorigin/matrixone/pkg/vm/process" + "go.uber.org/zap" ) var _ vm.Operator = new(Insert) @@ -96,9 +101,18 @@ func NewArgument() *Insert { } func (insert *Insert) Release() { - if insert != nil { - reuse.Free[Insert](insert, nil) + if insert == nil { + return + } + if insert.ctr.s3Writer != nil { + return + } + for _, writer := range insert.ctr.partitionS3Writers { + if writer != nil { + return + } } + reuse.Free[Insert](insert, nil) } type InsertCtx struct { @@ -115,18 +129,12 @@ type InsertCtx struct { } func (insert *Insert) Reset(proc *process.Process, pipelineFailed bool, err error) { - //@todo need add Reset method for s3Writer - if insert.ctr.s3Writer != nil { - insert.ctr.s3Writer.Close() - insert.ctr.s3Writer = nil + if closeErr := insert.closeS3Writers(proc, pipelineFailed); closeErr != nil { + logutil.Warn("failed to clean insert S3 writers", zap.Error(closeErr)) + // Prepared executions skip Free: rollback must own failed cleanup now. + insert.retainPendingS3Writers(proc) } insert.releaseS3MemGrant() - if insert.ctr.partitionS3Writers != nil { - for _, writer := range insert.ctr.partitionS3Writers { - writer.Close() - } - insert.ctr.partitionS3Writers = nil - } // A non-nil extWriter here means the input stream never reached its clean // end (insert_external nils it after a successful Close), i.e. the pipeline // failed or was cancelled: discard the half-written file rather than @@ -141,22 +149,74 @@ func (insert *Insert) Reset(proc *process.Process, pipelineFailed bool, err erro } } -// The Argument for insert data directly to s3 can not be free when this function called as some datastructure still needed. -// therefore, those argument in remote CN will be free in connector operator, and local argument will be free in mergeBlock operator -func (insert *Insert) Free(proc *process.Process, pipelineFailed bool, err error) { +func (insert *Insert) retryPendingS3Writers(proc *process.Process) error { + return insert.closeS3Writers(proc, false) +} + +func (insert *Insert) closeS3Writers(proc *process.Process, pipelineFailed bool) error { + var cleanupErrs []error if insert.ctr.s3Writer != nil { - insert.ctr.s3Writer.Close() - insert.ctr.s3Writer = nil + if closeErr := insert.ctr.s3Writer.CloseWithCleanup(proc.Ctx, pipelineFailed); closeErr != nil { + cleanupErrs = append(cleanupErrs, closeErr) + logutil.Warn("failed to clean insert S3 writer", zap.Error(closeErr)) + } else { + insert.ctr.s3Writer = nil + } } - insert.releaseS3MemGrant() - - // Free the partition table S3writer object resources if insert.ctr.partitionS3Writers != nil { + pendingCleanup := false + for i, writer := range insert.ctr.partitionS3Writers { + if writer == nil { + continue + } + if closeErr := writer.CloseWithCleanup(proc.Ctx, pipelineFailed); closeErr != nil { + cleanupErrs = append(cleanupErrs, closeErr) + pendingCleanup = true + logutil.Warn("failed to clean partition insert S3 writer", zap.Error(closeErr)) + } else { + insert.ctr.partitionS3Writers[i] = nil + } + } + if !pendingCleanup { + insert.ctr.partitionS3Writers = nil + } + } + return errors.Join(cleanupErrs...) +} + +func (insert *Insert) retainPendingS3Writers(proc *process.Process) { + retain := func(writer *colexec.CNS3Writer) bool { + return colexec.RetainUnpublishedS3Cleanup(proc, func(ctx context.Context) error { + return writer.CloseWithCleanup(ctx, true) + }) + } + if writer := insert.ctr.s3Writer; writer != nil && retain(writer) { + insert.ctr.s3Writer = nil + } + for i, writer := range insert.ctr.partitionS3Writers { + if writer != nil && retain(writer) { + insert.ctr.partitionS3Writers[i] = nil + } + } + if len(insert.ctr.partitionS3Writers) != 0 { + pending := false for _, writer := range insert.ctr.partitionS3Writers { - writer.Close() + pending = pending || writer != nil + } + if !pending { + insert.ctr.partitionS3Writers = nil } - insert.ctr.partitionS3Writers = nil } +} + +// The Argument for insert data directly to s3 can not be free when this function called as some datastructure still needed. +// therefore, those argument in remote CN will be free in connector operator, and local argument will be free in mergeBlock operator +func (insert *Insert) Free(proc *process.Process, pipelineFailed bool, err error) { + if closeErr := insert.closeS3Writers(proc, pipelineFailed); closeErr != nil { + logutil.Warn("failed to clean insert S3 writers", zap.Error(closeErr)) + insert.retainPendingS3Writers(proc) + } + insert.releaseS3MemGrant() // See Reset: a writer still alive at Free means the stream did not end // cleanly; abort instead of persisting a partial file. diff --git a/pkg/sql/colexec/multi_update/multi_update.go b/pkg/sql/colexec/multi_update/multi_update.go index 03bf89c6cf2e2..2fd91bdca39e9 100644 --- a/pkg/sql/colexec/multi_update/multi_update.go +++ b/pkg/sql/colexec/multi_update/multi_update.go @@ -47,6 +47,13 @@ func (update *MultiUpdate) OpType() vm.OpType { } func (update *MultiUpdate) Prepare(proc *process.Process) error { + if writer := update.ctr.s3Writer; writer != nil && writer.cleanupErr != nil { + if err := writer.reset(proc, true); err != nil { + return err + } + writer.cleanupErr = nil + } + if update.OpAnalyzer == nil { update.OpAnalyzer = process.NewAnalyzer(update.GetIdx(), update.IsFirst, update.IsLast, opName) } else { @@ -154,7 +161,7 @@ func (update *MultiUpdate) Prepare(proc *process.Process) error { writer.segmentMap = colexec.MustGetServer(proc.GetService()).GetCnSegmentMap() update.MultiUpdateCtx = writer.updateCtxs - err = writer.free(proc) + err = writer.free(proc, false) if err != nil { return err } diff --git a/pkg/sql/colexec/multi_update/multi_update_partition.go b/pkg/sql/colexec/multi_update/multi_update_partition.go index d53512bf9bbd8..c5657c290929a 100644 --- a/pkg/sql/colexec/multi_update/multi_update_partition.go +++ b/pkg/sql/colexec/multi_update/multi_update_partition.go @@ -16,14 +16,18 @@ package multi_update import ( "bytes" + "errors" "github.com/matrixorigin/matrixone/pkg/container/batch" + "github.com/matrixorigin/matrixone/pkg/logutil" "github.com/matrixorigin/matrixone/pkg/partitionprune" "github.com/matrixorigin/matrixone/pkg/pb/partition" + "github.com/matrixorigin/matrixone/pkg/sql/colexec" "github.com/matrixorigin/matrixone/pkg/sql/features" "github.com/matrixorigin/matrixone/pkg/vm" "github.com/matrixorigin/matrixone/pkg/vm/engine" "github.com/matrixorigin/matrixone/pkg/vm/process" + "go.uber.org/zap" ) type PartitionMultiUpdate struct { @@ -85,6 +89,10 @@ func (op *PartitionMultiUpdate) OpType() vm.OpType { func (op *PartitionMultiUpdate) Prepare( proc *process.Process, ) error { + if err := op.freePartitionWriters(proc, true); err != nil { + return err + } + if op.OpAnalyzer == nil { op.OpAnalyzer = process.NewAnalyzer(op.GetIdx(), op.IsFirst, op.IsLast, "partition_multi_update") } else { @@ -515,10 +523,16 @@ func (op *PartitionMultiUpdate) Free( err error, ) { op.raw.Free(proc, pipelineFailed, err) - op.freePartitionWriters(proc) + if cleanupErr := op.freePartitionWriters(proc, pipelineFailed); cleanupErr != nil { + logutil.Warn("failed to clean partition multi-update writers", zap.Error(cleanupErr)) + op.retainPendingPartitionWriters(proc) + } } func (op *PartitionMultiUpdate) Release() { + if len(op.writers) != 0 || len(op.freeWriters) != 0 { + return + } op.raw.Release() } @@ -530,7 +544,10 @@ func (op *PartitionMultiUpdate) Reset( op.raw.MultiUpdateCtx = op.rawContexts op.raw.Reset(proc, pipelineFailed, err) op.raw.resetMultiUpdateCtxs() - op.freePartitionWriters(proc) + if cleanupErr := op.freePartitionWriters(proc, pipelineFailed); cleanupErr != nil { + logutil.Warn("failed to clean partition multi-update writers", zap.Error(cleanupErr)) + op.retainPendingPartitionWriters(proc) + } op.s3AffectedRows = 0 for _, target := range op.targets { clear(target.writerIDs) @@ -538,15 +555,64 @@ func (op *PartitionMultiUpdate) Reset( op.nextWriterID = 0 } -func (op *PartitionMultiUpdate) freePartitionWriters(proc *process.Process) { +func (op *PartitionMultiUpdate) freePartitionWriters(proc *process.Process, pipelineFailed bool) error { + var cleanupErrs []error for id, writer := range op.writers { - _ = writer.free(proc) - delete(op.writers, id) + if err := writer.free(proc, pipelineFailed); err == nil { + delete(op.writers, id) + } else { + cleanupErrs = append(cleanupErrs, err) + } } + pending := op.freeWriters[:0] for _, writer := range op.freeWriters { - _ = writer.free(proc) + if err := writer.free(proc, pipelineFailed); err != nil { + pending = append(pending, writer) + cleanupErrs = append(cleanupErrs, err) + } + } + if len(pending) == 0 { + op.freeWriters = nil + } else { + op.freeWriters = pending + } + return errors.Join(cleanupErrs...) +} + +func (op *PartitionMultiUpdate) retainPendingPartitionWriters(proc *process.Process) { + for id, writer := range op.writers { + if writer == nil { + continue + } + task := writer.retryTask() + if task.empty() || colexec.RetainUnpublishedS3Cleanup(proc, task.cleanup) { + writer.releaseBuffers(proc.Mp()) + writer.syncedObjectOwners = nil + writer.failedWriters = nil + writer.insertSinkers = nil + delete(op.writers, id) + } + } + pending := op.freeWriters[:0] + for _, writer := range op.freeWriters { + if writer == nil { + continue + } + task := writer.retryTask() + if task.empty() || colexec.RetainUnpublishedS3Cleanup(proc, task.cleanup) { + writer.releaseBuffers(proc.Mp()) + writer.syncedObjectOwners = nil + writer.failedWriters = nil + writer.insertSinkers = nil + continue + } + pending = append(pending, writer) + } + if len(pending) == 0 { + op.freeWriters = nil + } else { + op.freeWriters = pending } - op.freeWriters = nil } func (op *PartitionMultiUpdate) GetOperatorBase() *vm.OperatorBase { diff --git a/pkg/sql/colexec/multi_update/s3writer_delegate.go b/pkg/sql/colexec/multi_update/s3writer_delegate.go index 65f222915293b..c3b51c5a0b34d 100644 --- a/pkg/sql/colexec/multi_update/s3writer_delegate.go +++ b/pkg/sql/colexec/multi_update/s3writer_delegate.go @@ -16,6 +16,8 @@ package multi_update import ( "bytes" + "context" + "errors" "fmt" "slices" "strings" @@ -84,6 +86,13 @@ func newDeleteBlockData(inputBatch *batch.Batch, pkIdx int) *deleteBlockData { type s3WriterDelegate struct { // persistent per-table insert sinkers, created lazily on first append. insertSinkers []*colexec.CNS3Writer + // failedWriters retain the full sinker only when Write or Sync fails and + // cleanup may still need staged objects or asynchronous pipeline results. + failedWriters []*colexec.CNS3Writer + // Object-name owners outlive successful one-shot writers without retaining + // their sinker buffers. The workspace accepts them at entry append or + // cleans them if the pipeline aborts before registration. + syncedObjectOwners []*colexec.UnpublishedS3ObjectOwner // per-table delete column accumulators (rowid + pk only). deleteBatches []*batch.BatchSet segmentMap map[string]int32 @@ -125,6 +134,8 @@ type s3WriterDelegate struct { grantedSize int64 throttler rscthrottler.RSCThrottler } + cleanupErr error + cleanupMP *mpool.MPool } func newS3Writer( @@ -235,7 +246,14 @@ func retainedS3InputCols(updateCtxs []*MultiUpdateCtx, action actionType) []int // ensureInsertSinkers lazily creates persistent per-table insert sinkers // on first call. Requires proc for Mp() and fileservice. func (writer *s3WriterDelegate) ensureInsertSinkers(proc *process.Process) error { - if writer.insertSinkers[0] != nil { + need := false + for i, updateCtx := range writer.updateCtxs { + if len(updateCtx.InsertCols) != 0 && writer.insertSinkers[i] == nil { + need = true + break + } + } + if !need { return nil } fs, err := colexec.GetSharedFSFromProc(proc) @@ -243,7 +261,7 @@ func (writer *s3WriterDelegate) ensureInsertSinkers(proc *process.Process) error return err } for i, updateCtx := range writer.updateCtxs { - if len(updateCtx.InsertCols) == 0 { + if len(updateCtx.InsertCols) == 0 || writer.insertSinkers[i] != nil { continue } opts := []ioutil.SinkerOption{ @@ -684,11 +702,18 @@ func (writer *s3WriterDelegate) sortAndSyncOneTable( proc.GetService(), proc.Mp(), fs, tblDef, -1, false, opts..., ) } - - defer s3Writer.Close() - counterSet := analyzer.GetOpCounterSet() writeCtx := perfcounter.AttachS3RequestKey(proc.Ctx, counterSet) + transferred := false + defer func() { + if !transferred { + writer.failedWriters = append(writer.failedWriters, s3Writer) + return + } + if closeErr := s3Writer.Close(); closeErr != nil { + err = errors.Join(err, closeErr) + } + }() for i := range bats { rowCount += bats[i].RowCount() @@ -724,23 +749,127 @@ func (writer *s3WriterDelegate) sortAndSyncOneTable( bats[i] = nil } - if err = process.MeasureFilesystemWaitErr(analyzer, func() error { - _, syncErr := s3Writer.Sync(writeCtx) - return syncErr + if _, err = process.MeasureFilesystemWait(analyzer, func() ([]objectio.ObjectStats, error) { + return s3Writer.Sync(writeCtx) }); err != nil { return } - if blockInfoBat, err = s3Writer.FillBlockInfoBat(); err != nil { return } if isTombstone { - return writer.fillDeleteBlockInfo(proc, idx, blockInfoBat, rowCount) + err = writer.fillDeleteBlockInfo(proc, idx, blockInfoBat, rowCount) + } else { + err = writer.fillInsertBlockInfo(proc, idx, blockInfoBat, rowCount) + } + if err != nil { + return + } + owner, transferErr := s3Writer.TransferPersistedObjectsOwner(proc) + if transferErr != nil { + return transferErr } + if owner != nil { + writer.syncedObjectOwners = append(writer.syncedObjectOwners, owner) + } + transferred = true + return nil +} - return writer.fillInsertBlockInfo(proc, idx, blockInfoBat, rowCount) +// RetainOutputS3ObjectOwnership installs the cleanup lease in the receiving +// transaction workspace before a remote producer batch is acknowledged. The +// workspace later accepts each name only when its write entry is appended. +func RetainOutputS3ObjectOwnership(proc *process.Process, output *batch.Batch) error { + if proc == nil || output == nil || output.IsEmpty() { + return nil + } + if output.VectorCount() < 5 { + return moerr.NewInternalErrorNoCtx("invalid multi-update S3 output batch") + } + actions := vector.MustFixedColNoTypeCheck[uint8](output.Vecs[0]) + data, area := vector.MustVarlenaRawData(output.Vecs[4]) + names := make([]string, 0) + seen := make(map[string]struct{}) + for i, action := range actions { + switch actionType(action) { + case actionInsert, actionDelete, actionUpdate: + default: + continue + } + metadata := batch.NewOffHeapEmpty() + if err := metadata.UnmarshalBinaryWithAnyMp(data[i].GetByteSlice(area), proc.Mp()); err != nil { + metadata.Clean(proc.Mp()) + return err + } + for column, attr := range metadata.Attrs { + switch attr { + case catalog.ObjectMeta_ObjectStats: + for row := 0; row < metadata.Vecs[column].Length(); row++ { + stats := objectio.ObjectStats(metadata.Vecs[column].GetBytesAt(row)) + name := stats.ObjectName().String() + if name != "" { + if _, ok := seen[name]; !ok { + seen[name] = struct{}{} + names = append(names, name) + } + } + } + case catalog.BlockMeta_BlockInfo: + for row := 0; row < metadata.Vecs[column].Length(); row++ { + block := objectio.DecodeBlockInfo(metadata.Vecs[column].GetBytesAt(row)) + name := block.MetaLocation().Name().String() + if name != "" { + if _, ok := seen[name]; !ok { + seen[name] = struct{}{} + names = append(names, name) + } + } + } + } + } + metadata.Clean(proc.Mp()) + } + if len(names) == 0 { + return nil + } + fs, err := colexec.GetSharedFSFromProc(proc) + if err != nil { + return err + } + return colexec.RetainReceivedUnpublishedS3ObjectNames(proc, fs, names...) +} + +// finalizeSyncedObjects releases completed object names after the successful +// pipeline handoff, or deletes them when that handoff aborts. A failed delete +// remains sticky so a later success-shaped reset retries cleanup instead of +// treating unaccepted objects as published. +func (writer *s3WriterDelegate) finalizeSyncedObjects( + ctx context.Context, + pipelineFailed bool, +) error { + if len(writer.syncedObjectOwners) == 0 { + return nil + } + if !pipelineFailed && writer.cleanupErr == nil { + // The workspace owner outlives the producer and is retired only by + // registration or the remote batch-ownership handoff barrier. + writer.syncedObjectOwners = nil + return nil + } + cleanupCtx, cancel := colexec.UnpublishedS3CleanupContext(ctx) + defer cancel() + pendingOwners := make([]*colexec.UnpublishedS3ObjectOwner, 0, len(writer.syncedObjectOwners)) + var errs []error + for _, owner := range writer.syncedObjectOwners { + if err := owner.Cleanup(cleanupCtx); err != nil { + errs = append(errs, err) + pendingOwners = append(pendingOwners, owner) + } + } + writer.syncedObjectOwners = pendingOwners + return errors.Join(errs...) } // initBlockInfoBat creates a new off-heap batch matching the schema of src. @@ -837,6 +966,9 @@ func (writer *s3WriterDelegate) flushTailAndWriteToOutput(proc *process.Process, if err = writer.fillInsertBlockInfo(proc, i, blockInfoBat, rowCount); err != nil { return } + if err = s3w.TransferPersistedObjects(proc); err != nil { + return err + } } // Flush remaining deletes — always call sortAndSync so that accumulated @@ -903,18 +1035,40 @@ func (writer *s3WriterDelegate) flushTailAndWriteToOutput(proc *process.Process, return nil } -func (writer *s3WriterDelegate) reset(proc *process.Process) (err error) { +func (writer *s3WriterDelegate) reset(proc *process.Process, pipelineFailed bool) (err error) { writer.cleanDeleteBatches(proc.Mp()) + var cleanupErrs []error + if cleanupErr := writer.finalizeSyncedObjects(proc.Ctx, pipelineFailed); cleanupErr != nil { + logutil.Warn("failed to clean multi-update synced S3 objects", zap.Error(cleanupErr)) + cleanupErrs = append(cleanupErrs, cleanupErr) + } + + var pendingWriters []*colexec.CNS3Writer + for _, s3w := range writer.failedWriters { + if closeErr := s3w.CloseWithCleanup(proc.Ctx, true); closeErr != nil { + logutil.Warn("failed to clean multi-update S3 writer", zap.Error(closeErr)) + cleanupErrs = append(cleanupErrs, closeErr) + pendingWriters = append(pendingWriters, s3w) + } + } + writer.failedWriters = pendingWriters // Reset persistent insert sinkers so any buffered data from a failed // pipeline execution is discarded. The sinkers stay alive and reuse // their buffer pools and arenas on the next append() call. On the // success path the sinkers were already flushed by // flushTailAndWriteToOutput(), so Reset() is a no-op on the data side. - for _, s3w := range writer.insertSinkers { + for i, s3w := range writer.insertSinkers { if s3w != nil { - s3w.Reset() + if cleanupErr := s3w.ResetWithCleanup(proc.Ctx, pipelineFailed); cleanupErr != nil { + logutil.Warn("failed to clean multi-update insert S3 writer", zap.Error(cleanupErr)) + cleanupErrs = append(cleanupErrs, cleanupErr) + continue + } + if !s3w.ReadyForWrites() { + writer.insertSinkers[i] = nil + } } } @@ -946,29 +1100,74 @@ func (writer *s3WriterDelegate) reset(proc *process.Process) (err error) { } writer.buf.Reset() - return + writer.cleanupErr = errors.Join(cleanupErrs...) + return writer.cleanupErr } -func (writer *s3WriterDelegate) free(proc *process.Process) (err error) { +func (writer *s3WriterDelegate) free(proc *process.Process, pipelineFailed bool) (err error) { writer.cleanDeleteBatches(proc.Mp()) mp := proc.Mp() + writer.cleanupMP = mp + var cleanupErrs []error + if cleanupErr := writer.finalizeSyncedObjects(proc.Ctx, pipelineFailed); cleanupErr != nil { + cleanupErrs = append(cleanupErrs, cleanupErr) + logutil.Warn("failed to clean multi-update synced S3 objects", zap.Error(cleanupErr)) + } + + for i, s3w := range writer.failedWriters { + if s3w == nil { + continue + } + if closeErr := s3w.CloseWithCleanup(proc.Ctx, true); closeErr != nil { + cleanupErrs = append(cleanupErrs, closeErr) + logutil.Warn("failed to clean multi-update S3 writer", zap.Error(closeErr)) + } else { + writer.failedWriters[i] = nil + } + } + if len(cleanupErrs) == 0 { + writer.failedWriters = nil + } // Close persistent insert sinkers. for i, s3w := range writer.insertSinkers { if s3w != nil { - s3w.Close() - writer.insertSinkers[i] = nil + if closeErr := s3w.CloseWithCleanup(proc.Ctx, pipelineFailed); closeErr != nil { + cleanupErrs = append(cleanupErrs, closeErr) + logutil.Warn("failed to clean multi-update insert S3 writer", zap.Error(closeErr)) + } else { + writer.insertSinkers[i] = nil + } } } - writer.insertSinkers = nil + writer.cleanupErr = errors.Join(cleanupErrs...) + writer.releaseBuffers(mp) + return writer.cleanupErr +} +// releaseBuffers also serves Reset failures: detach the non-reusable delegate +// without issuing another synchronous Delete before handing it to rollback. +func (writer *s3WriterDelegate) releaseBuffers(mp *mpool.MPool) { + for i, s3w := range writer.insertSinkers { + if s3w != nil && !s3w.PendingUnpublishedCleanup() { + _ = s3w.Close() + writer.insertSinkers[i] = nil + } + } + if writer.cleanupErr == nil { + writer.insertSinkers = nil + } + // Every failed CNS3Writer has already detached its sinker. Retry ownership + // contains only names, so shared batch free lists and the session mpool can + // be released before the callback enters the transaction or CN queue. for _, fl := range writer.insertFreeLists { if fl != nil { fl.Close(mp) } } writer.insertFreeLists = nil + writer.cleanupMP = nil for _, bat := range writer.insertBlockInfo { if bat != nil { @@ -999,8 +1198,106 @@ func (writer *s3WriterDelegate) free(proc *process.Process) (err error) { writer.outputBat = nil } writer.buf.Reset() +} - return +type unpublishedS3RetryTask struct { + owners []*colexec.UnpublishedS3ObjectOwner + failedWriters []*colexec.CNS3Writer + insertSinkers []*colexec.CNS3Writer +} + +func (task *unpublishedS3RetryTask) empty() bool { + return len(task.owners) == 0 && len(task.failedWriters) == 0 && len(task.insertSinkers) == 0 +} + +func (writer *s3WriterDelegate) retryTask() *unpublishedS3RetryTask { + task := new(unpublishedS3RetryTask) + for _, owner := range writer.syncedObjectOwners { + if owner != nil && owner.Pending() { + task.owners = append(task.owners, owner) + } + } + for _, s3w := range writer.failedWriters { + if s3w != nil && s3w.PendingUnpublishedCleanup() { + task.failedWriters = append(task.failedWriters, s3w) + } + } + for _, s3w := range writer.insertSinkers { + if s3w != nil && s3w.PendingUnpublishedCleanup() { + task.insertSinkers = append(task.insertSinkers, s3w) + } + } + return task +} + +func (task *unpublishedS3RetryTask) cleanup(ctx context.Context) error { + cleanupCtx, cancel := colexec.UnpublishedS3CleanupContext(ctx) + defer cancel() + var errs []error + pendingOwners := task.owners[:0] + for _, owner := range task.owners { + if err := owner.Cleanup(cleanupCtx); err != nil { + errs = append(errs, err) + pendingOwners = append(pendingOwners, owner) + } + } + for i := len(pendingOwners); i < len(task.owners); i++ { + task.owners[i] = nil + } + task.owners = pendingOwners + for _, writers := range []*[]*colexec.CNS3Writer{&task.failedWriters, &task.insertSinkers} { + pending := (*writers)[:0] + for _, s3w := range *writers { + if err := s3w.CloseWithCleanup(cleanupCtx, true); err != nil { + errs = append(errs, err) + pending = append(pending, s3w) + } + } + for i := len(pending); i < len(*writers); i++ { + (*writers)[i] = nil + } + *writers = pending + } + return errors.Join(errs...) +} + +func (writer *s3WriterDelegate) cleanupUnpublishedS3Objects(ctx context.Context) error { + var cleanupErrs []error + if err := writer.finalizeSyncedObjects(ctx, true); err != nil { + cleanupErrs = append(cleanupErrs, err) + } + for i, s3w := range writer.failedWriters { + if s3w == nil { + continue + } + if err := s3w.CloseWithCleanup(ctx, true); err != nil { + cleanupErrs = append(cleanupErrs, err) + } else { + writer.failedWriters[i] = nil + } + } + for i, s3w := range writer.insertSinkers { + if s3w == nil { + continue + } + if err := s3w.CloseWithCleanup(ctx, true); err != nil { + cleanupErrs = append(cleanupErrs, err) + } else { + writer.insertSinkers[i] = nil + } + } + if err := errors.Join(cleanupErrs...); err != nil { + return err + } + writer.failedWriters = nil + writer.insertSinkers = nil + for _, freeList := range writer.insertFreeLists { + if freeList != nil { + freeList.Close(writer.cleanupMP) + } + } + writer.insertFreeLists = nil + return nil } func (writer *s3WriterDelegate) addBatchToOutput( diff --git a/pkg/sql/colexec/multi_update/types.go b/pkg/sql/colexec/multi_update/types.go index 6002588dbe238..0c57d437bdf5a 100644 --- a/pkg/sql/colexec/multi_update/types.go +++ b/pkg/sql/colexec/multi_update/types.go @@ -22,10 +22,13 @@ import ( "github.com/matrixorigin/matrixone/pkg/common/rscthrottler" "github.com/matrixorigin/matrixone/pkg/container/batch" "github.com/matrixorigin/matrixone/pkg/container/vector" + "github.com/matrixorigin/matrixone/pkg/logutil" + "github.com/matrixorigin/matrixone/pkg/sql/colexec" "github.com/matrixorigin/matrixone/pkg/sql/plan" "github.com/matrixorigin/matrixone/pkg/vm" "github.com/matrixorigin/matrixone/pkg/vm/engine" "github.com/matrixorigin/matrixone/pkg/vm/process" + "go.uber.org/zap" ) func updateCtxKey(ctx *MultiUpdateCtx) string { @@ -209,6 +212,9 @@ func NewArgument() *MultiUpdate { func (update *MultiUpdate) Release() { if update != nil { + if update.ctr.s3Writer != nil { + return + } reuse.Free[MultiUpdate](update, nil) } } @@ -237,7 +243,9 @@ func (update *MultiUpdate) Reset(proc *process.Process, pipelineFailed bool, err } } if update.ctr.s3Writer != nil { - update.ctr.s3Writer.reset(proc) + if resetErr := update.ctr.s3Writer.reset(proc, pipelineFailed); resetErr != nil { + update.retainPendingS3Writer(proc) + } } update.ctr.s3AffectedRows = 0 update.freeSeenTargetRows() @@ -261,8 +269,12 @@ func (update *MultiUpdate) Free(proc *process.Process, pipelineFailed bool, err update.ctr.deleteBuf = nil if update.ctr.s3Writer != nil { - update.ctr.s3Writer.free(proc) - update.ctr.s3Writer = nil + if freeErr := update.ctr.s3Writer.free(proc, pipelineFailed); freeErr != nil { + logutil.Warn("failed to clean multi-update S3 writer delegate", zap.Error(freeErr)) + update.retainPendingS3Writer(proc) + } else { + update.ctr.s3Writer = nil + } } update.freeSeenTargetRows() @@ -270,6 +282,28 @@ func (update *MultiUpdate) Free(proc *process.Process, pipelineFailed bool, err update.ctr.sources = nil } +func (update *MultiUpdate) retainPendingS3Writer(proc *process.Process) bool { + writer := update.ctr.s3Writer + if writer == nil { + return false + } + task := writer.retryTask() + if task.empty() { + writer.releaseBuffers(proc.Mp()) + update.ctr.s3Writer = nil + return true + } + if colexec.RetainUnpublishedS3Cleanup(proc, task.cleanup) { + writer.releaseBuffers(proc.Mp()) + writer.syncedObjectOwners = nil + writer.failedWriters = nil + writer.insertSinkers = nil + update.ctr.s3Writer = nil + return true + } + return false +} + func (update *MultiUpdate) freeSeenTargetRows() { if update.ctr.seenRowsGrant > 0 { update.ctr.seenRowsRSC.Release(update.ctr.seenRowsGrant) diff --git a/pkg/sql/colexec/multi_update/update_test.go b/pkg/sql/colexec/multi_update/update_test.go index 7cd713e09fc7a..12cfd45e5148f 100644 --- a/pkg/sql/colexec/multi_update/update_test.go +++ b/pkg/sql/colexec/multi_update/update_test.go @@ -17,20 +17,32 @@ package multi_update import ( "bytes" "context" + "errors" + "slices" "testing" "github.com/matrixorigin/matrixone/pkg/catalog" "github.com/matrixorigin/matrixone/pkg/common/hashmap" + "github.com/matrixorigin/matrixone/pkg/common/moerr" "github.com/matrixorigin/matrixone/pkg/common/mpool" + moruntime "github.com/matrixorigin/matrixone/pkg/common/runtime" "github.com/matrixorigin/matrixone/pkg/container/batch" "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/matrixorigin/matrixone/pkg/container/vector" "github.com/matrixorigin/matrixone/pkg/defines" + "github.com/matrixorigin/matrixone/pkg/fileservice" + mock_frontend "github.com/matrixorigin/matrixone/pkg/frontend/test" + mock_lock "github.com/matrixorigin/matrixone/pkg/frontend/test/mock_lock" + "github.com/matrixorigin/matrixone/pkg/lockservice" + "github.com/matrixorigin/matrixone/pkg/objectio" + "github.com/matrixorigin/matrixone/pkg/objectio/ioutil" "github.com/matrixorigin/matrixone/pkg/sql/colexec" "github.com/matrixorigin/matrixone/pkg/sql/plan" "github.com/matrixorigin/matrixone/pkg/sql/util" "github.com/matrixorigin/matrixone/pkg/testutil" + "github.com/matrixorigin/matrixone/pkg/txn/client" "github.com/matrixorigin/matrixone/pkg/vm" + "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/containers" "github.com/matrixorigin/matrixone/pkg/vm/process" "github.com/stretchr/testify/require" ) @@ -626,12 +638,61 @@ func TestNewS3WriterAllowsIndexOnlyContext(t *testing.T) { require.Equal(t, actionInsert, writer.action) require.Equal(t, InsertWriteS3Threshold, writer.flushThreshold) require.Equal(t, []int{0, 1}, writer.checkSizeCols) - require.NoError(t, writer.free(proc)) + require.NoError(t, writer.free(proc, false)) } -func TestSortAndSyncOneTableUsesDataWriter(t *testing.T) { +func TestPersistentInsertProducerTransfersOwnership(t *testing.T) { _, _, proc := prepareTestCtx(t, true) defer proc.Free() + workspace := proc.GetTxnOperator().GetWorkspace().(*multiUpdateS3CleanupWorkspace) + _, tableDef := getTestMainTable() + arg := &MultiUpdate{MultiUpdateCtx: []*MultiUpdateCtx{{TableDef: tableDef, InsertCols: []int{0, 1, 2, 3}}}} + arg.resetMultiUpdateCtxs() + writer, err := newS3Writer(proc.GetService(), arg) + require.NoError(t, err) + writer.addAffectedRows = func(uint64) {} + defer func() { require.NoError(t, writer.free(proc, true)) }() + bats, _ := prepareTestInsertBatchs(proc.Mp(), 1, 16, false, false) + defer bats[0].Clean(proc.Mp()) + analyzer := process.NewAnalyzer(0, false, false, "persistent-insert-owner") + require.NoError(t, writer.append(proc, analyzer, bats[0])) + require.NoError(t, writer.flushTailAndWriteToOutput(proc, analyzer)) + names := objectNamesFromInsertInfo(t, writer.insertBlockInfo[0]) + require.NotEmpty(t, names) + require.ElementsMatch(t, names, workspace.pendingObjectNames()) + fs, err := colexec.GetSharedFSFromProc(proc) + require.NoError(t, err) + // A successful producer reset cannot retire the workspace's cleanup owner. + require.NoError(t, writer.reset(proc, false)) + require.ElementsMatch(t, names, workspace.pendingObjectNames()) + require.NoError(t, writer.free(proc, true)) + for _, name := range names { + _, err = fs.StatFile(proc.Ctx, name) + require.NoError(t, err) + } + require.NoError(t, workspace.CleanupUnpublishedS3Objects(proc.Ctx)) + for _, name := range names { + _, err = fs.StatFile(proc.Ctx, name) + require.True(t, moerr.IsMoErrCode(err, moerr.ErrFileNotFound), "%v", err) + } +} + +func TestSortAndSyncOneTableUsesDataWriter(t *testing.T) { + _, ctrl, proc := prepareTestCtx(t, true) + defer proc.Free() + serviceID := t.Name() + rt := moruntime.DefaultRuntime() + throttler, _ := moruntime.ServiceRuntime("").GetGlobalVariables(moruntime.CNMemoryThrottler) + rt.SetGlobalVariables(moruntime.CNMemoryThrottler, throttler) + moruntime.SetupServiceBasedRuntime(serviceID, rt) + colexec.NewServer(serviceID) + lockSvc := mock_lock.NewMockLockService(ctrl) + lockSvc.EXPECT().GetConfig().Return(lockservice.Config{ServiceID: serviceID}).AnyTimes() + proc.Base.LockService = lockSvc + workspace := &multiUpdateS3CleanupWorkspace{} + txnOp := mock_frontend.NewMockTxnOperator(ctrl) + txnOp.EXPECT().GetWorkspace().Return(workspace).AnyTimes() + proc.Base.TxnOperator = txnOp _, tableDef := getTestMainTable() updateCtx := &MultiUpdateCtx{ TableDef: tableDef, @@ -641,7 +702,7 @@ func TestSortAndSyncOneTableUsesDataWriter(t *testing.T) { update.resetMultiUpdateCtxs() writer, err := newS3Writer(proc.GetService(), update) require.NoError(t, err) - defer func() { require.NoError(t, writer.free(proc)) }() + defer func() { require.NoError(t, writer.free(proc, false)) }() // Keep the full-block ownership-transfer path and its partial-block fallback // in one focused test instead of making every S3 matrix case use large data. @@ -658,7 +719,650 @@ func TestSortAndSyncOneTableUsesDataWriter(t *testing.T) { for _, bat := range bats { require.Nil(t, bat) } + require.Empty(t, writer.failedWriters, "successful one-shot writers must not be retained") require.NotNil(t, writer.insertBlockInfo[0]) + objectNames := objectNamesFromInsertInfo(t, writer.insertBlockInfo[0]) + wantNames := append([]string(nil), objectNames...) + slices.Sort(wantNames) + require.Equal(t, wantNames, unpublishedObjectOwnerNames(writer.syncedObjectOwners)) + server := colexec.GetServer(proc.GetService()) + require.Equal(t, len(objectNames), server.UnpublishedS3AdmissionStats().Used, + "closed one-shot writer must leave tickets with the workspace owner") + info := writer.insertBlockInfo[0] + statsData, statsArea := vector.MustVarlenaRawData(info.Vecs[1]) + stats := objectio.ObjectStats(statsData[0].GetByteSlice(statsArea)) + objectName := stats.ObjectName().String() + fs, err := colexec.GetSharedFSFromProc(proc) + require.NoError(t, err) + for _, name := range objectNames { + _, err = fs.StatFile(proc.Ctx, name) + require.NoError(t, err) + } + + require.NoError(t, writer.reset(proc, false)) + require.Empty(t, writer.syncedObjectOwners, "the workspace retains the lease after producer reset") + require.Equal(t, len(objectNames), server.UnpublishedS3AdmissionStats().Used) + workspace.AcceptUnpublishedS3ObjectNames(objectNames...) + require.Zero(t, server.UnpublishedS3AdmissionStats().Used) + require.NoError(t, workspace.CleanupUnpublishedS3Objects(proc.Ctx)) + _, err = fs.StatFile(proc.Ctx, objectName) + require.NoError(t, err, "registered metadata must keep its S3 object") +} + +func TestSortAndSyncOneTableRetainsOwnerWhenWorkspaceCannotAcceptIt(t *testing.T) { + _, ctrl, proc := prepareTestCtx(t, true) + defer proc.Free() + _, tableDef := getTestMainTable() + update := &MultiUpdate{MultiUpdateCtx: []*MultiUpdateCtx{{ + TableDef: tableDef, + InsertCols: []int{0, 1, 2, 3}, + }}} + update.resetMultiUpdateCtxs() + writer, err := newS3Writer(proc.GetService(), update) + require.NoError(t, err) + + workspace := &struct{ client.Workspace }{} + txnOp := mock_frontend.NewMockTxnOperator(ctrl) + txnOp.EXPECT().GetWorkspace().Return(workspace).AnyTimes() + proc.Base.TxnOperator = txnOp + batches, _ := prepareTestInsertBatchs(proc.Mp(), 2, colexec.DefaultBatchSize, false, false) + err = writer.sortAndSyncOneTable( + proc, + tableDef, + process.NewAnalyzer(0, false, false, "multi-update-workspace-owner-failure"), + 0, + false, + batches, + true, + ) + require.ErrorContains(t, err, "cannot retain unpublished S3 objects") + require.Len(t, writer.failedWriters, 1, + "the writer must keep cleanup ownership when workspace handoff is rejected") + objectNames := objectNamesFromInsertInfo(t, writer.insertBlockInfo[0]) + require.NotEmpty(t, objectNames) + + fs, err := colexec.GetSharedFSFromProc(proc) + require.NoError(t, err) + for _, name := range objectNames { + _, statErr := fs.StatFile(proc.Ctx, name) + require.NoError(t, statErr, "the sync succeeded before workspace retention failed") + } + require.NoError(t, writer.reset(proc, true)) + for _, name := range objectNames { + _, statErr := fs.StatFile(proc.Ctx, name) + require.True(t, moerr.IsMoErrCode(statErr, moerr.ErrFileNotFound), + "abort must clean an object whose owner could not be handed to the workspace: %s (%v)", name, statErr) + } +} + +func TestSortAndSyncOneTableReleasesWriterMemoryAcrossSpills(t *testing.T) { + _, ctrl, proc := prepareTestCtx(t, true) + defer proc.Free() + workspace := &multiUpdateS3CleanupWorkspace{} + txnOp := mock_frontend.NewMockTxnOperator(ctrl) + txnOp.EXPECT().GetWorkspace().Return(workspace).AnyTimes() + proc.Base.TxnOperator = txnOp + _, tableDef := getTestMainTable() + updateCtx := &MultiUpdateCtx{ + TableDef: tableDef, + InsertCols: []int{0, 1, 2, 3}, + } + update := &MultiUpdate{MultiUpdateCtx: []*MultiUpdateCtx{updateCtx}} + update.resetMultiUpdateCtxs() + writer, err := newS3Writer(proc.GetService(), update) + require.NoError(t, err) + defer func() { require.NoError(t, writer.free(proc, true)) }() + fs, err := colexec.GetSharedFSFromProc(proc) + require.NoError(t, err) + analyzer := process.NewAnalyzer(0, false, false, "multi-update-multi-spill-memory") + baselineMP := proc.Mp().CurrNB() + acceptedObjectNames := make([]string, 0, 4) + + spill := func() []string { + bats, _ := prepareTestInsertBatchs(proc.Mp(), 2, colexec.DefaultBatchSize, false, false) + before := len(writer.syncedObjectOwners) + require.NoError(t, writer.sortAndSyncOneTable( + proc, + tableDef, + analyzer, + 0, + false, + bats, + true, + )) + require.Empty(t, writer.failedWriters, "successful one-shot writers must be closed immediately") + require.Greater(t, len(writer.syncedObjectOwners), before, "each successful spill must transfer its cleanup names") + require.NotNil(t, writer.insertBlockInfo[0]) + spillObjectNames := objectNamesFromInsertInfo(t, writer.insertBlockInfo[0]) + ownerNames := append([]string(nil), spillObjectNames...) + slices.Sort(ownerNames) + require.Equal(t, ownerNames, unpublishedObjectOwnerNames(writer.syncedObjectOwners[before:])) + for _, name := range spillObjectNames { + _, statErr := fs.StatFile(proc.Ctx, name) + require.NoError(t, statErr) + } + + // Remove the required output metadata before sampling MPool. Any retained + // delta is then temporary Sinker memory, not published metadata growth. + writer.insertBlockInfo[0].Clean(proc.Mp()) + writer.insertBlockInfo[0] = nil + writer.insertBlockRowCount[0] = 0 + require.Equal(t, baselineMP, proc.Mp().CurrNB(), "completed one-shot writer memory must not accumulate across spills") + return spillObjectNames + } + + for range 3 { + acceptedObjectNames = append(acceptedObjectNames, spill()...) + } + require.NoError(t, writer.reset(proc, false)) + require.Empty(t, writer.syncedObjectOwners, "the workspace keeps the lease after producer reset") + workspace.AcceptUnpublishedS3ObjectNames(acceptedObjectNames...) + require.NoError(t, workspace.CleanupUnpublishedS3Objects(proc.Ctx)) + for _, name := range acceptedObjectNames { + _, statErr := fs.StatFile(proc.Ctx, name) + require.NoError(t, statErr, "a successful handoff must preserve accepted objects") + } + + unacceptedObjectNames := spill() + require.NoError(t, writer.reset(proc, true)) + require.NoError(t, workspace.CleanupUnpublishedS3Objects(proc.Ctx)) + for _, name := range unacceptedObjectNames { + _, err = fs.StatFile(proc.Ctx, name) + require.True(t, moerr.IsMoErrCode(err, moerr.ErrFileNotFound), "abort must delete the current execution's object %s, got %v", name, err) + } + for _, name := range acceptedObjectNames { + _, err = fs.StatFile(proc.Ctx, name) + require.NoError(t, err, "a later abort must not delete an earlier accepted object") + } +} + +func unpublishedObjectOwnerNames(owners []*colexec.UnpublishedS3ObjectOwner) []string { + var names []string + for _, owner := range owners { + names = append(names, owner.Names()...) + } + slices.Sort(names) + return names +} + +func TestRemoteMultiUpdateOutputTransfersCleanupBeforeRegistration(t *testing.T) { + _, ctrl, proc := prepareTestCtx(t, true) + defer proc.Free() + workerWorkspace := &multiUpdateS3CleanupWorkspace{} + coordinatorWorkspace := &multiUpdateS3CleanupWorkspace{} + workerTxn := mock_frontend.NewMockTxnOperator(ctrl) + workerTxn.EXPECT().GetWorkspace().Return(workerWorkspace).AnyTimes() + proc.Base.TxnOperator = workerTxn + + _, tableDef := getTestMainTable() + fs, err := colexec.GetSharedFSFromProc(proc) + require.NoError(t, err) + updateCtx := &MultiUpdateCtx{TableDef: tableDef, InsertCols: []int{0, 1, 2, 3}} + arg := &MultiUpdate{MultiUpdateCtx: []*MultiUpdateCtx{updateCtx}} + arg.resetMultiUpdateCtxs() + delegate, err := newS3Writer(proc.GetService(), arg) + require.NoError(t, err) + batches, _ := prepareTestInsertBatchs(proc.Mp(), 2, colexec.DefaultBatchSize, false, false) + require.NoError(t, delegate.sortAndSyncOneTable( + proc, + tableDef, + process.NewAnalyzer(0, false, false, "remote-multi-update-owner-handoff"), + 0, + false, + batches, + true, + )) + wantNames := objectNamesFromInsertInfo(t, delegate.insertBlockInfo[0]) + slices.Sort(wantNames) + output := makeS3OutputBatch() + outputWriter := &s3WriterDelegate{ + updateCtxs: []*MultiUpdateCtx{updateCtx}, + outputBat: output, + } + require.NoError(t, outputWriter.addBatchToOutput( + proc.Mp(), actionInsert, 0, uint64(delegate.insertBlockInfo[0].RowCount()), "", delegate.insertBlockInfo[0], + )) + require.NoError(t, delegate.reset(proc, false)) + require.True(t, workerWorkspace.HasUnpublishedS3ObjectOwners(), + "producer reset must leave names owned until the remote batch is transferred") + + coordinatorTxn := mock_frontend.NewMockTxnOperator(ctrl) + coordinatorTxn.EXPECT().GetWorkspace().Return(coordinatorWorkspace).AnyTimes() + proc.Base.TxnOperator = coordinatorTxn + require.NoError(t, RetainOutputS3ObjectOwnership(proc, output)) + require.Equal(t, wantNames, coordinatorWorkspace.pendingObjectNames()) + + // A successful batch ACK transfers ownership to the coordinator workspace; + // if registration then fails, its statement cleanup still has the names. + workerWorkspace.AcceptAllUnpublishedS3ObjectOwners() + output.Clean(proc.Mp()) + for _, name := range wantNames { + _, statErr := fs.StatFile(proc.Ctx, name) + require.NoError(t, statErr) + } + require.NoError(t, coordinatorWorkspace.CleanupUnpublishedS3Objects(proc.Ctx)) + for _, name := range wantNames { + _, statErr := fs.StatFile(proc.Ctx, name) + require.True(t, moerr.IsMoErrCode(statErr, moerr.ErrFileNotFound), + "coordinator must clean an object when registration never happens: %s (%v)", name, statErr) + } +} + +func objectNamesFromInsertInfo(t *testing.T, info *batch.Batch) []string { + t.Helper() + statsData, statsArea := vector.MustVarlenaRawData(info.Vecs[1]) + objectNames := make([]string, len(statsData)) + for i := range statsData { + stats := objectio.ObjectStats(statsData[i].GetByteSlice(statsArea)) + objectNames[i] = stats.ObjectName().String() + } + return objectNames +} + +func TestMultiUpdateResetKeepsFailedCleanupOutOfReusableState(t *testing.T) { + _, _, proc := prepareTestCtx(t, true) + defer proc.Free() + _, tableDef := getTestMainTable() + baseFS, err := colexec.GetSharedFSFromProc(proc) + require.NoError(t, err) + deleteErr := errors.New("injected multi-update cleanup failure") + fs := &failOnceMultiUpdateDeleteFS{FileService: baseFS, failErr: deleteErr} + freeList := containers.NewBatchFreeList(nil, nil, true) + s3Writer := colexec.NewCNS3DataWriter( + proc.Mp(), fs, tableDef, 1, false, + ioutil.WithBuffer(freeList, false), + ) + cleanupDone := false + var nextBatches []*batch.Batch + defer func() { + if !cleanupDone { + _ = s3Writer.CloseWithCleanup(proc.Ctx, true) + freeList.Close(proc.Mp()) + } + for _, bat := range nextBatches { + if bat != nil { + bat.Clean(proc.Mp()) + } + } + }() + bats, _ := prepareTestInsertBatchs(proc.Mp(), 1, colexec.DefaultBatchSize, false, false) + blockInfo, err := func() (*batch.Batch, error) { + defer bats[0].Clean(proc.Mp()) + if err := s3Writer.Write(proc.Ctx, bats[0]); err != nil { + return nil, err + } + return s3Writer.SyncAndFillBlockInfoBat(proc.Ctx) + }() + require.NoError(t, err) + data, area := vector.MustVarlenaRawData(blockInfo.Vecs[1]) + firstStats := objectio.ObjectStats(data[0].GetByteSlice(area)) + firstObject := firstStats.ObjectName().String() + _, err = baseFS.StatFile(proc.Ctx, firstObject) + require.NoError(t, err) + + delegate := &s3WriterDelegate{ + insertSinkers: []*colexec.CNS3Writer{s3Writer}, + insertFreeLists: []*containers.BatchFreeList{freeList}, + insertBlockInfo: []*batch.Batch{blockInfo}, + insertBlockRowCount: []uint64{0}, + } + require.ErrorIs(t, delegate.reset(proc, true), deleteErr) + require.Error(t, delegate.cleanupErr) + require.Same(t, freeList, delegate.insertFreeLists[0], "pending sinker still borrows this buffer pool") + _, err = baseFS.StatFile(proc.Ctx, firstObject) + require.NoError(t, err, "failed cleanup must retain the old object for retry") + + // A later success-shaped reset is not a handoff. It retries abort cleanup + // and discards the detached writer; a new execution needs a fresh sinker. + require.NoError(t, delegate.reset(proc, false)) + require.NoError(t, delegate.cleanupErr) + require.Same(t, freeList, delegate.insertFreeLists[0]) + require.Nil(t, delegate.insertSinkers[0]) + _, err = baseFS.StatFile(proc.Ctx, firstObject) + require.True(t, moerr.IsMoErrCode(err, moerr.ErrFileNotFound), "old object should be deleted before reuse, got %v", err) + + nextWriter := colexec.NewCNS3DataWriter(proc.Mp(), fs, tableDef, 1, false, + ioutil.WithBuffer(freeList, false)) + defer func() { + if !cleanupDone { + _ = nextWriter.CloseWithCleanup(proc.Ctx, true) + } + }() + delegate.insertSinkers[0] = nextWriter + nextBatches, _ = prepareTestInsertBatchs(proc.Mp(), 1, colexec.DefaultBatchSize, false, false) + require.NoError(t, nextWriter.Write(proc.Ctx, nextBatches[0])) + secondInfo, err := nextWriter.SyncAndFillBlockInfoBat(proc.Ctx) + require.NoError(t, err) + secondData, secondArea := vector.MustVarlenaRawData(secondInfo.Vecs[1]) + secondStats := objectio.ObjectStats(secondData[0].GetByteSlice(secondArea)) + secondObject := secondStats.ObjectName().String() + require.NoError(t, delegate.free(proc, true)) + cleanupDone = true + _, err = baseFS.StatFile(proc.Ctx, secondObject) + require.True(t, moerr.IsMoErrCode(err, moerr.ErrFileNotFound), "failed next execution cleanup should delete its object, got %v", err) +} + +type multiUpdateS3CleanupWorkspace struct { + client.Workspace + cleanups []func(context.Context) error + owners []*colexec.UnpublishedS3ObjectOwner +} + +func (w *multiUpdateS3CleanupWorkspace) RetainUnpublishedS3Cleanup( + cleanup func(context.Context) error, +) { + w.cleanups = append(w.cleanups, cleanup) +} + +func (w *multiUpdateS3CleanupWorkspace) RetainUnpublishedS3ObjectOwner( + owner *colexec.UnpublishedS3ObjectOwner, +) { + w.owners = append(w.owners, owner) +} + +func (w *multiUpdateS3CleanupWorkspace) AcceptUnpublishedS3ObjectNames(names ...string) { + for _, owner := range w.owners { + owner.Accept(names...) + } + w.dropAcceptedOwners() +} + +func (w *multiUpdateS3CleanupWorkspace) HasUnpublishedS3ObjectOwners() bool { + return len(w.owners) != 0 +} + +func (w *multiUpdateS3CleanupWorkspace) AcceptAllUnpublishedS3ObjectOwners() { + for _, owner := range w.owners { + owner.AcceptAll() + } + w.owners = nil +} + +func (w *multiUpdateS3CleanupWorkspace) CleanupUnpublishedS3Objects(ctx context.Context) error { + var errs []error + for _, cleanup := range w.cleanups { + if err := cleanup(ctx); err != nil { + errs = append(errs, err) + } + } + w.cleanups = nil + for _, owner := range w.owners { + if err := owner.Cleanup(ctx); err != nil { + errs = append(errs, err) + } + } + w.dropAcceptedOwners() + return errors.Join(errs...) +} + +func (w *multiUpdateS3CleanupWorkspace) pendingObjectNames() []string { + return unpublishedObjectOwnerNames(w.owners) +} + +func (w *multiUpdateS3CleanupWorkspace) dropAcceptedOwners() { + owners := w.owners[:0] + for _, owner := range w.owners { + if owner.Pending() { + owners = append(owners, owner) + } + } + w.owners = owners +} + +func TestMultiUpdateFreeTransfersFailedCleanupWithBorrowedBuffer(t *testing.T) { + t.Run("Free", func(t *testing.T) { checkMultiUpdateFreeTransfersFailedCleanupWithBorrowedBuffer(t, false) }) + t.Run("prepared Reset", func(t *testing.T) { checkMultiUpdateFreeTransfersFailedCleanupWithBorrowedBuffer(t, true) }) +} + +func checkMultiUpdateFreeTransfersFailedCleanupWithBorrowedBuffer(t *testing.T, prepared bool) { + _, ctrl, proc := prepareTestCtx(t, true) + defer proc.Free() + baseFS, err := colexec.GetSharedFSFromProc(proc) + require.NoError(t, err) + deleteErr := errors.New("injected multi-update final cleanup failure") + fs := &failOnceMultiUpdateDeleteFS{FileService: baseFS, failErr: deleteErr} + freeList := containers.NewBatchFreeList(nil, nil, true) + _, tableDef := getTestMainTable() + s3Writer := colexec.NewCNS3DataWriter( + proc.Mp(), fs, tableDef, 1, false, + ioutil.WithBuffer(freeList, false), + ) + batches, _ := prepareTestInsertBatchs(proc.Mp(), 1, colexec.DefaultBatchSize, false, false) + require.NoError(t, s3Writer.Write(proc.Ctx, batches[0])) + batches[0].Clean(proc.Mp()) + blockInfo, err := s3Writer.SyncAndFillBlockInfoBat(proc.Ctx) + require.NoError(t, err) + data, area := vector.MustVarlenaRawData(blockInfo.Vecs[1]) + stats := objectio.ObjectStats(data[0].GetByteSlice(area)) + objectName := stats.ObjectName().String() + + delegate := &s3WriterDelegate{ + insertSinkers: []*colexec.CNS3Writer{s3Writer}, + insertFreeLists: []*containers.BatchFreeList{freeList}, + insertBlockInfo: []*batch.Batch{blockInfo}, + insertBlockRowCount: []uint64{0}, + } + txnOp := mock_frontend.NewMockTxnOperator(ctrl) + workspace := &multiUpdateS3CleanupWorkspace{} + txnOp.EXPECT().GetWorkspace().Return(workspace).AnyTimes() + proc.Base.TxnOperator = txnOp + update := &MultiUpdate{} + update.ctr.s3Writer = delegate + + if prepared { + update.Reset(proc, true, deleteErr) + } else { + update.Free(proc, true, deleteErr) + } + require.Nil(t, update.ctr.s3Writer, "the transaction owns a detached cleanup task after operator release") + require.Len(t, workspace.cleanups, 1) + require.Nil(t, delegate.insertFreeLists, "retry must not retain the shared buffer pool") + require.Nil(t, delegate.insertSinkers, "retry must not retain the delegate") + require.Nil(t, delegate.cleanupMP, "retry must not retain the session mpool") + _, err = baseFS.StatFile(proc.Ctx, objectName) + require.NoError(t, err) + // A second storage outage during transaction teardown retains the name-only + // task without pinning the producer's buffer pool or session mpool. + fs.failed = false + require.ErrorIs(t, workspace.cleanups[0](proc.Ctx), deleteErr) + require.Nil(t, delegate.insertSinkers) + require.Nil(t, delegate.insertFreeLists) + _, err = baseFS.StatFile(proc.Ctx, objectName) + require.NoError(t, err) + require.NoError(t, workspace.cleanups[0](proc.Ctx)) + require.Nil(t, delegate.insertSinkers) + require.Nil(t, delegate.insertFreeLists) + require.Zero(t, freeList.Len()) + _, err = baseFS.StatFile(proc.Ctx, objectName) + require.True(t, moerr.IsMoErrCode(err, moerr.ErrFileNotFound), "transaction retry should delete the object, got %v", err) +} + +func TestMultiUpdateFailedOneShotWriterCleanupRetries(t *testing.T) { + _, _, proc := prepareTestCtx(t, true) + defer proc.Free() + baseFS, err := colexec.GetSharedFSFromProc(proc) + require.NoError(t, err) + deleteErr := errors.New("injected one-shot writer cleanup failure") + fs := &failOnceMultiUpdateDeleteFS{FileService: baseFS, failErr: deleteErr} + _, tableDef := getTestMainTable() + writer := colexec.NewCNS3DataWriter(proc.Mp(), fs, tableDef, 1, false) + batches, _ := prepareTestInsertBatchs(proc.Mp(), 1, colexec.DefaultBatchSize, false, false) + require.NoError(t, writer.Write(proc.Ctx, batches[0])) + batches[0].Clean(proc.Mp()) + blockInfo, err := writer.SyncAndFillBlockInfoBat(proc.Ctx) + require.NoError(t, err) + stats := objectio.ObjectStats(blockInfo.Vecs[1].GetBytesAt(0)) + name := stats.ObjectName().String() + blockInfo.Clean(proc.Mp()) + + delegate := &s3WriterDelegate{failedWriters: []*colexec.CNS3Writer{nil, writer}} + require.ErrorIs(t, delegate.cleanupUnpublishedS3Objects(proc.Ctx), deleteErr) + require.Same(t, writer, delegate.failedWriters[1]) + _, err = baseFS.StatFile(proc.Ctx, name) + require.NoError(t, err) + require.NoError(t, delegate.cleanupUnpublishedS3Objects(proc.Ctx)) + require.Empty(t, delegate.failedWriters) + _, err = baseFS.StatFile(proc.Ctx, name) + require.True(t, moerr.IsMoErrCode(err, moerr.ErrFileNotFound), "%v", err) +} + +func TestMultiUpdateFreeTransfersSyncedObjectCleanup(t *testing.T) { + _, ctrl, proc := prepareTestCtx(t, true) + defer proc.Free() + baseFS, err := colexec.GetSharedFSFromProc(proc) + require.NoError(t, err) + objectName := "multi-update-free-synced-object" + require.NoError(t, baseFS.Write(proc.Ctx, fileservice.IOVector{ + FilePath: objectName, + Entries: []fileservice.IOEntry{{Size: 1, Data: []byte("x")}}, + Policy: fileservice.SkipAllCache, + })) + deleteErr := errors.New("injected synced-object cleanup failure during Free") + owner, err := colexec.NewUnpublishedS3ObjectOwner( + &failOnceMultiUpdateDeleteFS{FileService: baseFS, failErr: deleteErr}, + objectName, + ) + require.NoError(t, err) + txnOp := mock_frontend.NewMockTxnOperator(ctrl) + workspace := &multiUpdateS3CleanupWorkspace{} + txnOp.EXPECT().GetWorkspace().Return(workspace).AnyTimes() + proc.Base.TxnOperator = txnOp + workspace.RetainUnpublishedS3ObjectOwner(owner) + delegate := &s3WriterDelegate{syncedObjectOwners: []*colexec.UnpublishedS3ObjectOwner{owner}} + update := &MultiUpdate{} + update.ctr.s3Writer = delegate + update.Free(proc, true, deleteErr) + require.Nil(t, update.ctr.s3Writer, "the transaction owns the object-name cleanup record after operator release") + require.Len(t, workspace.cleanups, 1) + require.True(t, owner.Pending()) + _, err = baseFS.StatFile(proc.Ctx, objectName) + require.NoError(t, err, "failed deletion must preserve the object for the transaction retry") + + require.NoError(t, workspace.cleanups[0](proc.Ctx)) + require.Empty(t, delegate.syncedObjectOwners) + require.False(t, owner.Pending()) + require.Nil(t, delegate.insertFreeLists) + _, err = baseFS.StatFile(proc.Ctx, objectName) + require.True(t, moerr.IsMoErrCode(err, moerr.ErrFileNotFound), "transaction retry should delete the object, got %v", err) +} + +func TestMultiUpdateResetRetriesFailedSyncedObjectCleanup(t *testing.T) { + _, _, proc := prepareTestCtx(t, true) + defer proc.Free() + fs, err := colexec.GetSharedFSFromProc(proc) + require.NoError(t, err) + objectName := "multi-update-reset-retry-object" + require.NoError(t, fs.Write(proc.Ctx, fileservice.IOVector{ + FilePath: objectName, + Entries: []fileservice.IOEntry{{Size: 1, Data: []byte("x")}}, + Policy: fileservice.SkipAllCache, + })) + deleteErr := errors.New("injected reset cleanup failure") + owner, err := colexec.NewUnpublishedS3ObjectOwner( + &failOnceMultiUpdateDeleteFS{FileService: fs, failErr: deleteErr}, + objectName, + ) + require.NoError(t, err) + delegate := &s3WriterDelegate{syncedObjectOwners: []*colexec.UnpublishedS3ObjectOwner{owner}} + + require.ErrorIs(t, delegate.reset(proc, true), deleteErr) + require.True(t, owner.Pending(), "a failed delete must retain the exact cleanup owner") + require.NoError(t, delegate.reset(proc, false), "a later reset must retry, not accept, the failed cleanup") + require.False(t, owner.Pending()) + _, err = fs.StatFile(proc.Ctx, objectName) + require.True(t, moerr.IsMoErrCode(err, moerr.ErrFileNotFound), "retry should delete the unaccepted object, got %v", err) +} + +func TestMultiUpdateRetryTaskSkipsReusableSinkers(t *testing.T) { + _, _, proc := prepareTestCtx(t, true) + defer proc.Free() + fs, err := colexec.GetSharedFSFromProc(proc) + require.NoError(t, err) + _, tableDef := getTestMainTable() + idle := colexec.NewCNS3DataWriter(proc.Mp(), fs, tableDef, 1, false) + owner, err := colexec.NewUnpublishedS3ObjectOwner(fs, "pending-object") + require.NoError(t, err) + delegate := &s3WriterDelegate{ + insertSinkers: []*colexec.CNS3Writer{idle}, + syncedObjectOwners: []*colexec.UnpublishedS3ObjectOwner{owner}, + cleanupErr: errors.New("unpublished object pending"), + } + task := delegate.retryTask() + require.Len(t, task.owners, 1) + require.Empty(t, task.insertSinkers, "a reusable writer with no ticket must not enter the retry task") + delegate.releaseBuffers(proc.Mp()) + require.Nil(t, delegate.insertSinkers[0]) + require.Nil(t, delegate.cleanupMP) +} + +func TestPartitionMultiUpdateFreeTransfersFailedCleanup(t *testing.T) { + t.Run("Free", func(t *testing.T) { checkPartitionMultiUpdateFreeTransfersFailedCleanup(t, false) }) + t.Run("prepared Reset", func(t *testing.T) { checkPartitionMultiUpdateFreeTransfersFailedCleanup(t, true) }) +} + +func checkPartitionMultiUpdateFreeTransfersFailedCleanup(t *testing.T, prepared bool) { + _, ctrl, proc := prepareTestCtx(t, true) + defer proc.Free() + baseFS, err := colexec.GetSharedFSFromProc(proc) + require.NoError(t, err) + deleteErr := errors.New("injected partition multi-update cleanup failure") + fs := &failOnceMultiUpdateDeleteFS{FileService: baseFS, failErr: deleteErr} + freeList := containers.NewBatchFreeList(nil, nil, true) + _, tableDef := getTestMainTable() + s3Writer := colexec.NewCNS3DataWriter( + proc.Mp(), fs, tableDef, 1, false, + ioutil.WithBuffer(freeList, false), + ) + batches, _ := prepareTestInsertBatchs(proc.Mp(), 1, colexec.DefaultBatchSize, false, false) + require.NoError(t, s3Writer.Write(proc.Ctx, batches[0])) + batches[0].Clean(proc.Mp()) + blockInfo, err := s3Writer.SyncAndFillBlockInfoBat(proc.Ctx) + require.NoError(t, err) + data, area := vector.MustVarlenaRawData(blockInfo.Vecs[1]) + stats := objectio.ObjectStats(data[0].GetByteSlice(area)) + objectName := stats.ObjectName().String() + delegate := &s3WriterDelegate{ + insertSinkers: []*colexec.CNS3Writer{s3Writer}, + insertFreeLists: []*containers.BatchFreeList{freeList}, + insertBlockInfo: []*batch.Batch{blockInfo}, + insertBlockRowCount: []uint64{0}, + } + workspace := &multiUpdateS3CleanupWorkspace{} + txnOp := mock_frontend.NewMockTxnOperator(ctrl) + txnOp.EXPECT().GetWorkspace().Return(workspace).AnyTimes() + proc.Base.TxnOperator = txnOp + op := &PartitionMultiUpdate{ + raw: NewArgument(), + writers: map[uint64]*s3WriterDelegate{1: delegate}, + } + + if prepared { + op.Reset(proc, true, deleteErr) + } else { + op.Free(proc, true, deleteErr) + } + require.Empty(t, op.writers, "transaction ownership must outlive the partition operator") + require.Len(t, workspace.cleanups, 1) + op.Release() + require.NoError(t, workspace.cleanups[0](proc.Ctx)) + require.Nil(t, delegate.insertFreeLists) + require.Zero(t, freeList.Len()) + _, err = baseFS.StatFile(proc.Ctx, objectName) + require.True(t, moerr.IsMoErrCode(err, moerr.ErrFileNotFound), "transaction retry should delete the object, got %v", err) +} + +type failOnceMultiUpdateDeleteFS struct { + fileservice.FileService + failErr error + failed bool +} + +func (fs *failOnceMultiUpdateDeleteFS) Delete(ctx context.Context, names ...string) error { + if !fs.failed { + fs.failed = true + return fs.failErr + } + return fs.FileService.Delete(ctx, names...) } // update table s3 diff --git a/pkg/sql/colexec/multi_update/util_for_test.go b/pkg/sql/colexec/multi_update/util_for_test.go index 0d1c547d41a79..a0be6d8269d32 100644 --- a/pkg/sql/colexec/multi_update/util_for_test.go +++ b/pkg/sql/colexec/multi_update/util_for_test.go @@ -35,6 +35,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/fileservice" mock_frontend "github.com/matrixorigin/matrixone/pkg/frontend/test" pbPlan "github.com/matrixorigin/matrixone/pkg/pb/plan" + "github.com/matrixorigin/matrixone/pkg/pb/txn" "github.com/matrixorigin/matrixone/pkg/perfcounter" "github.com/matrixorigin/matrixone/pkg/sql/colexec" "github.com/matrixorigin/matrixone/pkg/sql/plan" @@ -213,6 +214,8 @@ func prepareTestCtx(t *testing.T, withFs bool) (context.Context, *gomock.Control txnOperator := mock_frontend.NewMockTxnOperator(ctrl) txnOperator.EXPECT().Commit(gomock.Any()).Return(nil).AnyTimes() txnOperator.EXPECT().Rollback(ctx).Return(nil).AnyTimes() + txnOperator.EXPECT().Txn().Return(txn.TxnMeta{}).AnyTimes() + txnOperator.EXPECT().GetWorkspace().Return(&multiUpdateS3CleanupWorkspace{}).AnyTimes() txnClient := mock_frontend.NewMockTxnClient(ctrl) txnClient.EXPECT().New(gomock.Any(), gomock.Any()).Return(txnOperator, nil).AnyTimes() @@ -253,6 +256,7 @@ func prepareTestCtx(t *testing.T, withFs bool) (context.Context, *gomock.Control }) proc.Base.TxnClient = txnClient + proc.Base.TxnOperator = txnOperator proc.Ctx = ctx _ = colexec.NewServer(proc.GetService()) diff --git a/pkg/sql/colexec/s3util.go b/pkg/sql/colexec/s3util.go index 25b2648279d42..475c1cb6bf855 100644 --- a/pkg/sql/colexec/s3util.go +++ b/pkg/sql/colexec/s3util.go @@ -19,6 +19,9 @@ import ( "context" "fmt" "math" + "strings" + "sync" + "time" "github.com/matrixorigin/matrixone/pkg/catalog" "github.com/matrixorigin/matrixone/pkg/common/moerr" @@ -48,11 +51,288 @@ const ( type CNS3Writer struct { sinker *ioutil.Sinker + fs fileservice.FileService + serviceID string + ownedPersistedNames []string + cleanupPending bool isTombstone bool blockInfoBat *batch.Batch memorySizeThreshold int } +// PendingUnpublishedCleanup reports whether a failed writer still owns object +// names that must be deleted. A reusable writer without debt must not enter a +// transaction retry task. +func (w *CNS3Writer) PendingUnpublishedCleanup() bool { + return w != nil && (w.cleanupPending || len(w.ownedPersistedNames) != 0) +} + +func (w *CNS3Writer) ReadyForWrites() bool { + return w != nil && w.sinker != nil && !w.cleanupPending +} + +// UnpublishedS3CleanupRetainer is implemented by workspaces that can own a +// failed cleanup after the operator that created the object is released. +type UnpublishedS3CleanupRetainer interface { + RetainUnpublishedS3Cleanup(func(context.Context) error) +} + +// UnpublishedS3ObjectOwner keeps only the persisted object names that have +// not yet crossed the transaction-workspace registration boundary. It can be +// retained after the writer's buffers are released, then either accept names +// as workspace entries are appended or delete the remainder on abort. +type UnpublishedS3ObjectOwner struct { + mu sync.Mutex + fs fileservice.FileService + names map[string]struct{} + admission *unpublishedS3Admission + charged map[string]struct{} +} + +func NewUnpublishedS3ObjectOwner( + fs fileservice.FileService, + names ...string, +) (*UnpublishedS3ObjectOwner, error) { + return newUnpublishedS3ObjectOwner(fs, nil, nil, names...) +} + +// NewUnpublishedS3ObjectOwnerForService transfers names already charged by a +// local CN sinker into a lightweight transaction owner. +func NewUnpublishedS3ObjectOwnerForService( + serviceID string, + fs fileservice.FileService, + names ...string, +) (*UnpublishedS3ObjectOwner, error) { + if serviceID == "" { + return NewUnpublishedS3ObjectOwner(fs, names...) + } + srv := GetServer(serviceID) + if srv == nil { + return nil, moerr.NewInvalidStateNoCtx("missing CN unpublished S3 admission service") + } + return newUnpublishedS3ObjectOwner(fs, srv.unpublishedS3Admission, names, names...) +} + +func newUnpublishedS3ObjectOwner( + fs fileservice.FileService, + admission *unpublishedS3Admission, + charged []string, + names ...string, +) (*UnpublishedS3ObjectOwner, error) { + if fs == nil { + return nil, moerr.NewInternalErrorNoCtx("missing file service for unpublished S3 object owner") + } + owner := &UnpublishedS3ObjectOwner{ + fs: fs, + names: make(map[string]struct{}, len(names)), + admission: admission, + } + for _, name := range names { + if name != "" { + owner.names[strings.Clone(name)] = struct{}{} + } + } + if len(owner.names) == 0 { + return nil, nil + } + if admission != nil { + owner.charged = make(map[string]struct{}, len(charged)) + for _, name := range charged { + if _, exists := owner.names[name]; exists { + owner.charged[name] = struct{}{} + } + } + } + return owner, nil +} + +func (owner *UnpublishedS3ObjectOwner) Names() []string { + owner.mu.Lock() + defer owner.mu.Unlock() + names := make([]string, 0, len(owner.names)) + for name := range owner.names { + names = append(names, name) + } + return names +} + +func (owner *UnpublishedS3ObjectOwner) Accept(names ...string) { + owner.mu.Lock() + defer owner.mu.Unlock() + for _, name := range names { + delete(owner.names, name) + owner.release(name) + } +} + +func (owner *UnpublishedS3ObjectOwner) AcceptAll() { + owner.mu.Lock() + clear(owner.names) + for name := range owner.charged { + owner.release(name) + } + owner.mu.Unlock() +} + +func (owner *UnpublishedS3ObjectOwner) release(name string) { + if _, charged := owner.charged[name]; charged { + owner.admission.release(name) + delete(owner.charged, name) + } +} + +func (owner *UnpublishedS3ObjectOwner) Pending() bool { + owner.mu.Lock() + defer owner.mu.Unlock() + return len(owner.names) != 0 +} + +func (owner *UnpublishedS3ObjectOwner) Cleanup(ctx context.Context) error { + owner.mu.Lock() + defer owner.mu.Unlock() + if len(owner.names) == 0 { + return nil + } + names := make([]string, 0, len(owner.names)) + for name := range owner.names { + names = append(names, name) + } + completed, _, err := ioutil.DeleteUnpublishedObjectsWithProgress(ctx, owner.fs, names...) + for _, name := range completed { + delete(owner.names, name) + owner.release(name) + } + return err +} + +type UnpublishedS3ObjectOwnershipWorkspace interface { + RetainUnpublishedS3ObjectOwner(*UnpublishedS3ObjectOwner) + AcceptUnpublishedS3ObjectNames(...string) + HasUnpublishedS3ObjectOwners() bool + AcceptAllUnpublishedS3ObjectOwners() +} + +func RetainUnpublishedS3ObjectOwner( + proc *process.Process, + owner *UnpublishedS3ObjectOwner, +) bool { + if proc == nil || owner == nil || !owner.Pending() || proc.GetTxnOperator() == nil { + return false + } + retainer, ok := proc.GetTxnOperator().GetWorkspace().(UnpublishedS3ObjectOwnershipWorkspace) + if !ok { + return false + } + retainer.RetainUnpublishedS3ObjectOwner(owner) + return true +} + +// RetainReceivedUnpublishedS3ObjectNames reserves coordinator capacity before +// acknowledging a remote worker's ownership handoff. A failed retention rolls +// back only newly reserved names; existing owners remain charged. +func RetainReceivedUnpublishedS3ObjectNames( + proc *process.Process, + fs fileservice.FileService, + names ...string, +) error { + if len(names) == 0 { + return nil + } + if proc == nil { + return moerr.NewInvalidStateNoCtx("missing process for remote S3 ownership") + } + srv := GetServer(proc.GetService()) + if srv == nil { + return moerr.NewInvalidStateNoCtx("missing CN unpublished S3 admission service") + } + charged, err := srv.reserveUnpublishedS3Received(names) + if err != nil { + return err + } + owner, err := newUnpublishedS3ObjectOwner(fs, srv.unpublishedS3Admission, charged, names...) + if err == nil && !RetainUnpublishedS3ObjectOwner(proc, owner) { + err = moerr.NewInvalidStateNoCtx("receiving transaction workspace cannot retain remote S3 ownership") + } + if err != nil { + for _, name := range charged { + srv.unpublishedS3Admission.release(name) + } + } + return err +} + +func AcceptUnpublishedS3ObjectNames(proc *process.Process, names ...string) { + if proc == nil || proc.GetTxnOperator() == nil { + return + } + accepter, ok := proc.GetTxnOperator().GetWorkspace().(UnpublishedS3ObjectOwnershipWorkspace) + if !ok { + return + } + accepter.AcceptUnpublishedS3ObjectNames(names...) +} + +func HasUnpublishedS3ObjectOwners(proc *process.Process) bool { + if proc == nil || proc.GetTxnOperator() == nil { + return false + } + owners, ok := proc.GetTxnOperator().GetWorkspace().(UnpublishedS3ObjectOwnershipWorkspace) + return ok && owners.HasUnpublishedS3ObjectOwners() +} + +func AcceptAllUnpublishedS3ObjectOwners(proc *process.Process) { + if proc == nil || proc.GetTxnOperator() == nil { + return + } + owners, ok := proc.GetTxnOperator().GetWorkspace().(UnpublishedS3ObjectOwnershipWorkspace) + if ok { + owners.AcceptAllUnpublishedS3ObjectOwners() + } +} + +// RetainUnpublishedS3Cleanup transfers cleanup ownership to the transaction +// workspace when a writer's final operator callback cannot delete its objects. +func RetainUnpublishedS3Cleanup( + proc *process.Process, + cleanup func(context.Context) error, +) bool { + if proc == nil || cleanup == nil || proc.GetTxnOperator() == nil { + return false + } + retainer, ok := proc.GetTxnOperator().GetWorkspace().(UnpublishedS3CleanupRetainer) + if !ok { + return false + } + retainer.RetainUnpublishedS3Cleanup(cleanup) + return true +} + +// UnpublishedS3CleanupContext ignores request cancellation while preserving an +// existing cleanup deadline. Nested writer cleanup must not restart the outer +// finalizer's budget for each object or writer. +func UnpublishedS3CleanupContext(ctx context.Context) (context.Context, context.CancelFunc) { + base := context.WithoutCancel(ctx) + deadline := time.Now().Add(10 * time.Minute) + if sharedDeadline, ok := process.PipelineCleanupDeadline(ctx); ok && sharedDeadline.Before(deadline) { + deadline = sharedDeadline + } + if parentDeadline, ok := ctx.Deadline(); ok && parentDeadline.Before(deadline) { + deadline = parentDeadline + } + return context.WithDeadlineCause(base, deadline, moerr.CauseCleanUpUselessFiles) +} + +// TerminalUnpublishedS3CleanupContext starts a bounded cleanup attempt after +// Commit or Rollback even when the request deadline has already expired. A +// pipeline teardown deadline, when present, is still shared by nested owners. +func TerminalUnpublishedS3CleanupContext(ctx context.Context) (context.Context, context.CancelFunc) { + deadline := time.Now().Add(10 * time.Minute) + if sharedDeadline, ok := process.PipelineCleanupDeadline(ctx); ok && sharedDeadline.Before(deadline) { + deadline = sharedDeadline + } + return context.WithDeadlineCause(context.WithoutCancel(ctx), deadline, moerr.CauseCleanUpUselessFiles) +} + func (w *CNS3Writer) String() string { buf := bytes.NewBuffer(nil) buf.WriteString(fmt.Sprintf("Sinker: %s\n", w.sinker.String())) @@ -111,6 +391,8 @@ func newCNS3TombstoneWriter( ) *CNS3Writer { writer := &CNS3Writer{ + fs: fs, + serviceID: serviceID, isTombstone: true, } @@ -123,6 +405,9 @@ func newCNS3TombstoneWriter( if policy := chunkedColumnPolicyForService(serviceID); policy != nil { opts = append(opts, ioutil.WithChunkedColumnPolicy(policy)) } + if serviceID != "" { + opts = append(opts, UnpublishedS3AdmissionOption(serviceID)) + } writer.sinker = ioutil.NewTombstoneSinker( objectio.HiddenColumnSelection_None, @@ -223,6 +508,40 @@ func NewCNS3DataWriterForService( ) } +// UnpublishedS3AdmissionOption attaches the CN's pre-Sync object-name budget +// to a direct sinker, such as tombstone transfer. +func UnpublishedS3AdmissionOption(serviceID string) ioutil.SinkerOption { + return ioutil.WithObjectSyncAdmission( + func(name string) error { + return GetServer(serviceID).reserveUnpublishedS3Upload(name) + }, + func(name string) { + if srv := GetServer(serviceID); srv != nil { + srv.unpublishedS3Admission.release(name) + } + }, + ) +} + +// ReleaseUnpublishedS3Name retires a ticket after a detached transfer flow +// successfully deletes its last unpublished name. +func ReleaseUnpublishedS3Name(serviceID, name string) { + if srv := GetServer(serviceID); srv != nil { + srv.unpublishedS3Admission.release(name) + } +} + +func (w *CNS3Writer) unpublishedS3Admission() (*unpublishedS3Admission, error) { + if w.serviceID == "" { + return nil, nil + } + srv := GetServer(w.serviceID) + if srv == nil { + return nil, moerr.NewInvalidStateNoCtx("missing CN unpublished S3 admission service") + } + return srv.unpublishedS3Admission, nil +} + func newCNS3DataWriter( serviceID string, mp *mpool.MPool, @@ -234,6 +553,8 @@ func newCNS3DataWriter( ) *CNS3Writer { writer := new(CNS3Writer) + writer.fs = fs + writer.serviceID = serviceID sequms, attrTypes, attrs, sortKeyIdx, isPrimaryKey := GetSequmsAttrsSortKeyIdxFromTableDef(tableDef) @@ -259,6 +580,9 @@ func newCNS3DataWriter( if policy := chunkedColumnPolicyForService(serviceID); policy != nil { sinkerOpts = append(sinkerOpts, ioutil.WithChunkedColumnPolicy(policy)) } + if serviceID != "" { + sinkerOpts = append(sinkerOpts, UnpublishedS3AdmissionOption(serviceID)) + } writer.sinker = ioutil.NewSinker( sortKeyIdx, attrs, @@ -315,6 +639,12 @@ func (w *CNS3Writer) SyncAndFillBlockInfoBat(ctx context.Context) (*batch.Batch, if err != nil { return nil, err } + // SyncAndTakeResults transfers the returned stats out of the sinker. Keep + // their cleanup ownership here until the enclosing transaction accepts the + // metadata or the failed operation deletes the objects. + for i := range stats { + w.ownedPersistedNames = append(w.ownedPersistedNames, stats[i].ObjectName().String()) + } w.ResetBlockInfoBat() if len(stats) == 0 { @@ -334,21 +664,175 @@ func (w *CNS3Writer) SyncAndFillBlockInfoBat(ctx context.Context) (*batch.Batch, return w.blockInfoBat, nil } +// TransferPersistedObjects hands completed objects to the transaction after the +// caller has copied their output metadata. No more writes may occur between +// Sync and this call. CN writers flush their entire tail (TailSizeCap=0), so +// resetting after retention releases only completed results and reusable state. +// Until retention succeeds, both sinker and detached names remain writer-owned. +func (w *CNS3Writer) TransferPersistedObjects(proc *process.Process) error { + _, err := w.TransferPersistedObjectsOwner(proc) + return err +} + +// TransferPersistedObjectsOwner returns the lightweight owner for callers +// that must also track a pipeline-level abort before workspace registration. +func (w *CNS3Writer) TransferPersistedObjectsOwner(proc *process.Process) (*UnpublishedS3ObjectOwner, error) { + stats, _ := w.sinker.GetResult() + names := append([]string(nil), w.ownedPersistedNames...) + for i := range stats { + names = append(names, stats[i].ObjectName().String()) + } + if len(names) == 0 { + return nil, nil + } + admission, err := w.unpublishedS3Admission() + if err != nil { + w.cleanupPending = true + return nil, err + } + owner, err := newUnpublishedS3ObjectOwner(w.fs, admission, names, names...) + if err != nil { + w.cleanupPending = true + return nil, err + } + if !RetainUnpublishedS3ObjectOwner(proc, owner) { + w.cleanupPending = true + return nil, moerr.NewInternalErrorNoCtx("transaction workspace cannot retain unpublished S3 objects") + } + w.sinker.Reset() + w.ownedPersistedNames = nil + w.cleanupPending = false + return owner, nil +} + +// DeletePersisted removes objects still owned by this writer, including +// results detached by SyncAndFillBlockInfoBat. On deletion failure the sinker +// and writer retain their exact names so the caller can retry before closing. +func (w *CNS3Writer) DeletePersisted(ctx context.Context) error { + if w == nil { + return nil + } + w.cleanupPending = true + admission, err := w.unpublishedS3Admission() + if err != nil { + return err + } + cleanupCtx, cancel := UnpublishedS3CleanupContext(ctx) + defer cancel() + + if w.sinker != nil { + if names, err := w.sinker.DeletePersisted(cleanupCtx); err != nil { + // The snapshot is complete even after an ambiguous pipeline Sync. + // Keep only names for the retry; the session mpool and sinker buffers + // must not outlive the writer's failed execution. + w.ownedPersistedNames = append(w.ownedPersistedNames, names...) + if w.blockInfoBat != nil { + w.blockInfoBat.Clean(w.sinker.GetMPool()) + w.blockInfoBat = nil + } + _ = w.sinker.Close() + w.sinker = nil + return err + } + } + if len(w.ownedPersistedNames) == 0 { + w.cleanupPending = false + return nil + } + completed, remaining, err := ioutil.DeleteUnpublishedObjectsWithProgress( + cleanupCtx, w.fs, w.ownedPersistedNames...) + if admission != nil { + for _, name := range completed { + admission.release(name) + } + } + w.ownedPersistedNames = remaining + if err != nil { + // Detached results are the only remaining ownership. Do not let a + // failed Delete keep the sinker, output batch, or session mpool alive. + if w.sinker != nil { + if w.blockInfoBat != nil { + w.blockInfoBat.Clean(w.sinker.GetMPool()) + w.blockInfoBat = nil + } + _ = w.sinker.Close() + w.sinker = nil + } + return err + } + w.cleanupPending = false + return nil +} + +// CloseWithCleanup removes still-owned objects after a failed operation, then +// closes the writer. A failed delete retains only file-service names so its +// caller can retry cleanup during the next lifecycle callback. +func (w *CNS3Writer) CloseWithCleanup(ctx context.Context, failed bool) error { + cleanupRequired := failed || w.cleanupPending + if cleanupRequired { + if err := w.DeletePersisted(ctx); err != nil { + return err + } + // Close may return the already-reported pipeline drain error, but it + // has still released all sinker resources. The unaccepted objects are + // deleted, so that execution error must not keep this owner alive. + _ = w.Close() + return nil + } + return w.Close() +} + func (w *CNS3Writer) Close() (err error) { + if w.cleanupPending { + return moerr.NewInternalErrorNoCtx("cannot close S3 writer with pending object cleanup") + } + // Close on the successful path means the caller has accepted the result + // metadata. Clone readers use this path instead of TransferPersistedObjects. + acceptedNames := append([]string(nil), w.ownedPersistedNames...) + if w.sinker != nil { + stats, _ := w.sinker.GetResult() + for i := range stats { + acceptedNames = append(acceptedNames, stats[i].ObjectName().String()) + } + } + admission, admissionErr := w.unpublishedS3Admission() + if admissionErr != nil { + return admissionErr + } var mp *mpool.MPool if w.sinker != nil { mp = w.sinker.GetMPool() - if err = w.sinker.Close(); err != nil { - return - } + // Sinker.Close always tears down its buffers and references before + // returning a pipeline-drain error. Do not retain a closed sinker and + // make a completed abort cleanup impossible to finish on retry. + err = w.sinker.Close() w.sinker = nil } + w.ownedPersistedNames = nil + if admission != nil { + for _, name := range acceptedNames { + admission.release(name) + } + } if w.blockInfoBat != nil { w.blockInfoBat.Clean(mp) w.blockInfoBat = nil } + return err +} + +// ResetWithCleanup discards this execution's data after failure and resets the +// sinker only after its persisted objects have been deleted. A successful +// execution treats the results as handed off before resetting reusable state. +func (w *CNS3Writer) ResetWithCleanup(ctx context.Context, failed bool) error { + if failed || w.cleanupPending { + if err := w.DeletePersisted(ctx); err != nil { + return err + } + } + w.Reset() return nil } @@ -358,9 +842,27 @@ func (w *CNS3Writer) Close() (err error) { // carried into the next execution. The sinker's buffer pool and arena // are kept alive for efficient reuse. func (w *CNS3Writer) Reset() { + if w.cleanupPending { + return + } + // Reset is used after the caller has accepted these completed results. + // TransferPersistedObjects clears the names only after moving their ticket + // to the transaction owner, so this release is idempotent there. + if admission, err := w.unpublishedS3Admission(); err == nil && admission != nil { + for _, name := range w.ownedPersistedNames { + admission.release(name) + } + if w.sinker != nil { + stats, _ := w.sinker.GetResult() + for i := range stats { + admission.release(stats[i].ObjectName().String()) + } + } + } if w.sinker != nil { w.sinker.Reset() } + w.ownedPersistedNames = nil if w.blockInfoBat != nil { w.blockInfoBat.CleanOnlyData() } diff --git a/pkg/sql/colexec/s3util_threshold_test.go b/pkg/sql/colexec/s3util_threshold_test.go index e318634e94398..4053a641966eb 100644 --- a/pkg/sql/colexec/s3util_threshold_test.go +++ b/pkg/sql/colexec/s3util_threshold_test.go @@ -15,11 +15,16 @@ package colexec import ( + "context" + "errors" "fmt" "math" "testing" + "time" + "github.com/golang/mock/gomock" "github.com/matrixorigin/matrixone/pkg/catalog" + "github.com/matrixorigin/matrixone/pkg/common/moerr" "github.com/matrixorigin/matrixone/pkg/common/mpool" moruntime "github.com/matrixorigin/matrixone/pkg/common/runtime" "github.com/matrixorigin/matrixone/pkg/compress" @@ -28,12 +33,72 @@ import ( "github.com/matrixorigin/matrixone/pkg/container/vector" "github.com/matrixorigin/matrixone/pkg/defines" "github.com/matrixorigin/matrixone/pkg/fileservice" + mock_frontend "github.com/matrixorigin/matrixone/pkg/frontend/test" "github.com/matrixorigin/matrixone/pkg/objectio" + "github.com/matrixorigin/matrixone/pkg/objectio/ioutil" "github.com/matrixorigin/matrixone/pkg/pb/plan" "github.com/matrixorigin/matrixone/pkg/testutil" + "github.com/matrixorigin/matrixone/pkg/txn/client" + "github.com/matrixorigin/matrixone/pkg/vm/process" "github.com/stretchr/testify/require" ) +type testUnpublishedS3CleanupWorkspace struct { + client.Workspace + cleanups []func(context.Context) error +} + +func TestUnpublishedS3CleanupContextSharesOuterDeadline(t *testing.T) { + outerDeadline := time.Now().Add(time.Minute) + outer, cancelOuter := context.WithDeadline(context.Background(), outerDeadline) + cancelOuter() + for range 4 { + inner, cancel := UnpublishedS3CleanupContext(outer) + deadline, ok := inner.Deadline() + require.True(t, ok) + require.Equal(t, outerDeadline, deadline) + require.NoError(t, inner.Err(), "request cancellation must not abort cleanup") + cancel() + } +} + +func TestTerminalUnpublishedS3CleanupContextSurvivesCanceledRequest(t *testing.T) { + request, cancelRequest := context.WithCancel(context.Background()) + cancelRequest() + cleanup, cancelCleanup := TerminalUnpublishedS3CleanupContext(request) + defer cancelCleanup() + require.NoError(t, cleanup.Err()) + deadline, ok := cleanup.Deadline() + require.True(t, ok) + require.Greater(t, time.Until(deadline), 0*time.Second) + require.LessOrEqual(t, time.Until(deadline), 10*time.Minute) +} + +func (w *testUnpublishedS3CleanupWorkspace) RetainUnpublishedS3Cleanup( + cleanup func(context.Context) error, +) { + w.cleanups = append(w.cleanups, cleanup) +} + +func TestRetainUnpublishedS3CleanupTransfersToWorkspace(t *testing.T) { + proc := testutil.NewProc(t) + defer proc.Free() + ctrl := gomock.NewController(t) + txnOp := mock_frontend.NewMockTxnOperator(ctrl) + workspace := &testUnpublishedS3CleanupWorkspace{} + txnOp.EXPECT().GetWorkspace().Return(workspace).AnyTimes() + proc.Base.TxnOperator = txnOp + + called := false + require.True(t, RetainUnpublishedS3Cleanup(proc, func(context.Context) error { + called = true + return nil + })) + require.Len(t, workspace.cleanups, 1) + require.NoError(t, workspace.cleanups[0](proc.Ctx)) + require.True(t, called) +} + func TestCNS3DataWriterMemoryThresholdAndSyncAndFillBlockInfoBat(t *testing.T) { proc := testutil.NewProc(t) defer proc.Free() @@ -80,6 +145,129 @@ func TestCNS3DataWriterMemoryThresholdAndSyncAndFillBlockInfoBat(t *testing.T) { }) } +func TestCNS3WriterRetainsDetachedObjectOwnershipUntilHandoff(t *testing.T) { + proc := testutil.NewProc(t) + defer proc.Free() + + baseFS, err := fileservice.Get[fileservice.FileService]( + proc.Base.FileService, defines.SharedFileServiceName, + ) + require.NoError(t, err) + + t.Run("failed cleanup can be retried", func(t *testing.T) { + fs := &failOnceDeleteFileService{FileService: baseFS} + writer, objectName := writeDetachedTestObject(t, proc, fs) + fs.failNextDelete = true + + err := writer.CloseWithCleanup(proc.Ctx, true) + require.ErrorIs(t, err, errInjectedDelete) + require.Len(t, writer.ownedPersistedNames, 1) + require.Nil(t, writer.sinker, "retry ownership must not retain sinker buffers") + require.Nil(t, writer.blockInfoBat, "retry ownership must not retain an mpool batch") + _, err = baseFS.StatFile(proc.Ctx, objectName) + require.NoError(t, err) + + // A later lifecycle callback may report success; pending abort ownership + // still requires deletion and must not be mistaken for a handoff. + require.NoError(t, writer.CloseWithCleanup(proc.Ctx, false)) + require.Empty(t, writer.ownedPersistedNames) + _, err = baseFS.StatFile(proc.Ctx, objectName) + require.True(t, moerr.IsMoErrCode(err, moerr.ErrFileNotFound), "object should be deleted, got %v", err) + }) + + t.Run("successful handoff preserves object", func(t *testing.T) { + writer, objectName := writeDetachedTestObject(t, proc, baseFS) + require.NoError(t, writer.CloseWithCleanup(proc.Ctx, false)) + _, err := baseFS.StatFile(proc.Ctx, objectName) + require.NoError(t, err) + }) +} + +func TestCNS3WriterCleanupCompletesDespitePipelineDrainError(t *testing.T) { + proc := testutil.NewProc(t) + defer proc.Free() + baseFS, err := fileservice.Get[fileservice.FileService]( + proc.Base.FileService, defines.SharedFileServiceName, + ) + require.NoError(t, err) + writeErr := errors.New("injected pipeline write failure") + fs := &failWriteFileService{FileService: baseFS, writeErr: writeErr} + writer := NewCNS3DataWriter( + proc.Mp(), fs, testCNS3WriterTableDef(), 1, false, + ioutil.WithPipelineFlush(), + ) + bat := &batch.Batch{ + Attrs: []string{"a", "b"}, + Vecs: []*vector.Vector{ + testutil.MakeInt64Vector([]int64{1}, nil, proc.Mp()), + testutil.MakeVarcharVector([]string{"x"}, nil, proc.Mp()), + }, + } + bat.SetRowCount(1) + defer bat.Clean(proc.Mp()) + require.NoError(t, writer.Write(proc.Ctx, bat)) + _, err = writer.Sync(proc.Ctx) + require.ErrorIs(t, err, writeErr) + + // The failed pipeline is already reported by Sync. Cleanup must still + // release a fully drained sinker rather than retaining a closed owner. + require.NoError(t, writer.CloseWithCleanup(proc.Ctx, true)) + require.Nil(t, writer.sinker) + require.False(t, writer.cleanupPending) +} + +var errInjectedDelete = errors.New("injected delete failure") + +type failOnceDeleteFileService struct { + fileservice.FileService + failNextDelete bool +} + +type failWriteFileService struct { + fileservice.FileService + writeErr error +} + +func (fs *failWriteFileService) Write(context.Context, fileservice.IOVector) error { + return fs.writeErr +} + +func (fs *failOnceDeleteFileService) Delete(ctx context.Context, paths ...string) error { + if fs.failNextDelete { + fs.failNextDelete = false + return errInjectedDelete + } + return fs.FileService.Delete(ctx, paths...) +} + +func writeDetachedTestObject( + t *testing.T, + proc *process.Process, + fs fileservice.FileService, +) (*CNS3Writer, string) { + t.Helper() + writer := NewCNS3DataWriter(proc.GetMPool(), fs, testCNS3WriterTableDef(), 1, false) + t.Cleanup(func() { + if writer.sinker != nil { + _ = writer.CloseWithCleanup(proc.Ctx, true) + } + }) + bat := &batch.Batch{ + Attrs: []string{"a", "b"}, + Vecs: []*vector.Vector{ + testutil.MakeInt64Vector([]int64{1}, nil, proc.GetMPool()), + testutil.MakeVarcharVector([]string{"x"}, nil, proc.GetMPool()), + }, + } + bat.SetRowCount(1) + defer bat.Clean(proc.GetMPool()) + require.NoError(t, writer.Write(proc.Ctx, bat)) + _, err := writer.SyncAndFillBlockInfoBat(proc.Ctx) + require.NoError(t, err) + require.Len(t, writer.ownedPersistedNames, 1) + return writer, writer.ownedPersistedNames[0] +} + func TestCNS3DataWriterChunkedColumnProtocolGateIsLive(t *testing.T) { previousObjectSizeLimit := objectio.ObjectSizeLimit objectio.SetObjectSizeLimit(3 * mpool.GB) @@ -103,6 +291,7 @@ func TestCNS3DataWriterChunkedColumnProtocolGateIsLive(t *testing.T) { } }) moruntime.SetupServiceBasedRuntime(serviceID, rt) + NewServer(serviceID) tableDef := &plan.TableDef{ Name: "wide_column", @@ -189,6 +378,7 @@ func TestCNS3TombstoneWriterProtocolGate(t *testing.T) { serviceID := fmt.Sprintf("chunked-tombstone-%s", t.Name()) rt := moruntime.DefaultRuntime() moruntime.SetupServiceBasedRuntime(serviceID, rt) + NewServer(serviceID) rt.SetGlobalVariables(moruntime.MOProtocolVersion, defines.MORPCVersion28) legacy := NewCNS3TombstoneWriterForService( diff --git a/pkg/sql/colexec/server.go b/pkg/sql/colexec/server.go index 287ddc80bb53f..485165490ebe2 100644 --- a/pkg/sql/colexec/server.go +++ b/pkg/sql/colexec/server.go @@ -22,6 +22,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/common/morpc" "github.com/matrixorigin/matrixone/pkg/common/runtime" "github.com/matrixorigin/matrixone/pkg/objectio" + metricv2 "github.com/matrixorigin/matrixone/pkg/util/metric/v2" "github.com/matrixorigin/matrixone/pkg/vm/process" ) @@ -29,6 +30,8 @@ import ( // Different CN services in the same process must always use different servers. func NewServer(serviceID string) *Server { s := &Server{ + unpublishedS3Admission: newUnpublishedS3Admission(defaultUnpublishedS3TicketLimit, serviceID), + unpublishedS3Cleanup: unpublishedS3CleanupQueue{serviceID: serviceID}, uuidCsChanMap: UuidProcMap{ mp: make(map[uuid.UUID]uuidProcMapItem, 1024), waiters: make(map[uuid.UUID]*remoteReceiverWaitState, 1024), @@ -39,6 +42,10 @@ func NewServer(serviceID string) *Server { sessionCleanupWaiters: make(map[morpc.ClientSession]struct{}, 128), }, } + if serviceID != "" { + metricv2.UnpublishedS3PendingTasksGauge.WithLabelValues(serviceID).Set(0) + metricv2.UnpublishedS3OldestTaskAgeGauge.WithLabelValues(serviceID).Set(0) + } rt := runtime.ServiceRuntime(serviceID) if rt == nil { panic("service runtime is not initialized for colexec server") diff --git a/pkg/sql/colexec/table_clone/table_clone.go b/pkg/sql/colexec/table_clone/table_clone.go index 59ee9dca73f67..9e0045187fd39 100644 --- a/pkg/sql/colexec/table_clone/table_clone.go +++ b/pkg/sql/colexec/table_clone/table_clone.go @@ -55,6 +55,32 @@ func init() { ) } +func closeReaders( + proc *process.Process, + readers map[string]engine.Reader, + retainFailedCleanup bool, +) map[string]engine.Reader { + var pending map[string]engine.Reader + for name, reader := range readers { + if err := reader.Close(); err != nil { + logutil.Warn("failed to close table clone reader", zap.String("reader", name), zap.Error(err)) + if retainFailedCleanup && colexec.RetainUnpublishedS3Cleanup(proc, func(ctx context.Context) error { + if retryable, ok := reader.(interface{ CloseWithCleanup(context.Context) error }); ok { + return retryable.CloseWithCleanup(ctx) + } + return reader.Close() + }) { + continue + } + if pending == nil { + pending = make(map[string]engine.Reader) + } + pending[name] = reader + } + } + return pending +} + func (tc *TableClone) Free(proc *process.Process, pipelineFailed bool, err error) { if tc.dataObjBat != nil { tc.dataObjBat.Clean(proc.Mp()) @@ -64,17 +90,10 @@ func (tc *TableClone) Free(proc *process.Process, pipelineFailed bool, err error tc.tombstoneObjBat.Clean(proc.Mp()) } - for _, r := range tc.srcReader { - r.Close() - } - - for _, r := range tc.srcIdxReader { - r.Close() - } + tc.srcReader = closeReaders(proc, tc.srcReader, true) + tc.srcIdxReader = closeReaders(proc, tc.srcIdxReader, true) tc.srcRel = nil - tc.srcReader = nil - tc.srcIdxReader = nil tc.dstRel = nil tc.dstIdxRel = nil } @@ -88,17 +107,10 @@ func (tc *TableClone) Reset(proc *process.Process, pipelineFailed bool, err erro tc.tombstoneObjBat.Clean(proc.Mp()) } - for _, r := range tc.srcReader { - r.Close() - } - - for _, r := range tc.srcIdxReader { - r.Close() - } + tc.srcReader = closeReaders(proc, tc.srcReader, true) + tc.srcIdxReader = closeReaders(proc, tc.srcIdxReader, true) tc.srcRel = nil - tc.srcReader = nil - tc.srcIdxReader = nil tc.dstRel = nil tc.dstIdxRel = nil @@ -223,6 +235,12 @@ func initRelAndReader( } func (tc *TableClone) Prepare(proc *process.Process) error { + tc.srcReader = closeReaders(proc, tc.srcReader, false) + tc.srcIdxReader = closeReaders(proc, tc.srcIdxReader, false) + if len(tc.srcReader) != 0 || len(tc.srcIdxReader) != 0 { + return moerr.NewInternalError(proc.Ctx, "unpublished table clone objects still require cleanup") + } + if tc.OpAnalyzer == nil { tc.OpAnalyzer = process.NewAnalyzer( tc.GetIdx(), @@ -354,7 +372,8 @@ func clone( ) error { var ( - err error + err error + innerReader = reader.(*disttae.TableMetaReader) ) checkObjStatsFmt := func(bat *batch.Batch, isTombstone bool) error { @@ -382,7 +401,6 @@ func clone( } dstDef := dstRel.GetTableDef(dstCtx) - innerReader := reader.(*disttae.TableMetaReader) srcDef := innerReader.GetTableDef() logutil.Info("TABLE-CLONE", @@ -412,6 +430,9 @@ func clone( return err } } + if err = innerReader.AcceptDataObjects(); err != nil { + return err + } } // copy tombstone @@ -429,6 +450,9 @@ func clone( return err } } + if err = innerReader.AcceptTombstoneObjects(); err != nil { + return err + } } return nil @@ -590,6 +614,9 @@ func updateRelationAutoIncrement( func (tc *TableClone) Release() { if tc != nil { + if len(tc.srcReader) != 0 || len(tc.srcIdxReader) != 0 { + return + } reuse.Free[TableClone](tc, nil) } } diff --git a/pkg/sql/colexec/table_clone/table_clone_test.go b/pkg/sql/colexec/table_clone/table_clone_test.go index a403a6b9ace4f..adbb75658fe61 100644 --- a/pkg/sql/colexec/table_clone/table_clone_test.go +++ b/pkg/sql/colexec/table_clone/table_clone_test.go @@ -17,6 +17,7 @@ package table_clone import ( "bytes" "context" + "errors" "math" "testing" @@ -48,6 +49,114 @@ func TestTableCloneOperatorMetadata(t *testing.T) { require.Equal(t, "TableClone", tc.OpType().String()) } +type tableCloneS3CleanupWorkspace struct { + client.Workspace + cleanups []func(context.Context) error +} + +func (w *tableCloneS3CleanupWorkspace) RetainUnpublishedS3Cleanup( + cleanup func(context.Context) error, +) { + w.cleanups = append(w.cleanups, cleanup) +} + +type tableCloneCloseErrorReader struct { + engine.Reader + calls int + err error +} + +type tableCloneContextCleanupReader struct { + engine.Reader + closeErr error + seenCtx context.Context +} + +func (r *tableCloneContextCleanupReader) Close() error { return r.closeErr } + +func (r *tableCloneContextCleanupReader) CloseWithCleanup(ctx context.Context) error { + r.seenCtx = ctx + return ctx.Err() +} + +func TestTableCloneRetryForwardsCleanupAttemptContext(t *testing.T) { + proc := testutil.NewProcess(t) + defer proc.Free() + ctrl := gomock.NewController(t) + txnOp := mock_frontend.NewMockTxnOperator(ctrl) + workspace := &tableCloneS3CleanupWorkspace{} + txnOp.EXPECT().GetWorkspace().Return(workspace).AnyTimes() + proc.Base.TxnOperator = txnOp + + reader := &tableCloneContextCleanupReader{closeErr: errors.New("initial close failed")} + clone := NewTableClone() + clone.srcReader = map[string]engine.Reader{"source": reader} + clone.Free(proc, true, reader.closeErr) + require.Len(t, workspace.cleanups, 1) + clone.Release() + + attemptCtx := context.WithValue(context.Background(), struct{}{}, "retry") + require.NoError(t, workspace.cleanups[0](attemptCtx)) + require.Same(t, attemptCtx, reader.seenCtx) +} + +func (r *tableCloneCloseErrorReader) Close() error { + r.calls++ + if r.calls == 1 { + return r.err + } + return nil +} + +func TestTableCloneFreeTransfersReaderCleanupToTransaction(t *testing.T) { + t.Run("Free", func(t *testing.T) { checkTableCloneCleanup(t, false) }) + t.Run("prepared Reset", func(t *testing.T) { checkTableCloneCleanup(t, true) }) +} + +func checkTableCloneCleanup(t *testing.T, prepared bool) { + proc := testutil.NewProcess(t) + defer proc.Free() + ctrl := gomock.NewController(t) + txnOp := mock_frontend.NewMockTxnOperator(ctrl) + workspace := &tableCloneS3CleanupWorkspace{} + txnOp.EXPECT().GetWorkspace().Return(workspace).AnyTimes() + proc.Base.TxnOperator = txnOp + + closeErr := errors.New("injected clone reader cleanup failure") + reader := &tableCloneCloseErrorReader{err: closeErr} + clone := NewTableClone() + clone.srcReader = map[string]engine.Reader{"source": reader} + if prepared { + clone.Ctx = &TableCloneCtx{} + clone.Reset(proc, true, closeErr) + } else { + clone.Free(proc, true, closeErr) + } + require.Empty(t, clone.srcReader, "cleanup ownership should move before scope release") + require.Len(t, workspace.cleanups, 1) + clone.Release() + + require.NoError(t, workspace.cleanups[0](proc.Ctx)) + require.Equal(t, 2, reader.calls) +} + +func TestTableCloneReaderCloseRetriesWithoutWorkspace(t *testing.T) { + proc := testutil.NewProcess(t) + defer proc.Free() + closeErr := errors.New("injected clone reader close failure") + reader := &tableCloneCloseErrorReader{err: closeErr} + clone := NewTableClone() + clone.Ctx = &TableCloneCtx{} + clone.srcIdxReader = map[string]engine.Reader{"index": reader} + + clone.Reset(proc, true, closeErr) + require.Same(t, reader, clone.srcIdxReader["index"], "Reset must keep failed reader cleanup") + clone.Free(proc, true, closeErr) + require.Empty(t, clone.srcIdxReader) + require.Equal(t, 2, reader.calls) + clone.Release() +} + type autoIncrementTestRelation struct { engine.Relation tableID uint64 diff --git a/pkg/sql/colexec/types.go b/pkg/sql/colexec/types.go index 7b8bc70ec1010..232c7f90e864c 100644 --- a/pkg/sql/colexec/types.go +++ b/pkg/sql/colexec/types.go @@ -53,6 +53,8 @@ type Server struct { cnSegmentMap CnSegmentMap receivedRunningPipeline RunningPipelineMapForRemoteNode + unpublishedS3Cleanup unpublishedS3CleanupQueue + unpublishedS3Admission *unpublishedS3Admission } // RunningPipelineMapForRemoteNode diff --git a/pkg/sql/colexec/unpublished_cleanup_retry.go b/pkg/sql/colexec/unpublished_cleanup_retry.go new file mode 100644 index 0000000000000..8b308d5931c69 --- /dev/null +++ b/pkg/sql/colexec/unpublished_cleanup_retry.go @@ -0,0 +1,232 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package colexec + +import ( + "context" + "sync" + "time" + + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/logutil" + metricv2 "github.com/matrixorigin/matrixone/pkg/util/metric/v2" + "go.uber.org/zap" +) + +const unpublishedS3RetryInterval = time.Second + +// Failed remote-stream cleanup outlives the mirror transaction's handler. One +// worker per CN retries the exact workspace cleanup callback until it succeeds +// or the CN is closed. The queue is in-memory; process crashes need durable GC. +type unpublishedS3CleanupQueue struct { + closeMu sync.Mutex + mu sync.Mutex + pending []func(context.Context) error + pendingSince []time.Time + lastAgeSample time.Time + serviceID string + stop chan struct{} + done chan struct{} + closing bool +} + +func (srv *Server) RetryUnpublishedS3Cleanup(cleanup func(context.Context) error) error { + if srv == nil || cleanup == nil { + return moerr.NewInvalidStateNoCtx("missing CN owner for unpublished S3 cleanup") + } + q := &srv.unpublishedS3Cleanup + q.mu.Lock() + defer q.mu.Unlock() + if q.closing { + return moerr.NewInvalidStateNoCtx("CN is closing with unpublished S3 cleanup pending") + } + q.pending = append(q.pending, cleanup) + q.pendingSince = append(q.pendingSince, time.Now()) + q.updateMetricsLocked() + q.startLocked() + return nil +} + +func (q *unpublishedS3CleanupQueue) updateMetricsLocked() { + if q.serviceID == "" { + return + } + metricv2.UnpublishedS3PendingTasksGauge.WithLabelValues(q.serviceID).Set(float64(len(q.pending))) + if len(q.pendingSince) == 0 { + q.lastAgeSample = time.Time{} + metricv2.UnpublishedS3OldestTaskAgeGauge.WithLabelValues(q.serviceID).Set(0) + return + } + now := time.Now() + if !q.lastAgeSample.IsZero() && now.Sub(q.lastAgeSample) < unpublishedS3RetryInterval { + return + } + oldest := q.pendingSince[0] + for _, since := range q.pendingSince[1:] { + if since.Before(oldest) { + oldest = since + } + } + q.lastAgeSample = now + metricv2.UnpublishedS3OldestTaskAgeGauge.WithLabelValues(q.serviceID).Set(now.Sub(oldest).Seconds()) +} + +func (q *unpublishedS3CleanupQueue) startLocked() { + if q.stop == nil { + q.stop = make(chan struct{}) + q.done = make(chan struct{}) + go q.run(q.stop, q.done) + } +} + +// Clear the consumed slot before advancing the slice: the backing array must +// not retain completed callbacks and their captured cleanup resources. +func (q *unpublishedS3CleanupQueue) finishAttemptLocked(err error) { + if err != nil && len(q.pending) == 1 { + q.updateMetricsLocked() + return + } + var since time.Time + if len(q.pendingSince) != 0 { + since = q.pendingSince[0] + q.pendingSince[0] = time.Time{} + q.pendingSince = q.pendingSince[1:] + } + cleanup := q.pending[0] + q.pending[0] = nil + q.pending = q.pending[1:] + if err != nil { + q.pending = append(q.pending, cleanup) + if !since.IsZero() { + q.pendingSince = append(q.pendingSince, since) + } + } else if len(q.pending) == 0 { + q.pending = nil + q.pendingSince = nil + } + q.updateMetricsLocked() +} + +func (q *unpublishedS3CleanupQueue) run(stop <-chan struct{}, done chan<- struct{}) { + defer close(done) + retryAfterFailure := false + for { + if retryAfterFailure { + timer := time.NewTimer(unpublishedS3RetryInterval) + select { + case <-stop: + timer.Stop() + return + case <-timer.C: + } + } + select { + case <-stop: + return + default: + } + q.mu.Lock() + q.updateMetricsLocked() + if len(q.pending) == 0 { + q.stop = nil + q.done = nil + q.mu.Unlock() + return + } + cleanup := q.pending[0] + q.mu.Unlock() + + attemptCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + err := cleanup(attemptCtx) + cancel() + q.mu.Lock() + q.finishAttemptLocked(err) + q.mu.Unlock() + if err != nil { + logutil.Warn("remote unpublished S3 cleanup will be retried", zap.Error(err)) + } + retryAfterFailure = err != nil + } +} + +// CloseUnpublishedS3Cleanup joins the retry worker before CN file-service +// dependencies are closed. Failed tasks rotate within one bounded deadline. +// On timeout the CN keeps its dependencies alive (fail-stop), so resume the +// worker to keep those tasks retryable if storage subsequently recovers. +func (srv *Server) CloseUnpublishedS3Cleanup(ctx context.Context) error { + if srv == nil { + return nil + } + q := &srv.unpublishedS3Cleanup + q.closeMu.Lock() + defer q.closeMu.Unlock() + cleanupCtx, cancel := UnpublishedS3CleanupContext(ctx) + defer cancel() + q.mu.Lock() + q.closing = true + stop, done := q.stop, q.done + if stop != nil { + close(stop) + q.stop = nil + q.done = nil + } + q.mu.Unlock() + if done != nil { + <-done + } + + var lastErr error + for { + q.mu.Lock() + remaining := len(q.pending) + if remaining == 0 { + q.mu.Unlock() + return nil + } + if cleanupCtx.Err() != nil { + q.startLocked() + q.mu.Unlock() + return moerr.NewInternalErrorNoCtxf("%d unpublished S3 cleanup tasks remain at CN shutdown: %v (deadline: %v)", remaining, lastErr, cleanupCtx.Err()) + } + q.mu.Unlock() + for range remaining { + if cleanupCtx.Err() != nil { + break + } + q.mu.Lock() + cleanup := q.pending[0] + q.mu.Unlock() + attemptCtx, attemptCancel := context.WithTimeout(cleanupCtx, 30*time.Second) + err := cleanup(attemptCtx) + attemptCancel() + q.mu.Lock() + q.finishAttemptLocked(err) + remainingTasks := len(q.pending) + q.mu.Unlock() + if err != nil { + lastErr = err + } + if remainingTasks == 0 { + return nil + } + } + timer := time.NewTimer(unpublishedS3RetryInterval) + select { + case <-cleanupCtx.Done(): + case <-timer.C: + } + timer.Stop() + } +} diff --git a/pkg/sql/colexec/unpublished_cleanup_retry_test.go b/pkg/sql/colexec/unpublished_cleanup_retry_test.go new file mode 100644 index 0000000000000..ac3626de63956 --- /dev/null +++ b/pkg/sql/colexec/unpublished_cleanup_retry_test.go @@ -0,0 +1,294 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package colexec + +import ( + "context" + "errors" + "sync" + "sync/atomic" + "testing" + "time" + + moruntime "github.com/matrixorigin/matrixone/pkg/common/runtime" + metricv2 "github.com/matrixorigin/matrixone/pkg/util/metric/v2" + promtestutil "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/stretchr/testify/require" +) + +func TestUnpublishedS3RetryQueueMetrics(t *testing.T) { + serviceID := t.Name() + moruntime.SetupServiceBasedRuntime(serviceID, moruntime.DefaultRuntime()) + server := NewServer(serviceID) + entered := make(chan struct{}) + release := make(chan struct{}) + var releaseOnce sync.Once + t.Cleanup(func() { + releaseOnce.Do(func() { close(release) }) + require.NoError(t, server.CloseUnpublishedS3Cleanup(context.Background())) + }) + require.NoError(t, server.RetryUnpublishedS3Cleanup(func(context.Context) error { + close(entered) + <-release + return nil + })) + <-entered + require.Equal(t, float64(1), promtestutil.ToFloat64( + metricv2.UnpublishedS3PendingTasksGauge.WithLabelValues(serviceID))) + releaseOnce.Do(func() { close(release) }) + require.NoError(t, server.CloseUnpublishedS3Cleanup(context.Background())) + require.Zero(t, promtestutil.ToFloat64( + metricv2.UnpublishedS3PendingTasksGauge.WithLabelValues(serviceID))) +} + +func TestUnpublishedS3OldestRetryAgeSurvivesRotation(t *testing.T) { + serviceID := t.Name() + q := &unpublishedS3CleanupQueue{ + serviceID: serviceID, + pending: []func(context.Context) error{ + func(context.Context) error { return nil }, + func(context.Context) error { return nil }, + }, + pendingSince: []time.Time{ + time.Now().Add(-2 * time.Minute), + time.Now().Add(-time.Minute), + }, + } + q.mu.Lock() + q.updateMetricsLocked() + q.finishAttemptLocked(errors.New("temporary failure")) + q.lastAgeSample = time.Time{} + q.updateMetricsLocked() + q.mu.Unlock() + + age := promtestutil.ToFloat64(metricv2.UnpublishedS3OldestTaskAgeGauge.WithLabelValues(serviceID)) + require.GreaterOrEqual(t, age, float64(120), "failure rotation must keep the original enqueue age") + require.Less(t, age, float64(130)) + + q.mu.Lock() + q.finishAttemptLocked(nil) + q.finishAttemptLocked(nil) + q.mu.Unlock() + require.Zero(t, promtestutil.ToFloat64( + metricv2.UnpublishedS3OldestTaskAgeGauge.WithLabelValues(serviceID))) +} + +func TestServerCloseDrainsUnpublishedS3Retry(t *testing.T) { + server := NewServer("") + var calls atomic.Int32 + require.NoError(t, server.RetryUnpublishedS3Cleanup(func(context.Context) error { + calls.Add(1) + return nil + })) + require.NoError(t, server.CloseUnpublishedS3Cleanup(context.Background())) + require.Equal(t, int32(1), calls.Load()) + require.Error(t, server.RetryUnpublishedS3Cleanup(func(context.Context) error { return nil })) +} + +func TestServerCloseReportsUnresolvedS3Cleanup(t *testing.T) { + server := NewServer("") + cleanupErr := errors.New("S3 unavailable") + var recovered atomic.Bool + finished := make(chan struct{}) + t.Cleanup(func() { + recovered.Store(true) + require.NoError(t, server.CloseUnpublishedS3Cleanup(context.Background())) + }) + require.NoError(t, server.RetryUnpublishedS3Cleanup(func(context.Context) error { + if recovered.Load() { + close(finished) + return nil + } + return cleanupErr + })) + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + err := server.CloseUnpublishedS3Cleanup(ctx) + require.ErrorContains(t, err, "1 unpublished S3 cleanup tasks remain") + server.unpublishedS3Cleanup.mu.Lock() + require.Len(t, server.unpublishedS3Cleanup.pending, 1, "shutdown must not report success or discard the failed owner") + server.unpublishedS3Cleanup.mu.Unlock() + recovered.Store(true) + select { + case <-finished: + case <-time.After(5 * time.Second): + t.Fatal("cleanup lost its retry entry after the shutdown deadline") + } + require.NoError(t, server.CloseUnpublishedS3Cleanup(context.Background())) +} + +func TestServerCloseRetriesFailureWithoutBlockingOtherTasks(t *testing.T) { + server := &Server{} + server.unpublishedS3Cleanup.pending = make([]func(context.Context) error, 0, 8) + firstEntered := make(chan struct{}) + allowFirstFailure := make(chan struct{}) + var releaseOnce sync.Once + t.Cleanup(func() { + releaseOnce.Do(func() { close(allowFirstFailure) }) + require.NoError(t, server.CloseUnpublishedS3Cleanup(context.Background())) + }) + var order []int + require.NoError(t, server.RetryUnpublishedS3Cleanup(func(context.Context) error { + order = append(order, 1) + if len(order) == 1 { + close(firstEntered) + <-allowFirstFailure + return errors.New("temporary failure") + } + return nil + })) + <-firstEntered + require.NoError(t, server.RetryUnpublishedS3Cleanup(func(context.Context) error { + order = append(order, 2) + return nil + })) + server.unpublishedS3Cleanup.mu.Lock() + backing := server.unpublishedS3Cleanup.pending[:8] + server.unpublishedS3Cleanup.mu.Unlock() + releaseOnce.Do(func() { close(allowFirstFailure) }) + require.NoError(t, server.CloseUnpublishedS3Cleanup(context.Background())) + require.Equal(t, []int{1, 2, 1}, order) + for _, callback := range backing { + require.Nil(t, callback, "completed cleanup must not remain in the backing array") + } + require.Nil(t, server.unpublishedS3Cleanup.pending) +} + +func TestServerRetriesAndRotatesFailedS3Cleanup(t *testing.T) { + server := &Server{} + // Keep one backing array across rotation so the retention assertion does + // not pin an abandoned array that the production queue already released. + server.unpublishedS3Cleanup.pending = make([]func(context.Context) error, 0, 8) + t.Cleanup(func() { require.NoError(t, server.CloseUnpublishedS3Cleanup(context.Background())) }) + var firstCalls atomic.Int32 + firstDone := make(chan struct{}) + secondDone := make(chan struct{}) + require.NoError(t, server.RetryUnpublishedS3Cleanup(func(context.Context) error { + if firstCalls.Add(1) == 1 { + return errors.New("temporary storage failure") + } + close(firstDone) + return nil + })) + require.NoError(t, server.RetryUnpublishedS3Cleanup(func(context.Context) error { + close(secondDone) + return nil + })) + server.unpublishedS3Cleanup.mu.Lock() + backing := server.unpublishedS3Cleanup.pending[:cap(server.unpublishedS3Cleanup.pending)] + server.unpublishedS3Cleanup.mu.Unlock() + select { + case <-secondDone: + case <-time.After(5 * time.Second): + t.Fatal("healthy cleanup was blocked behind a failed task") + } + select { + case <-firstDone: + case <-time.After(5 * time.Second): + t.Fatal("failed cleanup was not retried") + } + require.Equal(t, int32(2), firstCalls.Load()) + require.NoError(t, server.CloseUnpublishedS3Cleanup(context.Background())) + for _, callback := range backing { + require.Nil(t, callback) + } +} + +func TestUnpublishedS3RetryConcurrentEnqueueAndClose(t *testing.T) { + server := NewServer("") + entered := make(chan struct{}) + release := make(chan struct{}) + var executed atomic.Int32 + require.NoError(t, server.RetryUnpublishedS3Cleanup(func(context.Context) error { + close(entered) + <-release + executed.Add(1) + return nil + })) + <-entered + + const competitors = 32 + var accepted atomic.Int32 + var wg sync.WaitGroup + for range competitors { + wg.Add(1) + go func() { + defer wg.Done() + if err := server.RetryUnpublishedS3Cleanup(func(context.Context) error { + executed.Add(1) + return nil + }); err == nil { + accepted.Add(1) + } + }() + } + closeResult := make(chan error, 1) + go func() { + closeResult <- server.CloseUnpublishedS3Cleanup(context.Background()) + }() + wg.Wait() + close(release) + select { + case err := <-closeResult: + require.NoError(t, err) + case <-time.After(5 * time.Second): + t.Fatal("CN close did not join accepted cleanup tasks") + } + require.Equal(t, 1+accepted.Load(), executed.Load()) + require.Error(t, server.RetryUnpublishedS3Cleanup(func(context.Context) error { return nil })) +} + +// Measures the normal worker after a previously blocked cleanup succeeds. +// Run with -benchtime=1x so the number of queued tasks stays explicit. +func BenchmarkUnpublishedS3RetrySuccessfulDrain(b *testing.B) { + const taskCount = 8 + for range b.N { + b.StopTimer() + server := NewServer("") + started := make(chan struct{}) + release := make(chan struct{}) + done := make(chan struct{}) + var completed atomic.Int32 + finish := func() { + if completed.Add(1) == taskCount { + close(done) + } + } + if err := server.RetryUnpublishedS3Cleanup(func(context.Context) error { + close(started) + <-release + finish() + return nil + }); err != nil { + b.Fatal(err) + } + <-started + for range taskCount - 1 { + if err := server.RetryUnpublishedS3Cleanup(func(context.Context) error { + finish() + return nil + }); err != nil { + b.Fatal(err) + } + } + b.StartTimer() + close(release) + <-done + b.StopTimer() + if err := server.CloseUnpublishedS3Cleanup(context.Background()); err != nil { + b.Fatal(err) + } + } +} diff --git a/pkg/sql/colexec/unpublished_s3_admission.go b/pkg/sql/colexec/unpublished_s3_admission.go new file mode 100644 index 0000000000000..747647c9d1ee4 --- /dev/null +++ b/pkg/sql/colexec/unpublished_s3_admission.go @@ -0,0 +1,193 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package colexec + +import ( + "context" + "sync" + + "github.com/matrixorigin/matrixone/pkg/common/moerr" + metricv2 "github.com/matrixorigin/matrixone/pkg/util/metric/v2" + "github.com/prometheus/client_golang/prometheus" +) + +const ( + defaultUnpublishedS3TicketLimit = 65_536 + maxUnpublishedS3ObjectNameLen = 256 +) + +// unpublishedS3Admission charges each live CN for object names it may still +// need to delete. Admission precedes the object Sync, so a full ledger cannot +// force an already-persisted object's cleanup owner to be discarded. +type unpublishedS3Admission struct { + mu sync.Mutex + limit int + names map[string]struct{} + highWater int + failedReservations uint64 + usedGauge prometheus.Gauge + peakGauge prometheus.Gauge + failedCounter prometheus.Counter +} + +// UnpublishedS3AdmissionStats is a bounded, name-free snapshot for CN +// diagnostics and tests. Names themselves must never be metric labels. +type UnpublishedS3AdmissionStats struct { + Used int + Limit int + HighWater int + FailedReservations uint64 +} + +func (srv *Server) UnpublishedS3AdmissionStats() UnpublishedS3AdmissionStats { + if srv == nil || srv.unpublishedS3Admission == nil { + return UnpublishedS3AdmissionStats{} + } + a := srv.unpublishedS3Admission + a.mu.Lock() + defer a.mu.Unlock() + return UnpublishedS3AdmissionStats{ + Used: len(a.names), + Limit: a.limit, + HighWater: a.highWater, + FailedReservations: a.failedReservations, + } +} + +func newUnpublishedS3Admission(limit int, serviceID ...string) *unpublishedS3Admission { + a := &unpublishedS3Admission{limit: limit, names: make(map[string]struct{})} + if len(serviceID) != 0 && serviceID[0] != "" { + a.usedGauge = metricv2.UnpublishedS3TicketsGauge.WithLabelValues(serviceID[0], "used") + a.peakGauge = metricv2.UnpublishedS3TicketsGauge.WithLabelValues(serviceID[0], "high_water") + a.failedCounter = metricv2.UnpublishedS3AdmissionFailuresCounter.WithLabelValues(serviceID[0]) + a.usedGauge.Set(0) + a.peakGauge.Set(0) + } + return a +} + +func (a *unpublishedS3Admission) reserveUpload(name string) error { + if err := validateUnpublishedS3ObjectName(name); err != nil { + return err + } + a.mu.Lock() + defer a.mu.Unlock() + if _, exists := a.names[name]; exists { + return moerr.NewInvalidStateNoCtx("unpublished S3 object name already reserved") + } + if len(a.names) >= a.limit { + a.failedReservations++ + if a.failedCounter != nil { + a.failedCounter.Inc() + } + return moerr.NewResourceExhaustedf(context.Background(), + "CN unpublished S3 cleanup capacity exhausted: used=%d limit=%d", len(a.names), a.limit) + } + a.names[name] = struct{}{} + if a.usedGauge != nil { + a.usedGauge.Set(float64(len(a.names))) + } + if len(a.names) > a.highWater { + a.highWater = len(a.names) + if a.peakGauge != nil { + a.peakGauge.Set(float64(a.highWater)) + } + } + return nil +} + +// reserveReceived reserves only names not already charged on this CN. A batch +// failure rolls back its new reservations; earlier owners keep their tickets. +func (a *unpublishedS3Admission) reserveReceived(names []string) ([]string, error) { + for _, name := range names { + if err := validateUnpublishedS3ObjectName(name); err != nil { + return nil, err + } + } + a.mu.Lock() + defer a.mu.Unlock() + newNames := make([]string, 0, len(names)) + seen := make(map[string]struct{}, len(names)) + for _, name := range names { + if _, repeated := seen[name]; repeated { + continue + } + seen[name] = struct{}{} + if _, exists := a.names[name]; !exists { + newNames = append(newNames, name) + } + } + if len(newNames) > a.limit-len(a.names) { + a.failedReservations++ + if a.failedCounter != nil { + a.failedCounter.Inc() + } + return nil, moerr.NewResourceExhaustedf(context.Background(), + "CN unpublished S3 cleanup capacity exhausted: used=%d requested=%d limit=%d", + len(a.names), len(newNames), a.limit) + } + for _, name := range newNames { + a.names[name] = struct{}{} + } + if a.usedGauge != nil { + a.usedGauge.Set(float64(len(a.names))) + } + if len(a.names) > a.highWater { + a.highWater = len(a.names) + if a.peakGauge != nil { + a.peakGauge.Set(float64(a.highWater)) + } + } + return newNames, nil +} + +func (a *unpublishedS3Admission) release(name string) { + if a == nil || name == "" { + return + } + a.mu.Lock() + delete(a.names, name) + if a.usedGauge != nil { + a.usedGauge.Set(float64(len(a.names))) + } + a.mu.Unlock() +} + +func (a *unpublishedS3Admission) count() int { + a.mu.Lock() + defer a.mu.Unlock() + return len(a.names) +} + +func validateUnpublishedS3ObjectName(name string) error { + if name == "" || len(name) > maxUnpublishedS3ObjectNameLen { + return moerr.NewInvalidArgNoCtx("unpublished S3 object name", name) + } + return nil +} + +func (srv *Server) reserveUnpublishedS3Upload(name string) error { + if srv == nil { + return moerr.NewInvalidStateNoCtx("missing CN unpublished S3 admission service") + } + return srv.unpublishedS3Admission.reserveUpload(name) +} + +func (srv *Server) reserveUnpublishedS3Received(names []string) ([]string, error) { + if srv == nil { + return nil, moerr.NewInvalidStateNoCtx("missing CN unpublished S3 admission service") + } + return srv.unpublishedS3Admission.reserveReceived(names) +} diff --git a/pkg/sql/colexec/unpublished_s3_admission_test.go b/pkg/sql/colexec/unpublished_s3_admission_test.go new file mode 100644 index 0000000000000..dd6b6e1287ccf --- /dev/null +++ b/pkg/sql/colexec/unpublished_s3_admission_test.go @@ -0,0 +1,304 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package colexec + +import ( + "context" + "errors" + "fmt" + "runtime" + "sync" + "testing" + + "github.com/matrixorigin/matrixone/pkg/common/moerr" + moruntime "github.com/matrixorigin/matrixone/pkg/common/runtime" + "github.com/matrixorigin/matrixone/pkg/fileservice" + metricv2 "github.com/matrixorigin/matrixone/pkg/util/metric/v2" + promtestutil "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/stretchr/testify/require" +) + +func TestUnpublishedS3AdmissionBoundsUploadsAndRemoteReceipt(t *testing.T) { + a := newUnpublishedS3Admission(2) + require.NoError(t, a.reserveUpload("local")) + require.Error(t, a.reserveUpload("local"), "duplicate object names must not borrow a ticket") + require.NoError(t, a.reserveUpload("other")) + require.Error(t, a.reserveUpload("third"), "a full CN must stop before Sync") + require.Equal(t, 2, a.count()) + require.Equal(t, 2, a.highWater) + require.Equal(t, uint64(1), a.failedReservations) + + charged, err := a.reserveReceived([]string{"local", "remote"}) + require.Error(t, err, "remote receipt must fail atomically when full") + require.Empty(t, charged) + require.Equal(t, 2, a.count()) + require.Equal(t, uint64(2), a.failedReservations) + + a.release("other") + charged, err = a.reserveReceived([]string{"local", "remote", "remote"}) + require.NoError(t, err) + require.Equal(t, []string{"remote"}, charged) + require.Equal(t, 2, a.count()) + + fs, err := fileservice.NewMemoryFS("shared", fileservice.DisabledCacheConfig, nil) + require.NoError(t, err) + owner, err := newUnpublishedS3ObjectOwner(fs, a, charged, "remote") + require.NoError(t, err) + owner.Accept("remote") + require.Equal(t, 1, a.count()) + owner.Accept("remote") + require.Equal(t, 1, a.count(), "accepting twice must not release another owner's ticket") + a.release("local") + require.Zero(t, a.count()) +} + +func TestUnpublishedS3AdmissionConcurrentBound(t *testing.T) { + const limit = 16 + a := newUnpublishedS3Admission(limit) + var wg sync.WaitGroup + for i := range 128 { + wg.Add(1) + go func() { + defer wg.Done() + _ = a.reserveUpload(fmt.Sprintf("object-%d", i)) + }() + } + wg.Wait() + require.Equal(t, limit, a.count()) + for name := range a.names { + a.release(name) + } + require.Zero(t, a.count()) +} + +func TestUnpublishedS3AdmissionDefaultLimit(t *testing.T) { + a := newUnpublishedS3Admission(defaultUnpublishedS3TicketLimit) + for i := range defaultUnpublishedS3TicketLimit { + require.NoError(t, a.reserveUpload(fmt.Sprintf("object-%d", i))) + } + require.Equal(t, defaultUnpublishedS3TicketLimit, a.count()) + require.Error(t, a.reserveUpload("overflow")) + a.release("object-0") + require.NoError(t, a.reserveUpload("overflow")) +} + +func TestUnpublishedS3OwnerRetainsTicketUntilDelete(t *testing.T) { + a := newUnpublishedS3Admission(1) + baseFS, err := fileservice.NewMemoryFS("shared", fileservice.DisabledCacheConfig, nil) + require.NoError(t, err) + const name = "unpublished-object" + require.NoError(t, baseFS.Write(context.Background(), fileservice.IOVector{ + FilePath: name, + Entries: []fileservice.IOEntry{{Size: 1, Data: []byte("x")}}, + })) + fs := &failOnceDeleteFileService{FileService: baseFS, failNextDelete: true} + require.NoError(t, a.reserveUpload(name)) + owner, err := newUnpublishedS3ObjectOwner(fs, a, []string{name}, name) + require.NoError(t, err) + require.Error(t, a.reserveUpload("next")) + require.Error(t, owner.Cleanup(context.Background())) + require.Equal(t, 1, a.count(), "failed Delete must retain the cleanup ticket") + require.Error(t, a.reserveUpload("next")) + require.NoError(t, owner.Cleanup(context.Background())) + require.Zero(t, a.count()) + require.NoError(t, a.reserveUpload("next")) +} + +type failSecondDeleteBatchFS struct { + fileservice.FileService + batchSizes []int + batchNames [][]string +} + +func (fs *failSecondDeleteBatchFS) Delete(ctx context.Context, names ...string) error { + fs.batchSizes = append(fs.batchSizes, len(names)) + fs.batchNames = append(fs.batchNames, append([]string(nil), names...)) + if len(fs.batchSizes) == 2 { + return errors.New("second batch unavailable") + } + if fs.FileService != nil { + return fs.FileService.Delete(ctx, names...) + } + return nil +} + +func TestUnpublishedS3OwnerReleasesCompletedDeleteBatches(t *testing.T) { + const objectCount = 1001 + admission := newUnpublishedS3Admission(objectCount) + baseFS, err := fileservice.NewMemoryFS("shared", fileservice.DisabledCacheConfig, nil) + require.NoError(t, err) + names := make([]string, objectCount) + for i := range names { + names[i] = fmt.Sprintf("unpublished-%d", i) + require.NoError(t, admission.reserveUpload(names[i])) + require.NoError(t, baseFS.Write(context.Background(), fileservice.IOVector{ + FilePath: names[i], + Entries: []fileservice.IOEntry{{Size: 1, Data: []byte("x")}}, + })) + } + fs := &failSecondDeleteBatchFS{FileService: baseFS} + owner, err := newUnpublishedS3ObjectOwner(fs, admission, names, names...) + require.NoError(t, err) + require.Error(t, admission.reserveUpload("new-upload"), "full admission must reject before another upload") + + require.ErrorContains(t, owner.Cleanup(context.Background()), "second batch unavailable") + require.Equal(t, 1, admission.count(), "the completed batch no longer needs cleanup tickets") + require.Equal(t, []int{1000, 1}, fs.batchSizes) + _, err = baseFS.StatFile(context.Background(), fs.batchNames[0][0]) + require.True(t, moerr.IsMoErrCode(err, moerr.ErrFileNotFound), "a confirmed batch must be physically absent: %v", err) + _, err = baseFS.StatFile(context.Background(), fs.batchNames[1][0]) + require.NoError(t, err, "the unconfirmed batch must remain available for retry") + require.True(t, owner.Pending(), "the failed batch keeps its cleanup owner") + require.NoError(t, admission.reserveUpload("new-upload"), + "confirmed deletion must reopen only its released capacity") + require.Equal(t, 2, admission.count()) + admission.release("new-upload") + + require.NoError(t, owner.Cleanup(context.Background())) + require.Zero(t, admission.count()) + require.Equal(t, []int{1000, 1, 1}, fs.batchSizes, "retry only the unconfirmed object") + _, err = baseFS.StatFile(context.Background(), fs.batchNames[1][0]) + require.True(t, moerr.IsMoErrCode(err, moerr.ErrFileNotFound), "the retried object must be physically absent: %v", err) + require.False(t, owner.Pending()) +} + +func TestUnpublishedS3WriterRetryReleasesCompletedDeleteBatches(t *testing.T) { + const objectCount = 1001 + serviceID := t.Name() + moruntime.SetupServiceBasedRuntime(serviceID, moruntime.DefaultRuntime()) + server := NewServer(serviceID) + names := make([]string, objectCount) + for i := range names { + names[i] = fmt.Sprintf("writer-unpublished-%d", i) + require.NoError(t, server.reserveUnpublishedS3Upload(names[i])) + } + fs := &failSecondDeleteBatchFS{} + writer := &CNS3Writer{ + fs: fs, serviceID: serviceID, cleanupPending: true, + ownedPersistedNames: names, + } + + require.ErrorContains(t, writer.CloseWithCleanup(context.Background(), true), "second batch unavailable") + require.Equal(t, 1, server.UnpublishedS3AdmissionStats().Used) + require.Len(t, writer.ownedPersistedNames, 1) + require.True(t, writer.PendingUnpublishedCleanup()) + require.Nil(t, writer.sinker, "the retry shell must not retain its sinker") + + require.NoError(t, writer.CloseWithCleanup(context.Background(), true)) + require.Zero(t, server.UnpublishedS3AdmissionStats().Used) + require.False(t, writer.PendingUnpublishedCleanup()) + require.Equal(t, []int{1000, 1, 1}, fs.batchSizes) +} + +type ambiguousDeleteBatchFS struct { + fileservice.FileService + deleteCalls int +} + +func (fs *ambiguousDeleteBatchFS) Delete(_ context.Context, _ ...string) error { + fs.deleteCalls++ + if fs.deleteCalls == 1 { + return errors.New("response lost after physical deletion") + } + return nil +} + +func TestUnpublishedS3OwnerKeepsAmbiguousDeleteBatchCharged(t *testing.T) { + admission := newUnpublishedS3Admission(1) + require.NoError(t, admission.reserveUpload("unpublished-object")) + fs := &ambiguousDeleteBatchFS{} + owner, err := newUnpublishedS3ObjectOwner(fs, admission, + []string{"unpublished-object"}, "unpublished-object") + require.NoError(t, err) + + require.ErrorContains(t, owner.Cleanup(context.Background()), "response lost") + require.Equal(t, 1, admission.count()) + require.True(t, owner.Pending()) + require.NoError(t, owner.Cleanup(context.Background())) + require.Zero(t, admission.count()) + require.Equal(t, 2, fs.deleteCalls) +} + +func TestUnpublishedS3AdmissionRejectsUnboundedNames(t *testing.T) { + a := newUnpublishedS3Admission(1) + require.Error(t, a.reserveUpload("")) + require.Error(t, a.reserveUpload(string(make([]byte, maxUnpublishedS3ObjectNameLen+1)))) + require.Zero(t, a.count()) +} + +func TestUnpublishedS3AdmissionMetrics(t *testing.T) { + serviceID := t.Name() + a := newUnpublishedS3Admission(1, serviceID) + require.NoError(t, a.reserveUpload("first")) + require.Error(t, a.reserveUpload("second")) + require.Equal(t, float64(1), promtestutil.ToFloat64( + metricv2.UnpublishedS3TicketsGauge.WithLabelValues(serviceID, "used"))) + require.Equal(t, float64(1), promtestutil.ToFloat64( + metricv2.UnpublishedS3TicketsGauge.WithLabelValues(serviceID, "high_water"))) + require.Equal(t, float64(1), promtestutil.ToFloat64( + metricv2.UnpublishedS3AdmissionFailuresCounter.WithLabelValues(serviceID))) + a.release("first") + require.Zero(t, promtestutil.ToFloat64( + metricv2.UnpublishedS3TicketsGauge.WithLabelValues(serviceID, "used"))) +} + +func TestCNS3WriterCloseReleasesAcceptedCloneTicket(t *testing.T) { + serviceID := t.Name() + moruntime.SetupServiceBasedRuntime(serviceID, moruntime.DefaultRuntime()) + srv := NewServer(serviceID) + fs, err := fileservice.NewMemoryFS("shared", fileservice.DisabledCacheConfig, nil) + require.NoError(t, err) + require.NoError(t, srv.reserveUnpublishedS3Upload("clone-object")) + writer := &CNS3Writer{fs: fs, serviceID: serviceID, ownedPersistedNames: []string{"clone-object"}} + require.NoError(t, writer.Close()) + require.Zero(t, srv.unpublishedS3Admission.count()) +} + +func BenchmarkUnpublishedS3AdmissionReserveRelease(b *testing.B) { + a := newUnpublishedS3Admission(defaultUnpublishedS3TicketLimit, b.Name()) + names := make([]string, 1024) + for i := range names { + names[i] = fmt.Sprintf("bench-object-%d", i) + } + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + name := names[i%len(names)] + if err := a.reserveUpload(name); err != nil { + b.Fatal(err) + } + a.release(name) + } +} + +func BenchmarkUnpublishedS3AdmissionRetainedBytes(b *testing.B) { + const tickets = 65_536 + names := make([]string, tickets) + for i := range names { + names[i] = fmt.Sprintf("bench-object-%020d", i) + } + runtime.GC() + var before, after runtime.MemStats + runtime.ReadMemStats(&before) + a := newUnpublishedS3Admission(tickets) + for _, name := range names { + if err := a.reserveUpload(name); err != nil { + b.Fatal(err) + } + } + runtime.ReadMemStats(&after) + b.ReportMetric(float64(int64(after.HeapAlloc)-int64(before.HeapAlloc))/tickets, "ledger-bytes/ticket") + runtime.KeepAlive(a) +} diff --git a/pkg/sql/compile/compile.go b/pkg/sql/compile/compile.go index b2a1967489721..4a33a4d94a080 100644 --- a/pkg/sql/compile/compile.go +++ b/pkg/sql/compile/compile.go @@ -9067,9 +9067,21 @@ func (c *Compile) compileMultiUpdate(node *plan.Node, ss []*Scope) ([]*Scope, er // Determine whether to Write S3 toWriteS3 := node.Stats.GetOutcnt()*float64(SingleLineSizeEstimate) > float64(DistributedThreshold) || c.anal.qry.LoadWriteS3 + partitioned := c.proc.GetPartitionService().Enabled() && hasPartitionedUpdateTarget(node.UpdateCtxList) currentFirstFlag := c.anal.isFirst if toWriteS3 { + if partitioned { + // PartitionMultiUpdate has no remote instruction codec. Keep its + // writer on this CN and send only the source pipeline over MORPC. + // Group shuffle buckets first so each remote source is standalone. + ss = c.groupShuffleBucketsByCNIfNeeded(ss) + for i, source := range ss { + if !source.ipAddrMatch(c.addr) { + ss[i] = c.newMergeScope([]*Scope{source}) + } + } + } if len(ss) == 1 && ss[0].NodeInfo.Mcpu == 1 { mcpu := c.getParallelSizeForExternalScan(node, c.ncpu) if mcpu > 1 { @@ -9124,6 +9136,11 @@ func (c *Compile) compileMultiUpdate(node *plan.Node, ss []*Scope) ([]*Scope, er rs.setRootOperator(multiUpdateArg) ss = []*Scope{rs} } else { + if partitioned && len(ss) == 1 && !ss[0].ipAddrMatch(c.addr) { + // The TP path below otherwise attaches an unencodable partition + // writer directly to a single remote source. + ss = []*Scope{c.newMergeScope(ss)} + } // The operator below is attached to ss[0] alone, so more than one input // pipeline must be merged first or the others' rows are dropped. A TP // query normally builds a single scope, but a multi-file LOAD fanout diff --git a/pkg/sql/compile/partition_multi_update_placement_test.go b/pkg/sql/compile/partition_multi_update_placement_test.go new file mode 100644 index 0000000000000..e78e2e6e49616 --- /dev/null +++ b/pkg/sql/compile/partition_multi_update_placement_test.go @@ -0,0 +1,186 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package compile + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/matrixorigin/matrixone/pkg/partitionservice" + "github.com/matrixorigin/matrixone/pkg/pb/plan" + "github.com/matrixorigin/matrixone/pkg/sql/colexec/merge" + "github.com/matrixorigin/matrixone/pkg/sql/colexec/multi_update" + "github.com/matrixorigin/matrixone/pkg/sql/colexec/value_scan" + "github.com/matrixorigin/matrixone/pkg/sql/features" + plan2 "github.com/matrixorigin/matrixone/pkg/sql/plan" + "github.com/matrixorigin/matrixone/pkg/vm" + "github.com/matrixorigin/matrixone/pkg/vm/engine" +) + +type enabledCompilePartitionService struct { + partitionservice.PartitionService +} + +func (enabledCompilePartitionService) Enabled() bool { return true } + +func TestPartitionMultiUpdateWriterStaysOnCoordinator(t *testing.T) { + for _, action := range []struct { + name string + writeS3 bool + }{ + {"s3", true}, + {"table", false}, + } { + for _, sourceAddr := range []string{"cn1:6001", "cn2:6001"} { + t.Run(action.name+"/"+sourceAddr, func(t *testing.T) { + c := NewMockCompile(t) + defer c.proc.Free() + c.addr = "cn1:6001" + c.ncpu = 1 + c.execType = plan2.ExecTypeTP + c.anal = &AnalyzeModule{qry: &plan.Query{LoadWriteS3: action.writeS3}} + c.proc.Base.PartitionService = enabledCompilePartitionService{partitionservice.DisabledService} + source := newScope(Remote) + source.NodeInfo = engine.Node{Addr: sourceAddr, Mcpu: 1} + source.IsRemote = sourceAddr != c.addr + source.Proc = c.proc.NewNoContextChildProc(1) + source.RootOp = value_scan.NewArgument() + node := &plan.Node{ + Stats: &plan.Stats{}, + UpdateCtxList: []*plan.UpdateCtx{{TableDef: &plan.TableDef{TblId: 1, FeatureFlag: features.Partitioned}}}, + } + + result, err := c.compileMultiUpdate(node, []*Scope{source}) + require.NoError(t, err) + require.Len(t, result, 1) + var writers []*Scope + var remoteSources []*Scope + var visit func(*Scope) + visit = func(scope *Scope) { + if scope == nil { + return + } + if scope.Magic == Remote && !scope.ipAddrMatch(c.addr) { + remoteSources = append(remoteSources, scope) + } + var walkOp func(vm.Operator) + walkOp = func(op vm.Operator) { + if op == nil { + return + } + if _, ok := op.(*multi_update.PartitionMultiUpdate); ok { + writers = append(writers, scope) + } + for _, child := range op.GetOperatorBase().Children { + walkOp(child) + } + } + walkOp(scope.RootOp) + for _, pre := range scope.PreScopes { + visit(pre) + } + } + visit(result[0]) + require.Len(t, writers, 1) + require.True(t, writers[0].ipAddrMatch(c.addr), "partition writer must execute on the coordinator") + if sourceAddr != c.addr { + require.Len(t, remoteSources, 1) + encoded, _ := getScopeForRemoteRunEncoding(remoteSources[0]) + require.IsType(t, &value_scan.ValueScan{}, encoded.RootOp) + _, _, err = convertToPipelineInstruction(encoded.RootOp, c.proc, &scopeContext{}, 1) + require.NoError(t, err, "the remote source must use the existing operator codec") + } else { + require.Empty(t, remoteSources) + } + }) + } + } +} + +func TestPartitionMultiUpdateKeepsParallelWritersForRemoteSources(t *testing.T) { + c := NewMockCompile(t) + defer c.proc.Free() + c.addr = "cn1:6001" + c.ncpu = 2 + c.execType = plan2.ExecTypeAP_MULTICN + c.anal = &AnalyzeModule{qry: &plan.Query{LoadWriteS3: true}} + c.proc.Base.PartitionService = enabledCompilePartitionService{partitionservice.DisabledService} + sources := make([]*Scope, 2) + for i, addr := range []string{"cn2:6001", "cn3:6001"} { + sources[i] = newScope(Remote) + sources[i].NodeInfo = engine.Node{Addr: addr, Mcpu: 1} + sources[i].IsRemote = true + sources[i].Proc = c.proc.NewNoContextChildProc(1) + sources[i].RootOp = value_scan.NewArgument() + } + node := &plan.Node{ + Stats: &plan.Stats{}, + UpdateCtxList: []*plan.UpdateCtx{{TableDef: &plan.TableDef{TblId: 1, FeatureFlag: features.Partitioned}}}, + } + originalSources := append([]*Scope(nil), sources...) + result, err := c.compileMultiUpdate(node, sources) + require.NoError(t, err) + require.Len(t, result, 1) + require.Len(t, result[0].PreScopes, 2) + for i, writer := range result[0].PreScopes { + require.True(t, writer.ipAddrMatch(c.addr)) + require.Len(t, writer.PreScopes, 1) + require.Same(t, originalSources[i], writer.PreScopes[0]) + require.IsType(t, &multi_update.PartitionMultiUpdate{}, writer.RootOp.GetOperatorBase().GetChildren(0)) + require.Equal(t, remoteS3None, scopeS3Output(originalSources[i])) + } +} + +func TestPartitionMultiUpdateGroupsRemoteShuffleBeforeMovingWriter(t *testing.T) { + c := NewMockCompile(t) + defer c.proc.Free() + c.addr = "cn1:6001" + c.ncpu = 2 + c.execType = plan2.ExecTypeAP_MULTICN + c.cnList = engine.Nodes{{Addr: "cn1:6001", Mcpu: 2}, {Addr: "cn2:6001", Mcpu: 2}} + c.anal = &AnalyzeModule{qry: &plan.Query{LoadWriteS3: true}} + c.proc.Base.PartitionService = enabledCompilePartitionService{partitionservice.DisabledService} + c.proc.Base.TxnOperator = fakeTxnOperator{} + buckets := make([]*Scope, 4) + for i, addr := range []string{"cn1:6001", "cn1:6001", "cn2:6001", "cn2:6001"} { + buckets[i] = newScope(Remote) + buckets[i].NodeInfo = engine.Node{Addr: addr, Mcpu: 1} + buckets[i].Proc = c.proc.NewContextChildProc(1) + buckets[i].setRootOperator(merge.NewArgument()) + } + remoteDispatch := newDispatchSrcScopeForTest(c.proc, "cn2:6001", buckets[2:], buckets[:2]) + buckets[2].PreScopes = append(buckets[2].PreScopes, remoteDispatch) + require.False(t, checkPipelineStandaloneExecutableAtRemote(buckets[2])) + node := &plan.Node{ + Stats: &plan.Stats{}, + UpdateCtxList: []*plan.UpdateCtx{{TableDef: &plan.TableDef{TblId: 1, FeatureFlag: features.Partitioned}}}, + } + result, err := c.compileMultiUpdate(node, buckets) + require.NoError(t, err) + require.Len(t, result, 1) + require.Len(t, result[0].PreScopes, 2) + var remoteGroup *Scope + for _, writer := range result[0].PreScopes { + if len(writer.PreScopes) == 1 && writer.PreScopes[0].NodeInfo.Addr == "cn2:6001" { + remoteGroup = writer.PreScopes[0] + require.True(t, writer.ipAddrMatch(c.addr)) + require.IsType(t, &multi_update.PartitionMultiUpdate{}, writer.RootOp.GetOperatorBase().GetChildren(0)) + } + } + require.NotNil(t, remoteGroup) + require.Len(t, remoteGroup.PreScopes, 2) + require.True(t, checkPipelineStandaloneExecutableAtRemote(remoteGroup)) +} diff --git a/pkg/sql/compile/remoterunClient.go b/pkg/sql/compile/remoterunClient.go index f631d9781f2a5..cbcd30ef00203 100644 --- a/pkg/sql/compile/remoterunClient.go +++ b/pkg/sql/compile/remoterunClient.go @@ -24,15 +24,23 @@ import ( "time" "github.com/google/uuid" + "github.com/matrixorigin/matrixone/pkg/catalog" "github.com/matrixorigin/matrixone/pkg/cnservice/cnclient" "github.com/matrixorigin/matrixone/pkg/common/moerr" "github.com/matrixorigin/matrixone/pkg/common/morpc" "github.com/matrixorigin/matrixone/pkg/common/mpool" moruntime "github.com/matrixorigin/matrixone/pkg/common/runtime" "github.com/matrixorigin/matrixone/pkg/container/batch" + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/container/vector" + "github.com/matrixorigin/matrixone/pkg/objectio" "github.com/matrixorigin/matrixone/pkg/pb/pipeline" + "github.com/matrixorigin/matrixone/pkg/sql/colexec" "github.com/matrixorigin/matrixone/pkg/sql/colexec/connector" + "github.com/matrixorigin/matrixone/pkg/sql/colexec/deletion" "github.com/matrixorigin/matrixone/pkg/sql/colexec/dispatch" + "github.com/matrixorigin/matrixone/pkg/sql/colexec/insert" + "github.com/matrixorigin/matrixone/pkg/sql/colexec/multi_update" "github.com/matrixorigin/matrixone/pkg/sql/colexec/value_scan" "github.com/matrixorigin/matrixone/pkg/sql/models" "github.com/matrixorigin/matrixone/pkg/util/fault" @@ -316,6 +324,7 @@ func receiveMessageFromCnServerIfOnlyRun(s *Scope, sender *messageSenderOnClient var err error mp := s.Proc.Mp() + outputKind := scopeS3Output(s) // Waiting the EndMessage or ErrorMessage. // In fact, for a pipeline that only needs to be executed remotely but without sending data back, // there should be no message sent back except for EndMessage or ErrorMessage. @@ -325,6 +334,10 @@ func receiveMessageFromCnServerIfOnlyRun(s *Scope, sender *messageSenderOnClient if err != nil || end || bat == nil { return err } + if outputKind != remoteS3None && !bat.IsEmpty() { + bat.Clean(mp) + return moerr.NewInternalErrorNoCtx("unexpected remote S3 output on a no-output stream") + } bat.Clean(mp) if err = sender.acknowledgeRemoteBatch(); err != nil { return err @@ -343,12 +356,17 @@ func receiveMessageFromCnServerIfConnector(s *Scope, sender *messageSenderOnClie mp := s.Proc.Mp() nextReg := s.RootOp.(*connector.Connector).Reg + outputKind := scopeS3Output(s) for { bat, end, err = sender.receiveBatch() if err != nil || end || bat == nil { return err } connectorAnalyze.Network(bat) + if err = retainRemoteS3Output(s.Proc, outputKind, bat); err != nil { + bat.Clean(mp) + return err + } var receiverDone bool if receiverDone, err = forwardRemoteBatchWithContext(sender, nextReg, bat, mp); err != nil { @@ -356,7 +374,7 @@ func receiveMessageFromCnServerIfConnector(s *Scope, sender *messageSenderOnClie } // A stopped receiver intentionally discarded the decoded batch, but the // remote sender still owns its credit until this ACK is sent. - if err = sender.acknowledgeRemoteBatch(); err != nil { + if err = sender.acknowledgeRemoteBatchWithOwnership(outputKind != remoteS3None); err != nil { return err } if receiverDone { @@ -365,6 +383,182 @@ func receiveMessageFromCnServerIfConnector(s *Scope, sender *messageSenderOnClie } } +type remoteS3Output uint8 + +const ( + remoteS3None remoteS3Output = iota + remoteS3MultiUpdate + remoteS3Insert + remoteS3Delete +) + +// Follow only operators that preserve the producer's metadata format. In +// particular, do not search through FlushS3Info, MergeBlock or MergeDelete: +// they have already registered their input in the worker's workspace. +func scopeS3Output(s *Scope) remoteS3Output { + if s == nil { + return remoteS3None + } + preOutput := func() remoteS3Output { + for _, pre := range s.PreScopes { + if kind := scopeS3Output(pre); kind != remoteS3None { + return kind + } + } + return remoteS3None + } + if s.RootOp == nil { + return preOutput() + } + var output func(vm.Operator) remoteS3Output + output = func(op vm.Operator) remoteS3Output { + switch arg := op.(type) { + case *multi_update.MultiUpdate: + if arg.Action == multi_update.UpdateWriteS3 { + return remoteS3MultiUpdate + } + case *insert.Insert: + if arg.ToWriteS3 { + return remoteS3Insert + } + case *deletion.Deletion: + if arg.RemoteDelete { + return remoteS3Delete + } + default: + switch op.OpType() { + case vm.Connector, vm.Dispatch: + if children := op.GetOperatorBase().Children; len(children) != 0 { + return output(children[0]) + } + return preOutput() + case vm.Merge: + for _, child := range op.GetOperatorBase().Children { + if kind := output(child); kind != remoteS3None { + return kind + } + } + return preOutput() + } + } + return remoteS3None + } + return output(s.RootOp) +} + +// Retain before forwarding or discarding, and before releasing the remote +// batch credit. A failed takeover must leave the batch unacknowledged. +func retainRemoteS3Output(proc *process.Process, kind remoteS3Output, bat *batch.Batch) error { + if kind == remoteS3None || bat == nil || bat.IsEmpty() { + return nil + } + if kind == remoteS3MultiUpdate { + return multi_update.RetainOutputS3ObjectOwnership(proc, bat) + } + var names []string + if kind == remoteS3Insert { + var err error + names, err = remoteS3MetadataNames(bat) + if err != nil { + return err + } + } else { + if len(bat.Vecs) != 5 || bat.Vecs[1] == nil || bat.Vecs[2] == nil || + !bat.Vecs[1].GetType().IsVarlen() || bat.Vecs[2].GetType().Oid != types.T_int8 || + bat.Vecs[1].Length() != bat.RowCount() || bat.Vecs[2].Length() != bat.RowCount() { + return moerr.NewInternalErrorNoCtx("invalid remote delete S3 output") + } + for row := 0; row < bat.RowCount(); row++ { + if vector.GetFixedAtWithTypeCheck[int8](bat.Vecs[2], row) != deletion.FlushDeltaLoc { + continue + } + metadata := batch.NewOffHeapEmpty() + err := metadata.UnmarshalBinaryWithAnyMp(bat.Vecs[1].GetBytesAt(row), proc.Mp()) + if err == nil { + var objectNames []string + objectNames, err = remoteS3MetadataNames(metadata) + names = append(names, objectNames...) + } + metadata.Clean(proc.Mp()) + if err != nil { + return err + } + } + } + if len(names) == 0 { + return nil + } + fs, err := colexec.GetSharedFSFromProc(proc) + if err != nil { + return err + } + return colexec.RetainReceivedUnpublishedS3ObjectNames(proc, fs, names...) +} + +func remoteS3MetadataNames(bat *batch.Batch) ([]string, error) { + var names []string + foundMetadata := false + legacy := len(bat.Attrs) > 0 && bat.Attrs[0] == catalog.BlockMeta_TableIdx_Insert + if legacy && (len(bat.Vecs) == 0 || bat.Vecs[0] == nil || + bat.Vecs[0].GetType().Oid != types.T_int16 || bat.Vecs[0].Length() != bat.RowCount()) { + return nil, moerr.NewInternalErrorNoCtx("invalid remote S3 table index") + } + for col, attr := range bat.Attrs { + if attr != catalog.ObjectMeta_ObjectStats && attr != catalog.BlockMeta_BlockInfo { + continue + } + foundMetadata = true + if col >= len(bat.Vecs) || bat.Vecs[col] == nil || !bat.Vecs[col].GetType().IsVarlen() { + return nil, moerr.NewInternalErrorNoCtx("invalid remote S3 metadata column") + } + for row := 0; row < bat.Vecs[col].Length(); row++ { + data := bat.Vecs[col].GetBytesAt(row) + // Legacy insert output can carry raw batches in negative table-index + // rows. Those rows do not describe newly persisted objects. + if legacy && attr == catalog.BlockMeta_BlockInfo && + row < bat.Vecs[0].Length() && vector.GetFixedAtWithTypeCheck[int16](bat.Vecs[0], row) < 0 { + continue + } + if len(data) == 0 { // stats are sparse in legacy block-info batches + continue + } + if attr == catalog.ObjectMeta_ObjectStats { + if len(data) != objectio.ObjectStatsLen { + return nil, moerr.NewInternalErrorNoCtx("invalid remote S3 object stats") + } + stats := objectio.ObjectStats(data) + names = append(names, stats.ObjectName().String()) + } else { + if len(data) != objectio.BlockInfoSize { + return nil, moerr.NewInternalErrorNoCtx("invalid remote S3 block info") + } + names = append(names, objectio.DecodeBlockInfo(data).MetaLocation().Name().String()) + } + } + } + if !foundMetadata { + return nil, moerr.NewInternalErrorNoCtx("missing remote S3 metadata columns") + } + if bat.RowCount() > 0 && len(names) == 0 { + // Stats are packed per object, while block-info is packed per block; + // their lengths need not match. Only legacy raw-batch-only output may + // legitimately contain rows without any persisted object names. + rawOnly := legacy + if legacy { + for row := 0; row < bat.RowCount(); row++ { + if vector.GetFixedAtWithTypeCheck[int16](bat.Vecs[0], row) >= 0 { + rawOnly = false + break + } + } + } + if !rawOnly { + return nil, moerr.NewInternalErrorNoCtx("remote S3 metadata contains no object names") + } + } + return names, nil +} + func receiveMessageFromCnServerIfDispatch(s *Scope, sender *messageSenderOnClient) error { var bat *batch.Batch var end bool @@ -391,6 +585,7 @@ func receiveMessageFromCnServerIfDispatch(s *Scope, sender *messageSenderOnClien dispatchAnalyze := dispatchRunner.GetOperatorBase().OpAnalyzer mp := s.Proc.Mp() + outputKind := scopeS3Output(s) for { bat, end, err = sender.receiveBatch() if err != nil || end || bat == nil { @@ -398,6 +593,10 @@ func receiveMessageFromCnServerIfDispatch(s *Scope, sender *messageSenderOnClien } dispatchAnalyze.Network(bat) + if err = retainRemoteS3Output(s.Proc, outputKind, bat); err != nil { + bat.Clean(mp) + return err + } fakeValueScanOperator.Batchs = append(fakeValueScanOperator.Batchs, bat) result, errCall := vm.Exec(dispatchRunner, s.Proc) @@ -407,7 +606,7 @@ func receiveMessageFromCnServerIfDispatch(s *Scope, sender *messageSenderOnClien } // ExecStop can mean that every receiver has already stopped. Release the // decoded batch's remote credit before ending the receive loop. - if err = sender.acknowledgeRemoteBatch(); err != nil { + if err = sender.acknowledgeRemoteBatchWithOwnership(outputKind != remoteS3None); err != nil { return err } if result.Status == vm.ExecStop { @@ -773,6 +972,10 @@ func (sender *messageSenderOnClient) receiveBatch() (bat *batch.Batch, over bool } func (sender *messageSenderOnClient) acknowledgeRemoteBatch() error { + return sender.acknowledgeRemoteBatchWithOwnership(false) +} + +func (sender *messageSenderOnClient) acknowledgeRemoteBatchWithOwnership(retained bool) error { sequence := sender.pendingBatchAck if sequence == 0 { return nil @@ -782,6 +985,7 @@ func (sender *messageSenderOnClient) acknowledgeRemoteBatch() error { message.SetMessageType(pipeline.Method_PipelineBatchAck) message.SetSid(pipeline.Status_Last) message.BatchAckSequence = sequence + message.BatchAckS3OwnershipRetained = retained if err := sender.streamSender.Send(sender.ctx, message); err != nil { return err } diff --git a/pkg/sql/compile/remoterunClient_test.go b/pkg/sql/compile/remoterunClient_test.go index 846e725d2aa24..16e60cecb9433 100644 --- a/pkg/sql/compile/remoterunClient_test.go +++ b/pkg/sql/compile/remoterunClient_test.go @@ -15,7 +15,9 @@ package compile import ( + "bytes" "context" + "fmt" "sync/atomic" "testing" "time" @@ -32,21 +34,340 @@ import ( "github.com/matrixorigin/matrixone/pkg/common/morpc/mock_morpc" "github.com/matrixorigin/matrixone/pkg/common/mpool" "github.com/matrixorigin/matrixone/pkg/common/runtime" + "github.com/matrixorigin/matrixone/pkg/container/batch" + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/container/vector" "github.com/matrixorigin/matrixone/pkg/defines" + "github.com/matrixorigin/matrixone/pkg/fileservice" + mock_frontend "github.com/matrixorigin/matrixone/pkg/frontend/test" + "github.com/matrixorigin/matrixone/pkg/objectio" "github.com/matrixorigin/matrixone/pkg/pb/pipeline" "github.com/matrixorigin/matrixone/pkg/pb/plan" "github.com/matrixorigin/matrixone/pkg/sql/colexec" "github.com/matrixorigin/matrixone/pkg/sql/colexec/connector" + "github.com/matrixorigin/matrixone/pkg/sql/colexec/deletion" "github.com/matrixorigin/matrixone/pkg/sql/colexec/dispatch" + "github.com/matrixorigin/matrixone/pkg/sql/colexec/insert" + "github.com/matrixorigin/matrixone/pkg/sql/colexec/merge" + "github.com/matrixorigin/matrixone/pkg/sql/colexec/mergeblock" + "github.com/matrixorigin/matrixone/pkg/sql/colexec/mergedelete" + "github.com/matrixorigin/matrixone/pkg/sql/colexec/multi_update" "github.com/matrixorigin/matrixone/pkg/sql/colexec/value_scan" "github.com/matrixorigin/matrixone/pkg/testutil" "github.com/matrixorigin/matrixone/pkg/util/fault" + "github.com/matrixorigin/matrixone/pkg/vm" "github.com/matrixorigin/matrixone/pkg/vm/engine" "github.com/matrixorigin/matrixone/pkg/vm/process" ) var _ cnclient.PipelineClient = new(testPipelineClient) +func TestScopeS3Output(t *testing.T) { + update := multi_update.NewArgument() + update.Action = multi_update.UpdateWriteS3 + update.IsRemote = true + connectorOp := connector.NewArgument() + connectorOp.AppendChild(update) + + require.Equal(t, remoteS3MultiUpdate, scopeS3Output(&Scope{RootOp: connectorOp})) + update.IsRemote = false + require.Equal(t, remoteS3MultiUpdate, scopeS3Output(&Scope{RootOp: connectorOp})) + update.IsRemote = true + update.Action = multi_update.UpdateFlushS3Info + require.Equal(t, remoteS3None, scopeS3Output(&Scope{RootOp: connectorOp})) + update.Action = multi_update.UpdateWriteS3 + group := &Scope{RootOp: merge.NewArgument(), PreScopes: []*Scope{{RootOp: connectorOp}}} + require.Equal(t, remoteS3MultiUpdate, scopeS3Output(group)) + group.RootOp = nil + require.Equal(t, remoteS3MultiUpdate, scopeS3Output(group)) + group.RootOp = dispatch.NewArgument() + require.Equal(t, remoteS3MultiUpdate, scopeS3Output(group)) + // Consumers are a boundary even when a grouped PreScope still contains a producer. + for _, consumer := range []vm.Operator{&multi_update.MultiUpdate{Action: multi_update.UpdateFlushS3Info}, &mergeblock.MergeBlock{}, &mergedelete.MergeDelete{}} { + consumer.GetOperatorBase().AppendChild(update) + group.RootOp = consumer + require.Equal(t, remoteS3None, scopeS3Output(group)) + wrapper := &connector.Connector{} + wrapper.AppendChild(consumer) + group.RootOp = wrapper + require.Equal(t, remoteS3None, scopeS3Output(group)) + } + for _, tc := range []struct { + op vm.Operator + kind remoteS3Output + }{ + {&insert.Insert{ToWriteS3: true}, remoteS3Insert}, + {&insert.Insert{}, remoteS3None}, + {&deletion.Deletion{RemoteDelete: true}, remoteS3Delete}, + {&deletion.Deletion{}, remoteS3None}, + } { + require.Equal(t, tc.kind, scopeS3Output(&Scope{RootOp: tc.op})) + } + + connectorOp.Release() + update.Release() +} + +func TestRemoteS3MetadataNames(t *testing.T) { + proc := testutil.NewProcess(t) + defer proc.Free() + out, name := remoteS3TestOutput(t, proc, remoteS3Insert) + defer out.Clean(proc.Mp()) + names, err := remoteS3MetadataNames(out) + require.NoError(t, err) + require.Equal(t, []string{name}, names) + // Real producer layout: two block records but only one object-stats record. + multiBlock := colexec.AllocCNS3ResultBat(false) + defer multiBlock.Clean(proc.Mp()) + multiStats := objectio.ObjectStats(append([]byte(nil), out.Vecs[1].GetBytesAt(0)...)) + require.NoError(t, objectio.SetObjectStatsBlkCnt(&multiStats, 2)) + require.NoError(t, objectio.SetObjectStatsRowCnt(&multiStats, 8193)) + require.NoError(t, colexec.ExpandObjectStatsToBatch(proc.Mp(), false, multiBlock, true, multiStats)) + require.Equal(t, 2, multiBlock.RowCount()) + require.Equal(t, 1, multiBlock.Vecs[1].Length()) + names, err = remoteS3MetadataNames(multiBlock) + require.NoError(t, err) + require.NotEmpty(t, names) + for _, parsed := range names { + require.Equal(t, name, parsed) + } + for _, vec := range multiBlock.Vecs { + vec.CleanOnlyData() + } + _, err = remoteS3MetadataNames(multiBlock) + require.ErrorContains(t, err, "no object names") + + // Legacy mixed output: a negative table index carries serialized raw data, + // while block-info rows carry names even without an object-stats column. + legacy := batch.NewWithSize(2) + defer legacy.Clean(proc.Mp()) + legacy.Attrs = []string{catalog.BlockMeta_TableIdx_Insert, catalog.BlockMeta_BlockInfo} + legacy.Vecs[0] = vector.NewVec(types.T_int16.ToType()) + legacy.Vecs[1] = vector.NewVec(types.T_text.ToType()) + require.NoError(t, vector.AppendFixed(legacy.Vecs[0], int16(-1), false, proc.Mp())) + require.NoError(t, vector.AppendBytes(legacy.Vecs[1], []byte("raw batch, not block info"), false, proc.Mp())) + legacy.SetRowCount(1) + names, err = remoteS3MetadataNames(legacy) + require.NoError(t, err) + require.Empty(t, names) + require.NoError(t, vector.SetFixedAtNoTypeCheck(legacy.Vecs[0], 0, int16(0))) + _, err = remoteS3MetadataNames(legacy) + require.ErrorContains(t, err, "invalid remote S3 block info") + require.NoError(t, vector.SetFixedAtNoTypeCheck(legacy.Vecs[0], 0, int16(-1))) + stats := objectio.ObjectStats(out.Vecs[1].GetBytesAt(0)) + block := &objectio.BlockInfo{} + block.SetMetaLocation(objectio.BuildLocation(stats.ObjectName(), objectio.Extent{}, 1, 0)) + require.NoError(t, vector.AppendFixed(legacy.Vecs[0], int16(0), false, proc.Mp())) + require.NoError(t, vector.AppendBytes(legacy.Vecs[1], objectio.EncodeBlockInfo(block), false, proc.Mp())) + legacy.SetRowCount(2) + names, err = remoteS3MetadataNames(legacy) + require.NoError(t, err) + require.Equal(t, []string{name}, names) + // Stats are packed by object, not aligned with the legacy table-index rows. + legacy.Attrs = append(legacy.Attrs, catalog.ObjectMeta_ObjectStats) + legacy.Vecs = append(legacy.Vecs, vector.NewVec(types.T_binary.ToType())) + require.NoError(t, vector.AppendBytes(legacy.Vecs[2], stats[:], false, proc.Mp())) + names, err = remoteS3MetadataNames(legacy) + require.NoError(t, err) + require.Equal(t, []string{name, name}, names) + + malformed := colexec.AllocCNS3ResultBat(true) + defer malformed.Clean(proc.Mp()) + require.NoError(t, vector.AppendBytes(malformed.Vecs[0], []byte("short"), false, proc.Mp())) + malformed.SetRowCount(1) + _, err = remoteS3MetadataNames(malformed) + require.ErrorContains(t, err, "invalid remote S3 object stats") +} + +func TestRemoteS3DeleteIgnoresRawRowsAndRejectsMalformedMetadata(t *testing.T) { + proc := testutil.NewProcess(t) + defer proc.Free() + out, _ := remoteS3TestOutput(t, proc, remoteS3Delete) + defer out.Clean(proc.Mp()) + out.Vecs[1].CleanOnlyData() + require.NoError(t, vector.AppendBytes(out.Vecs[1], []byte("not a nested metadata batch"), false, proc.Mp())) + require.NoError(t, vector.SetFixedAtNoTypeCheck(out.Vecs[2], 0, int8(deletion.FlushDeltaLoc+1))) + require.NoError(t, retainRemoteS3Output(proc, remoteS3Delete, out)) + require.NoError(t, vector.SetFixedAtNoTypeCheck(out.Vecs[2], 0, int8(deletion.FlushDeltaLoc))) + require.Error(t, retainRemoteS3Output(proc, remoteS3Delete, out)) +} + +func TestRemoteS3RejectsMissingNamesBeforeAck(t *testing.T) { + proc := testutil.NewProcess(t) + defer proc.Free() + out := colexec.AllocCNS3ResultBat(false) + out.SetRowCount(1) + data, err := out.MarshalBinaryForPipeline(&bytes.Buffer{}, true, true) + require.NoError(t, err) + out.Clean(proc.Mp()) + responses := make(chan morpc.Message, 1) + responses <- &pipeline.Message{Cmd: pipeline.Method_BatchMessage, Sid: pipeline.Status_Last, Data: data, BatchSequence: 1} + close(responses) + // No Send expectation: malformed metadata must never be acknowledged. + stream := mock_morpc.NewMockStream(gomock.NewController(t)) + sender := &messageSenderOnClient{ctx: proc.Ctx, mp: proc.Mp(), receiveCh: responses, streamSender: stream} + reg := process.NewPipelineEdge(1, 0) + op := connector.NewArgument() + defer op.Release() + op.Reg = reg + op.AppendChild(&insert.Insert{ToWriteS3: true}) + err = receiveMessageFromCnServerIfConnector(&Scope{Proc: proc, RootOp: op}, sender) + require.ErrorContains(t, err, "no object names") + require.Empty(t, reg.Ch2) + require.Equal(t, uint64(1), sender.pendingBatchAck) +} + +func remoteS3TestOutput(t *testing.T, proc *process.Process, kind remoteS3Output) (*batch.Batch, string) { + t.Helper() + id := types.Uuid{1, 2, 3} + name := objectio.BuildObjectName(&id, 1) + stats := objectio.NewObjectStats() + require.NoError(t, objectio.SetObjectStatsObjectName(stats, name)) + metadata := colexec.AllocCNS3ResultBat(kind != remoteS3Insert) + statsCol := 0 + if kind == remoteS3Insert { + statsCol = 1 + require.NoError(t, vector.AppendBytes(metadata.Vecs[0], nil, false, proc.Mp())) + } + require.NoError(t, vector.AppendBytes(metadata.Vecs[statsCol], stats[:], false, proc.Mp())) + metadata.SetRowCount(1) + if kind == remoteS3Insert { + return metadata, name.String() + } + data, err := metadata.MarshalBinary() + require.NoError(t, err) + metadata.Clean(proc.Mp()) + out := batch.NewWithSize(5) + for i := range out.Vecs { + out.Vecs[i] = vector.NewVec(types.T_text.ToType()) + } + if kind == remoteS3Delete { + out.Vecs[2] = vector.NewVec(types.T_int8.ToType()) + require.NoError(t, vector.AppendFixed(out.Vecs[2], int8(deletion.FlushDeltaLoc), false, proc.Mp())) + require.NoError(t, vector.AppendBytes(out.Vecs[1], data, false, proc.Mp())) + } else { + out.Vecs[0] = vector.NewVec(types.T_uint8.ToType()) + require.NoError(t, vector.AppendFixed(out.Vecs[0], uint8(1), false, proc.Mp())) // actionDelete + require.NoError(t, vector.AppendBytes(out.Vecs[4], data, false, proc.Mp())) + } + for _, vec := range out.Vecs { + if vec.Length() == 0 { + require.NoError(t, vector.AppendBytes(vec, nil, false, proc.Mp())) + } + } + out.SetRowCount(1) + return out, name.String() +} + +func TestRemoteS3ReceiveTakeoverBeforeAck(t *testing.T) { + oldRuntime := runtime.ServiceRuntime("") + runtime.SetupServiceBasedRuntime("", runtime.DefaultRuntime()) + server := colexec.NewServer("") + t.Cleanup(func() { runtime.SetupServiceBasedRuntime("", oldRuntime) }) + + for _, path := range []string{"connector", "dispatch", "no-output"} { + for _, kind := range []remoteS3Output{remoteS3MultiUpdate, remoteS3Insert, remoteS3Delete} { + for _, failTakeover := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/%d/fail=%t", path, kind, failTakeover), func(t *testing.T) { + ctrl := gomock.NewController(t) + fs, err := fileservice.NewMemoryFS(defines.SharedFileServiceName, fileservice.CacheConfig{}, nil) + require.NoError(t, err) + proc := testutil.NewProcess(t, testutil.WithFileService(fs)) + defer proc.Free() + workspace := &remoteS3CleanupWorkspace{} + if !failTakeover { + txn := mock_frontend.NewMockTxnOperator(ctrl) + txn.EXPECT().GetWorkspace().Return(workspace).AnyTimes() + proc.Base.TxnOperator = txn + } else { + proc.Base.TxnOperator = nil + } + out, name := remoteS3TestOutput(t, proc, kind) + data, err := out.MarshalBinaryForPipeline(&bytes.Buffer{}, true, true) + require.NoError(t, err) + out.Clean(proc.Mp()) + responses := make(chan morpc.Message, 2) + responses <- &pipeline.Message{Cmd: pipeline.Method_BatchMessage, Sid: pipeline.Status_Last, Data: data, BatchSequence: 1} + close(responses) + stream := mock_morpc.NewMockStream(ctrl) + reg := process.NewPipelineEdge(2, 0) + var root vm.Operator + var producer vm.Operator + switch kind { + case remoteS3MultiUpdate: + producer = &multi_update.MultiUpdate{Action: multi_update.UpdateWriteS3} + case remoteS3Insert: + producer = &insert.Insert{ToWriteS3: true} + case remoteS3Delete: + producer = &deletion.Deletion{RemoteDelete: true} + } + root = producer + if path == "connector" { + op := connector.NewArgument() + op.Reg = reg + op.AppendChild(producer) + root = op + defer op.Release() + } else if path == "dispatch" { + op := dispatch.NewArgument() + op.FuncId = dispatch.SendToAllLocalFunc + op.LocalRegs = []*process.WaitRegister{reg} + op.AppendChild(producer) + root = op + defer func() { op.Reset(proc, true, nil); op.Free(proc, true, nil); op.Release() }() + } + if path != "no-output" && !failTakeover { + stream.EXPECT().ID().Return(uint64(7)) + stream.EXPECT().Send(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, msg morpc.Message) error { + require.Equal(t, pipeline.Method_PipelineBatchAck, msg.(*pipeline.Message).Cmd) + require.True(t, msg.(*pipeline.Message).GetBatchAckS3OwnershipRetained()) + require.Len(t, workspace.owners, 1) + require.Equal(t, []string{name}, workspace.owners[0].Names()) + require.Equal(t, 1, server.UnpublishedS3AdmissionStats().Used) + require.Len(t, reg.Ch2, 1, "forward precedes ACK") + signal := <-reg.Ch2 + forwarded, err := signal.Action() + require.NoError(t, err) + require.Equal(t, 1, forwarded.RowCount()) + if path == "connector" { + forwarded.Clean(proc.Mp()) + } + return nil + }) + } + sender := &messageSenderOnClient{ctx: proc.Ctx, mp: proc.Mp(), receiveCh: responses, streamSender: stream} + scope := &Scope{Proc: proc, RootOp: root} + switch path { + case "connector": + err = receiveMessageFromCnServerIfConnector(scope, sender) + case "dispatch": + err = receiveMessageFromCnServerIfDispatch(scope, sender) + default: + err = receiveMessageFromCnServerIfOnlyRun(scope, sender) + } + if path == "no-output" { + require.ErrorContains(t, err, "no-output stream") + require.Empty(t, workspace.owners) + require.Equal(t, uint64(1), sender.pendingBatchAck) + require.Zero(t, server.UnpublishedS3AdmissionStats().Used) + } else if failTakeover { + require.ErrorContains(t, err, "cannot retain") + require.Empty(t, reg.Ch2) + require.Equal(t, uint64(1), sender.pendingBatchAck) + require.Zero(t, server.UnpublishedS3AdmissionStats().Used) + } else { + require.True(t, moerr.IsMoErrCode(err, moerr.ErrStreamClosed)) + require.Zero(t, sender.pendingBatchAck) + workspace.owners[0].Accept(name) + require.False(t, workspace.owners[0].Pending()) + require.Zero(t, server.UnpublishedS3AdmissionStats().Used) + } + }) + } + } + } +} + type testPipelineClient struct { genStream func(context.Context, string) (morpc.Stream, error) } diff --git a/pkg/sql/compile/remoterunServer.go b/pkg/sql/compile/remoterunServer.go index 0cdf1464bc444..a8d0d23127db1 100644 --- a/pkg/sql/compile/remoterunServer.go +++ b/pkg/sql/compile/remoterunServer.go @@ -167,6 +167,7 @@ func CnServerMessageHandler( zap.Uint64("outstanding-bytes", bytes)) }) } + handlerErr = receiver.finalizeUnpublishedS3Objects(handlerErr, lifecycle) responseSent := false if receiver.messageTyp != pipeline.Method_StopSending { // stop message only close a running pipeline, there is no need to reply the finished-message. @@ -205,6 +206,55 @@ func CnServerMessageHandler( return err } +func (receiver *messageReceiverOnServer) finalizeUnpublishedS3Objects( + handlerErr error, + lifecycle *pipelineStreamLifecycle, +) error { + if receiver.unpublishedS3CleanupWorkspace == nil { + return handlerErr + } + flowAccepted := false + if handlerErr == nil && !receiver.needNotReply && lifecycle != nil && lifecycle.batchFlow != nil { + flow := lifecycle.batchFlow + flow.mu.Lock() + // An empty/no-output stream is not evidence of a coordinator takeover. + // Only actual acknowledged output can release the worker's lease. + flowAccepted = flow.nextSeq > 0 && flow.ownershipAckedSeq == flow.nextSeq && len(flow.pending) == 0 && + flow.abortErr == nil && !flow.stoppedByReceiver + flow.mu.Unlock() + } + if flowAccepted && receiver.unpublishedS3ObjectOwners != nil { + receiver.unpublishedS3ObjectOwners.AcceptAllUnpublishedS3ObjectOwners() + } else if handlerErr == nil && receiver.unpublishedS3ObjectOwners != nil && + receiver.unpublishedS3ObjectOwners.HasUnpublishedS3ObjectOwners() { + // Without an ACK-capable stream the worker cannot prove the coordinator + // retained ownership before accepting its own cleanup lease. + handlerErr = moerr.NewNotSupportedNoCtx( + "remote S3 ownership handoff requires batch acknowledgements for exported output", + ) + } + baseCtx := receiver.unpublishedS3CleanupContext + if baseCtx == nil { + baseCtx = receiver.messageCtx + } + process.BeginPipelineCleanup(baseCtx) + cleanupCtx, cancel := colexec.UnpublishedS3CleanupContext(baseCtx) + cleanupErr := receiver.unpublishedS3CleanupWorkspace.CleanupUnpublishedS3Objects(cleanupCtx) + cancel() + if cleanupErr != nil { + if receiver.colexecServer != nil { + workspace := receiver.unpublishedS3CleanupWorkspace + if queueErr := workspace.QueueUnpublishedS3Cleanup(receiver.colexecServer); queueErr != nil { + return errors.Join(handlerErr, cleanupErr, queueErr) + } + logutil.Warn("remote unpublished S3 cleanup transferred to CN retry worker", zap.Error(cleanupErr)) + return handlerErr + } + return errors.Join(handlerErr, cleanupErr) + } + return handlerErr +} + // waitUntilPipelineBatchFlowDrained preserves the ownership boundary between // an internal receiver-driven StopSending and cancellation of the query or its // connection. StopSending aborts outstanding batch credits so this wait can @@ -462,6 +512,15 @@ func handlePipelineMessage(receiver *messageReceiverOnServer) (err error) { receiver.groupConcatReportingIncomplete = receiver.warningSession.incompleteGroupConcatReporting() } receiver.statementLastInsertID = runCompile.proc.GetStatementLastInsertID() + receiver.unpublishedS3CleanupContext = runCompile.proc.Ctx + if len(runCompile.scopes) != 0 && runCompile.proc.GetTxnOperator() != nil { + workspace := runCompile.proc.GetTxnOperator().GetWorkspace() + receiver.unpublishedS3CleanupWorkspace, _ = workspace.(interface { + CleanupUnpublishedS3Objects(context.Context) error + QueueUnpublishedS3Cleanup(*colexec.Server) error + }) + receiver.unpublishedS3ObjectOwners, _ = workspace.(colexec.UnpublishedS3ObjectOwnershipWorkspace) + } runCompile.clear() return nil })) @@ -849,9 +908,15 @@ type messageReceiverOnServer struct { needNotReply bool - requestedTeardownMode pipeline.StreamTeardownMode - acceptedTeardownMode pipeline.StreamTeardownMode - streamLifecycle *pipelineStreamLifecycle + requestedTeardownMode pipeline.StreamTeardownMode + acceptedTeardownMode pipeline.StreamTeardownMode + streamLifecycle *pipelineStreamLifecycle + unpublishedS3CleanupContext context.Context + unpublishedS3CleanupWorkspace interface { + CleanupUnpublishedS3Objects(context.Context) error + QueueUnpublishedS3Cleanup(*colexec.Server) error + } + unpublishedS3ObjectOwners colexec.UnpublishedS3ObjectOwnershipWorkspace colexecServer *colexec.Server diff --git a/pkg/sql/compile/remoterunServer_test.go b/pkg/sql/compile/remoterunServer_test.go index d1ac95c971a9a..325b875dbd03f 100644 --- a/pkg/sql/compile/remoterunServer_test.go +++ b/pkg/sql/compile/remoterunServer_test.go @@ -20,6 +20,7 @@ import ( "errors" "strings" "sync" + "sync/atomic" "testing" "time" @@ -37,6 +38,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/matrixorigin/matrixone/pkg/container/vector" "github.com/matrixorigin/matrixone/pkg/defines" + "github.com/matrixorigin/matrixone/pkg/fileservice" mock_frontend "github.com/matrixorigin/matrixone/pkg/frontend/test" mock_lock "github.com/matrixorigin/matrixone/pkg/frontend/test/mock_lock" "github.com/matrixorigin/matrixone/pkg/lockservice" @@ -51,6 +53,63 @@ import ( "github.com/matrixorigin/matrixone/pkg/vm/process" ) +type remoteS3CleanupWorkspace struct { + client.Workspace + cleanupErr error + calls int + pendingOwners bool + accepted bool + events []string + owners []*colexec.UnpublishedS3ObjectOwner + cleanupDeadline time.Time +} + +func (w *remoteS3CleanupWorkspace) QueueUnpublishedS3Cleanup(server *colexec.Server) error { + return server.RetryUnpublishedS3Cleanup(w.CleanupUnpublishedS3Objects) +} + +func (w *remoteS3CleanupWorkspace) CleanupUnpublishedS3Objects(ctx context.Context) error { + w.cleanupDeadline, _ = ctx.Deadline() + w.calls++ + w.events = append(w.events, "cleanup") + return w.cleanupErr +} + +func TestRemoteFinalizerPreservesPipelineCleanupBudget(t *testing.T) { + proc := testutil.NewProc(t) + defer proc.Free() + process.BeginPipelineCleanup(proc.Ctx) + deadline, ok := process.PipelineCleanupDeadline(proc.Ctx) + require.True(t, ok) + workspace := &remoteS3CleanupWorkspace{} + receiver := &messageReceiverOnServer{ + messageCtx: context.Background(), + unpublishedS3CleanupContext: proc.Ctx, + unpublishedS3CleanupWorkspace: workspace, + } + require.NoError(t, receiver.finalizeUnpublishedS3Objects(nil, nil)) + require.Equal(t, deadline, workspace.cleanupDeadline) +} + +func (w *remoteS3CleanupWorkspace) HasUnpublishedS3ObjectOwners() bool { + return w.pendingOwners +} + +func (w *remoteS3CleanupWorkspace) RetainUnpublishedS3ObjectOwner( + owner *colexec.UnpublishedS3ObjectOwner, +) { + w.pendingOwners = owner != nil && owner.Pending() + w.owners = append(w.owners, owner) +} + +func (w *remoteS3CleanupWorkspace) AcceptUnpublishedS3ObjectNames(...string) {} + +func (w *remoteS3CleanupWorkspace) AcceptAllUnpublishedS3ObjectOwners() { + w.accepted = true + w.pendingOwners = false + w.events = append(w.events, "accept") +} + func TestResolveRemoteCompileMPoolCap(t *testing.T) { const gib = uint64(1 << 30) @@ -71,6 +130,211 @@ func TestResolveRemoteCompileMPoolCap(t *testing.T) { require.Equal(t, int64(8*gib), cap) } +type remoteCleanupFailOnceFS struct { + fileservice.FileService + mu sync.Mutex + remaining int + err error + retryOnce sync.Once + retryStarted chan struct{} + retryRelease <-chan struct{} +} + +func (fs *remoteCleanupFailOnceFS) Delete(ctx context.Context, names ...string) error { + fs.mu.Lock() + if fs.remaining > 0 { + fs.remaining-- + fs.mu.Unlock() + return fs.err + } + fs.mu.Unlock() + if fs.retryStarted != nil { + fs.retryOnce.Do(func() { + close(fs.retryStarted) + <-fs.retryRelease + }) + } + return fs.FileService.Delete(ctx, names...) +} + +type remoteOwnerCleanupWorkspace struct { + owner *colexec.UnpublishedS3ObjectOwner + calls atomic.Int32 +} + +func (w *remoteOwnerCleanupWorkspace) QueueUnpublishedS3Cleanup(server *colexec.Server) error { + return server.RetryUnpublishedS3Cleanup(w.CleanupUnpublishedS3Objects) +} + +func (w *remoteOwnerCleanupWorkspace) CleanupUnpublishedS3Objects(ctx context.Context) error { + w.calls.Add(1) + return w.owner.Cleanup(ctx) +} + +func TestRemoteStreamExitRetriesFailedS3Cleanup(t *testing.T) { + proc := testutil.NewProcess(t) + defer proc.Free() + baseFS, err := colexec.GetSharedFSFromProc(proc) + require.NoError(t, err) + const objectName = "remote-stream-exit-retry-object" + require.NoError(t, baseFS.Write(proc.Ctx, fileservice.IOVector{ + FilePath: objectName, + Entries: []fileservice.IOEntry{{Size: 1, Data: []byte("x")}}, + Policy: fileservice.SkipAllCache, + })) + deleteErr := errors.New("temporary S3 delete failure") + retryStarted := make(chan struct{}) + retryRelease := make(chan struct{}) + var releaseOnce sync.Once + fs := &remoteCleanupFailOnceFS{ + FileService: baseFS, remaining: 1, err: deleteErr, + retryStarted: retryStarted, retryRelease: retryRelease, + } + owner, err := colexec.NewUnpublishedS3ObjectOwner(fs, objectName) + require.NoError(t, err) + workspace := &remoteOwnerCleanupWorkspace{owner: owner} + server := colexec.NewServer("") + t.Cleanup(func() { + releaseOnce.Do(func() { close(retryRelease) }) + require.NoError(t, server.CloseUnpublishedS3Cleanup(context.Background())) + }) + session := &lifecycleTestSession{ctx: context.Background()} + lifecycle, err := registerPipelineStreamLifecycle(session, 409, newPipelineBatchFlow(1, 1024)) + require.NoError(t, err) + defer lifecycle.remove() + receiver := &messageReceiverOnServer{ + messageCtx: context.Background(), + colexecServer: server, + unpublishedS3CleanupWorkspace: workspace, + } + handlerErr := moerr.NewDuplicateEntryNoCtx("value", "key") + err = receiver.finalizeUnpublishedS3Objects(handlerErr, lifecycle) + require.Same(t, handlerErr, err) + var rpcMessage pipeline.Message + rpcMessage.SetMoError(context.Background(), err) + decoded, ok := rpcMessage.TryToGetMoErr() + require.True(t, ok) + require.True(t, moerr.IsMoErrCode(decoded, moerr.ErrDuplicateEntry), "%v", decoded) + select { + case <-retryStarted: + case <-time.After(5 * time.Second): + t.Fatal("CN retry worker did not attempt the transferred cleanup") + } + require.Equal(t, int32(2), workspace.calls.Load()) + _, err = baseFS.StatFile(proc.Ctx, objectName) + require.NoError(t, err, "the first Delete failed and the retry is blocked") + + // The stream and its mirror transaction leave the handler while the CN + // retry holds the remaining owner. No test call retries cleanup directly. + lifecycle.remove() + _, registered := pipelineStreamLifecycles.Load(pipelineStreamLifecycleKey{session: session, id: 409}) + require.False(t, registered) + releaseOnce.Do(func() { close(retryRelease) }) + require.Eventually(t, func() bool { + _, statErr := baseFS.StatFile(proc.Ctx, objectName) + return moerr.IsMoErrCode(statErr, moerr.ErrFileNotFound) + }, 5*time.Second, 10*time.Millisecond) + require.GreaterOrEqual(t, workspace.calls.Load(), int32(2)) + require.False(t, owner.Pending()) +} + +func TestFinalizeRemoteS3OwnershipWaitsForBatchAcknowledgements(t *testing.T) { + t.Run("drained acknowledged stream transfers before cleanup", func(t *testing.T) { + workspace := &remoteS3CleanupWorkspace{pendingOwners: true} + flow := newPipelineBatchFlow(1, 1024) + seq, err := flow.reserve(context.Background(), context.Background(), 10) + require.NoError(t, err) + require.NoError(t, flow.acknowledgeOwnership(seq, true)) + receiver := messageReceiverOnServer{ + messageCtx: context.Background(), + unpublishedS3CleanupWorkspace: workspace, + unpublishedS3ObjectOwners: workspace, + } + + require.NoError(t, receiver.finalizeUnpublishedS3Objects(nil, &pipelineStreamLifecycle{ + batchFlow: flow, + })) + require.True(t, workspace.accepted) + require.Equal(t, []string{"accept", "cleanup"}, workspace.events) + }) + + t.Run("legacy stream cannot silently drop ownership", func(t *testing.T) { + workspace := &remoteS3CleanupWorkspace{pendingOwners: true} + receiver := messageReceiverOnServer{ + messageCtx: context.Background(), + unpublishedS3CleanupWorkspace: workspace, + unpublishedS3ObjectOwners: workspace, + } + + err := receiver.finalizeUnpublishedS3Objects(nil, nil) + require.ErrorContains(t, err, "ownership handoff requires batch acknowledgements") + require.False(t, workspace.accepted) + require.Equal(t, []string{"cleanup"}, workspace.events) + }) + + t.Run("failed handler keeps cleanup ownership", func(t *testing.T) { + workspace := &remoteS3CleanupWorkspace{pendingOwners: true} + receiver := messageReceiverOnServer{ + messageCtx: context.Background(), + unpublishedS3CleanupWorkspace: workspace, + unpublishedS3ObjectOwners: workspace, + } + handlerErr := errors.New("remote pipeline failed") + + err := receiver.finalizeUnpublishedS3Objects(handlerErr, nil) + require.ErrorIs(t, err, handlerErr) + require.False(t, workspace.accepted) + require.Equal(t, []string{"cleanup"}, workspace.events) + }) + + t.Run("successful cleanup preserves original SQL error", func(t *testing.T) { + workspace := &remoteS3CleanupWorkspace{} + receiver := messageReceiverOnServer{ + messageCtx: context.Background(), + unpublishedS3CleanupWorkspace: workspace, + unpublishedS3ObjectOwners: workspace, + } + handlerErr := moerr.NewDuplicateEntryNoCtx("value", "key") + + err := receiver.finalizeUnpublishedS3Objects(handlerErr, nil) + require.Same(t, handlerErr, err) + require.Same(t, handlerErr, moerr.ConvertGoError(context.Background(), err)) + require.Equal(t, 1, workspace.calls) + }) +} + +func TestFinalizeRemoteS3OwnershipRejectsUnprovenTakeover(t *testing.T) { + for _, scenario := range []string{"no-output", "empty", "pending", "stopped", "aborted", "old-coordinator"} { + t.Run(scenario, func(t *testing.T) { + workspace := &remoteS3CleanupWorkspace{pendingOwners: true} + flow := newPipelineBatchFlow(1, 1024) + if scenario != "empty" { + seq, err := flow.reserve(context.Background(), context.Background(), 10) + require.NoError(t, err) + if scenario != "pending" { + require.NoError(t, flow.acknowledgeOwnership(seq, scenario != "old-coordinator")) + } + } + if scenario == "stopped" { + flow.stop(context.Canceled) + } + if scenario == "aborted" { + flow.abort(context.Canceled) + } + receiver := messageReceiverOnServer{ + messageCtx: context.Background(), + needNotReply: scenario == "no-output", + unpublishedS3CleanupWorkspace: workspace, + unpublishedS3ObjectOwners: workspace, + } + err := receiver.finalizeUnpublishedS3Objects(nil, &pipelineStreamLifecycle{batchFlow: flow}) + require.ErrorContains(t, err, "ownership handoff requires batch acknowledgements") + require.False(t, workspace.accepted) + require.Equal(t, []string{"cleanup"}, workspace.events) + }) + } +} + // TestWorkspaceCreationInRemoteRun tests that workspace is created early in remote run scenario. // This is a critical test for the fix that prevents nil pointer panics. // diff --git a/pkg/sql/compile/remoterun_batch_flow.go b/pkg/sql/compile/remoterun_batch_flow.go index 9e8ac9a2b9c00..e033936e00d12 100644 --- a/pkg/sql/compile/remoterun_batch_flow.go +++ b/pkg/sql/compile/remoterun_batch_flow.go @@ -37,10 +37,12 @@ type pipelineBatchFlow struct { maxBytes uint64 nextSeq uint64 ackedSeq uint64 - bytes uint64 - pending map[uint64]uint64 - changed chan struct{} - abortErr error + // Contiguous per-batch ownership receipts, independent of cumulative credits. + ownershipAckedSeq uint64 + bytes uint64 + pending map[uint64]uint64 + changed chan struct{} + abortErr error // stoppedByReceiver distinguishes an internal StopSending from query or // connection cancellation at the terminal-response drain boundary. stoppedByReceiver bool @@ -165,6 +167,10 @@ func (f *pipelineBatchFlow) rollback(seq uint64) { } func (f *pipelineBatchFlow) acknowledge(seq uint64) error { + return f.acknowledgeOwnership(seq, false) +} + +func (f *pipelineBatchFlow) acknowledgeOwnership(seq uint64, retained bool) error { if f == nil || seq == 0 { return nil } @@ -180,6 +186,9 @@ func (f *pipelineBatchFlow) acknowledge(seq uint64) error { return moerr.NewInvalidStateNoCtxf( "pipeline batch ACK %d is ahead of sent sequence %d", seq, f.nextSeq) } + if retained && seq == f.ownershipAckedSeq+1 { + f.ownershipAckedSeq = seq + } for current := f.ackedSeq + 1; current <= seq; current++ { if size, ok := f.pending[current]; ok { delete(f.pending, current) @@ -254,7 +263,7 @@ func handlePipelineBatchAck(message *pipeline.Message, cs morpc.ClientSession) e _ = cs.Close() return moerr.NewInvalidStateNoCtx("pipeline batch ACK was not negotiated") } - if err := lifecycle.batchFlow.acknowledge(message.GetBatchAckSequence()); err != nil { + if err := lifecycle.batchFlow.acknowledgeOwnership(message.GetBatchAckSequence(), message.GetBatchAckS3OwnershipRetained()); err != nil { _ = cs.Close() return err } diff --git a/pkg/sql/compile/remoterun_batch_flow_test.go b/pkg/sql/compile/remoterun_batch_flow_test.go index f7d6a5f5b4e61..154a7aee33718 100644 --- a/pkg/sql/compile/remoterun_batch_flow_test.go +++ b/pkg/sql/compile/remoterun_batch_flow_test.go @@ -40,6 +40,47 @@ func TestPipelineBatchFlowNegotiation(t *testing.T) { require.Zero(t, bytes) } +func TestPipelineBatchFlowOwnershipReceipts(t *testing.T) { + for _, tc := range []struct { + name string + seqs []uint64 + marked []bool + want uint64 + }{ + {"marked success", []uint64{1, 2}, []bool{true, true}, 2}, + {"old coordinator", []uint64{1, 2}, []bool{false, false}, 0}, + {"mixed receipts", []uint64{1, 2}, []bool{false, true}, 0}, + {"cumulative gap", []uint64{2}, []bool{true}, 0}, + {"duplicate cannot upgrade", []uint64{1, 1, 2}, []bool{false, true, true}, 0}, + {"stale cannot repair gap", []uint64{2, 1}, []bool{true, true}, 0}, + {"duplicate marked", []uint64{1, 1, 2}, []bool{true, true, true}, 2}, + } { + t.Run(tc.name, func(t *testing.T) { + flow := newPipelineBatchFlow(8, 1024) + for i := 0; i < 2; i++ { + _, err := flow.reserve(context.Background(), context.Background(), 10) + require.NoError(t, err) + } + for i, seq := range tc.seqs { + require.NoError(t, flow.acknowledgeOwnership(seq, tc.marked[i])) + } + require.Equal(t, tc.want, flow.ownershipAckedSeq) + require.Equal(t, uint64(2), flow.ackedSeq) + require.Empty(t, flow.pending, "credit ACKs remain cumulative") + require.Zero(t, flow.bytes) + }) + } + flow := newPipelineBatchFlow(8, 1024) + _, err := flow.reserve(context.Background(), context.Background(), 10) + require.NoError(t, err) + require.NoError(t, flow.acknowledgeOwnership(0, true)) + require.Error(t, flow.acknowledgeOwnership(2, true)) + require.Zero(t, flow.ownershipAckedSeq) + flow.abort(context.Canceled) + require.NoError(t, flow.acknowledgeOwnership(1, true)) + require.Zero(t, flow.ownershipAckedSeq) +} + func TestPipelineBatchFlowBoundsAndReleasesCredits(t *testing.T) { flow := newPipelineBatchFlow(1, 10) seq, err := flow.reserve(context.Background(), context.Background(), 8) @@ -260,6 +301,20 @@ func TestHandlePipelineBatchAck(t *testing.T) { require.Zero(t, session.closeCalls) }) + t.Run("marked ack records ownership receipt", func(t *testing.T) { + session := &lifecycleTestSession{ctx: context.Background()} + lifecycle, err := registerPipelineStreamLifecycle(session, 305, newPipelineBatchFlow(1, 1024)) + require.NoError(t, err) + t.Cleanup(lifecycle.remove) + seq, err := lifecycle.batchFlow.reserve(context.Background(), context.Background(), 10) + require.NoError(t, err) + require.NoError(t, handlePipelineBatchAck(&pipeline.Message{ + Id: 305, BatchAckSequence: seq, BatchAckS3OwnershipRetained: true, + }, session)) + require.Equal(t, seq, lifecycle.batchFlow.ownershipAckedSeq) + require.Empty(t, lifecycle.batchFlow.pending) + }) + t.Run("ack without negotiation poisons the session", func(t *testing.T) { session := &lifecycleTestSession{ctx: context.Background()} lifecycle, err := registerPipelineStreamLifecycle(session, 302, nil) @@ -286,6 +341,7 @@ func TestHandlePipelineBatchAck(t *testing.T) { require.NoError(t, lifecycle.batchFlow.waitUntilDrained( context.Background(), context.Background(), nil)) require.Zero(t, session.closeCalls) + require.Zero(t, lifecycle.batchFlow.ownershipAckedSeq, "legacy ACK only releases credit") }) t.Run("ack ahead of sent data poisons the session", func(t *testing.T) { diff --git a/pkg/sql/compile/scope.go b/pkg/sql/compile/scope.go index e26a32f8c5066..e485aa500a9ad 100644 --- a/pkg/sql/compile/scope.go +++ b/pkg/sql/compile/scope.go @@ -419,6 +419,7 @@ func (s *Scope) Run(c *Compile) (err error) { } func (s *Scope) FreeOperator(c *Compile) { + process.BeginPipelineCleanup(c.proc.Ctx) for _, scope := range s.PreScopes { scope.FreeOperator(c) } diff --git a/pkg/util/metric/v2/metrics.go b/pkg/util/metric/v2/metrics.go index dbf47af54b817..168d5554e7ffa 100644 --- a/pkg/util/metric/v2/metrics.go +++ b/pkg/util/metric/v2/metrics.go @@ -61,6 +61,7 @@ func init() { initExecutionResourceMetrics() initHashBuildMetrics() initArrowLoadMetrics() + initUnpublishedS3Metrics() registry.MustRegister(HeartbeatHistogram) registry.MustRegister(HeartbeatFailureCounter) diff --git a/pkg/util/metric/v2/unpublished_s3.go b/pkg/util/metric/v2/unpublished_s3.go new file mode 100644 index 0000000000000..df5f30c793159 --- /dev/null +++ b/pkg/util/metric/v2/unpublished_s3.go @@ -0,0 +1,50 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v2 + +import "github.com/prometheus/client_golang/prometheus" + +// Service is a fixed CN identity. Object names and transaction IDs are never +// metric labels. +var ( + UnpublishedS3TicketsGauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: "mo", Subsystem: "unpublished_s3", Name: "tickets", + Help: "Current and peak CN cleanup tickets since process start.", + }, []string{"service", "state"}) + UnpublishedS3AdmissionFailuresCounter = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: "mo", Subsystem: "unpublished_s3", Name: "admission_failures_total", + Help: "S3 uploads or remote receipts rejected before ownership handoff because CN cleanup capacity was full.", + }, []string{"service"}) + UnpublishedS3PendingTasksGauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: "mo", Subsystem: "unpublished_s3", Name: "pending_cleanup_tasks", + Help: "CN retry callbacks still responsible for unpublished S3 objects.", + }, []string{"service"}) + UnpublishedS3OldestTaskAgeGauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: "mo", Subsystem: "unpublished_s3", Name: "oldest_cleanup_age_seconds", + Help: "Age of the oldest pending unpublished S3 cleanup callback.", + }, []string{"service"}) + UnpublishedS3DeleteFailuresCounter = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "mo", Subsystem: "unpublished_s3", Name: "delete_failures_total", + Help: "Failed deletion attempts for unpublished S3 objects in this process.", + }) +) + +func initUnpublishedS3Metrics() { + registry.MustRegister(UnpublishedS3TicketsGauge) + registry.MustRegister(UnpublishedS3AdmissionFailuresCounter) + registry.MustRegister(UnpublishedS3PendingTasksGauge) + registry.MustRegister(UnpublishedS3OldestTaskAgeGauge) + registry.MustRegister(UnpublishedS3DeleteFailuresCounter) +} diff --git a/pkg/vm/engine/disttae/prepared_cleanup_test.go b/pkg/vm/engine/disttae/prepared_cleanup_test.go new file mode 100644 index 0000000000000..d80c63aa29001 --- /dev/null +++ b/pkg/vm/engine/disttae/prepared_cleanup_test.go @@ -0,0 +1,141 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package disttae + +import ( + "context" + "errors" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/matrixorigin/matrixone/pkg/catalog" + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/container/batch" + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/container/vector" + "github.com/matrixorigin/matrixone/pkg/fileservice" + "github.com/matrixorigin/matrixone/pkg/objectio" + "github.com/matrixorigin/matrixone/pkg/pb/plan" + "github.com/matrixorigin/matrixone/pkg/sql/colexec" + "github.com/matrixorigin/matrixone/pkg/sql/colexec/insert" + "github.com/matrixorigin/matrixone/pkg/testutil" + "github.com/matrixorigin/matrixone/pkg/util/fault" + "github.com/matrixorigin/matrixone/pkg/vm" + "github.com/matrixorigin/matrixone/pkg/vm/pipeline" + "github.com/matrixorigin/matrixone/pkg/vm/process" + "github.com/stretchr/testify/require" +) + +type preparedCleanupFS struct { + fileservice.FileService + offline atomic.Bool + names []string +} + +func (fs *preparedCleanupFS) Write(ctx context.Context, v fileservice.IOVector) error { + if err := fs.FileService.Write(ctx, v); err != nil { + return err + } + fs.names = append(fs.names, v.FilePath) + return nil +} + +func (fs *preparedCleanupFS) Delete(ctx context.Context, names ...string) error { + if fs.offline.Load() { + return errors.New("injected storage outage") + } + return fs.FileService.Delete(ctx, names...) +} + +type failingPreparedInput struct{ *colexec.MockOperator } + +func (op *failingPreparedInput) Call(proc *process.Process) (vm.CallResult, error) { + r, err := op.MockOperator.Call(proc) + if err == nil && r.Batch == nil { + err = errors.New("input failed before ownership handoff") + } + return r, err +} + +func TestPreparedInsertRollbackRetriesCleanupWithoutAnotherExecute(t *testing.T) { + server := colexec.NewServer("") + t.Cleanup(func() { require.NoError(t, server.CloseUnpublishedS3Cleanup(context.Background())) }) + txn := newTransactionWithActivePKTableForTest(t, "pk") + proc := txn.proc + defer proc.Free() + proc.Base.TxnOperator = txn.op + proc.BuildPipelineContext(proc.Ctx) + baseFS, err := colexec.GetSharedFSFromProc(proc) + require.NoError(t, err) + fs := &preparedCleanupFS{FileService: baseFS} + fs.offline.Store(true) + t.Cleanup(func() { fs.offline.Store(false) }) + fss, err := fileservice.NewFileServices(baseFS.Name(), fs) + require.NoError(t, err) + proc.SetFileService(fss) + + // Exercise the real insert Call path, using the existing small-object fault + // to spill before the next input fails (no output/ownership handoff). + fault.Enable() + defer fault.Disable() + require.NoError(t, fault.AddFaultPoint(context.Background(), objectio.FJ_CNFlushSmallObjs, ":::", "echo", 0, "prepared_cleanup", false)) + defer func() { _, _ = fault.RemoveFaultPoint(context.Background(), objectio.FJ_CNFlushSmallObjs) }() + arg := insert.NewArgument() + arg.ToWriteS3 = true + arg.InsertCtx = &insert.InsertCtx{Attrs: []string{"a"}, TableDef: &plan.TableDef{ + DbName: "prepared_cleanup", Name: "t", + Pkey: &plan.PrimaryKeyDef{}, + Cols: []*plan.ColDef{ + {Name: "a", Seqnum: 0, Typ: plan.Type{Id: int32(types.T_varchar)}}, + {Name: catalog.Row_ID, Seqnum: 1, Typ: plan.Type{Id: int32(types.T_Rowid)}}, + }, + }} + bat := &batch.Batch{Attrs: []string{"a"}, Vecs: []*vector.Vector{ + testutil.MakeVarcharVector([]string{strings.Repeat("x", 2*colexec.FaultInjectedS3Threshold)}, nil, proc.Mp()), + }} + bat.SetRowCount(1) + input := &failingPreparedInput{colexec.NewMockOperator().WithBatchs([]*batch.Batch{bat})} + arg.AppendChild(input) + require.NoError(t, arg.Prepare(proc)) + _, err = arg.Call(proc) + require.ErrorContains(t, err, "input failed") + require.NotEmpty(t, fs.names, "the insert must actually spill before failing") + require.Empty(t, txn.unpublishedS3Cleanup) + pipeline.New(0, nil, arg).Cleanup(proc, true, true, err) + require.Len(t, txn.unpublishedS3Cleanup, 1, "Reset alone must transfer the failed writer") + require.NoError(t, txn.Rollback(context.Background())) + require.True(t, txn.removed) + require.Empty(t, txn.unpublishedS3Cleanup, "CN owns retry after rollback") + for _, name := range fs.names { + _, err := baseFS.StatFile(context.Background(), name) + require.NoError(t, err) + } + fs.offline.Store(false) + require.Eventually(t, func() bool { + for _, name := range fs.names { + _, err := baseFS.StatFile(context.Background(), name) + if !moerr.IsMoErrCode(err, moerr.ErrFileNotFound) { + return false + } + } + return true + }, 5*time.Second, 10*time.Millisecond) + // No further EXECUTE, Reset, Free or manual cleanup ran before recovery. + arg.Free(proc, true, nil) + input.Free(proc, true, nil) + arg.Release() +} diff --git a/pkg/vm/engine/disttae/table_meta_reader.go b/pkg/vm/engine/disttae/table_meta_reader.go index 4d802b9dcb77f..e999d3bdcf12b 100644 --- a/pkg/vm/engine/disttae/table_meta_reader.go +++ b/pkg/vm/engine/disttae/table_meta_reader.go @@ -16,6 +16,7 @@ package disttae import ( "context" + "errors" "fmt" "slices" @@ -53,6 +54,7 @@ const ( ) type TableMetaReader struct { + ctx context.Context table *txnTable fs fileservice.FileService snapshot types.TS @@ -61,6 +63,9 @@ type TableMetaReader struct { //tblDef *plan.TableDef pState *logtailreplay.PartitionState + pendingDataWriter *colexec.CNS3Writer + pendingTombstoneWriter *colexec.CNS3Writer + immutableOnly bool maxObjects int objectCount int @@ -75,6 +80,36 @@ func (r *TableMetaReader) GetTxnInfo() string { } func (r *TableMetaReader) Close() error { + return r.CloseWithCleanup(r.ctx) +} + +// CloseWithCleanup uses the current cleanup attempt's context. A retained +// reader may outlive the request that created it. +func (r *TableMetaReader) CloseWithCleanup(ctx context.Context) error { + var cleanupErrs []error + for _, writer := range []*colexec.CNS3Writer{ + r.pendingDataWriter, + r.pendingTombstoneWriter, + } { + if writer == nil { + continue + } + if err := writer.CloseWithCleanup(ctx, true); err != nil { + cleanupErrs = append(cleanupErrs, err) + } + } + if len(cleanupErrs) != 0 { + // Pending writers retain only file-service names after failed deletion. + // Release the reader's table/snapshot and request context now; the CN + // callback supplies a fresh context for the next cleanup attempt. + r.table = nil + r.pState = nil + r.ctx = context.Background() + r.state = endState + return errors.Join(cleanupErrs...) + } + r.pendingDataWriter = nil + r.pendingTombstoneWriter = nil //r.tblDef = nil r.table = nil r.pState = nil @@ -82,6 +117,28 @@ func (r *TableMetaReader) Close() error { return nil } +// AcceptDataObjects releases source-reader ownership after the destination +// transaction has accepted the data metadata. +func (r *TableMetaReader) AcceptDataObjects() error { + if r.pendingDataWriter == nil { + return nil + } + writer := r.pendingDataWriter + r.pendingDataWriter = nil + return writer.Close() +} + +// AcceptTombstoneObjects releases source-reader ownership after the +// destination transaction has accepted the tombstone metadata. +func (r *TableMetaReader) AcceptTombstoneObjects() error { + if r.pendingTombstoneWriter == nil { + return nil + } + writer := r.pendingTombstoneWriter + r.pendingTombstoneWriter = nil + return writer.Close() +} + func (r *TableMetaReader) addCloneSharedFile( stats *objectio.ObjectStats, source cloneObjectSource, @@ -157,6 +214,7 @@ func newTableMetaReader( } return &TableMetaReader{ + ctx: ctx, fs: fs, snapshot: snapshot, txnOffset: txnOffset, @@ -450,11 +508,11 @@ func (r *TableMetaReader) collectDataOfAObjsAndInMem( s3Writer = colexec.NewCNS3DataWriterForService( r.table.getTxn().proc.GetService(), mp, r.fs, r.table.tableDef, -1, false, ) + r.pendingDataWriter = s3Writer defer func() { if dataReader != nil { dataReader.Close() } - s3Writer.Close() }() source, err := NewLocalDataSource( @@ -517,13 +575,12 @@ func (r *TableMetaReader) collectTombstoneOfAObjsAndInMem( s3Writer = colexec.NewCNS3TombstoneWriterForService( r.table.getTxn().proc.GetService(), mp, r.fs, colTypes[1], -1, ) + r.pendingTombstoneWriter = s3Writer defer func() { if iter != nil { iter.Close() } - s3Writer.Close() - if tombstoneReader != nil { tombstoneReader.Close() } diff --git a/pkg/vm/engine/disttae/table_meta_reader_test.go b/pkg/vm/engine/disttae/table_meta_reader_test.go index 6c9059ba87c45..1af9b3db3588e 100644 --- a/pkg/vm/engine/disttae/table_meta_reader_test.go +++ b/pkg/vm/engine/disttae/table_meta_reader_test.go @@ -33,6 +33,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/testutil" "github.com/matrixorigin/matrixone/pkg/txn/client" "github.com/matrixorigin/matrixone/pkg/vm/engine/disttae/logtailreplay" + "github.com/matrixorigin/matrixone/pkg/vm/process" "github.com/panjf2000/ants/v2" "github.com/stretchr/testify/require" ) @@ -84,6 +85,118 @@ func TestTableMetaReaderRecordsCloneObjectOwnership(t *testing.T) { require.True(t, txn.engine.cloneTxnCache.IsTxnLocalSharedFile(txnID, stats[1].ObjectName().String())) } +func TestTableMetaReaderRetainsCloneObjectOwnershipUntilAccepted(t *testing.T) { + for _, tc := range []struct { + name string + accepted bool + }{ + {name: "unaccepted clone object is cleaned on close"}, + {name: "accepted clone object survives close", accepted: true}, + } { + t.Run(tc.name, func(t *testing.T) { + proc := testutil.NewProc(t) + defer proc.Free() + baseFS, err := colexec.GetSharedFSFromProc(proc) + require.NoError(t, err) + var fs fileservice.FileService = baseFS + var failingFS *failOnceCloneDeleteFS + if !tc.accepted { + failingFS = &failOnceCloneDeleteFS{ + FileService: baseFS, + failErr: errors.New("injected clone cleanup failure"), + } + fs = failingFS + } + writer, objectName := newCloneTombstoneObjectForTest(t, proc, fs) + reader := &TableMetaReader{ + ctx: proc.Ctx, + pendingTombstoneWriter: writer, + } + + if tc.accepted { + require.NoError(t, reader.AcceptTombstoneObjects()) + } + if tc.accepted { + require.NoError(t, reader.Close()) + _, err = baseFS.StatFile(proc.Ctx, objectName) + require.NoError(t, err, "accepted object must remain available to the destination transaction") + } else { + err = reader.Close() + require.ErrorIs(t, err, failingFS.failErr) + require.Same(t, writer, reader.pendingTombstoneWriter, "failed deletion must retain its writer for retry") + _, err = baseFS.StatFile(proc.Ctx, objectName) + require.NoError(t, err, "the first failed cleanup must not discard the object reference") + require.NoError(t, reader.Close()) + _, err = baseFS.StatFile(proc.Ctx, objectName) + require.True(t, moerr.IsMoErrCode(err, moerr.ErrFileNotFound), "unaccepted object should be deleted, got %v", err) + } + }) + } +} + +func TestTableMetaReaderCleanupUsesFreshAttemptContext(t *testing.T) { + proc := testutil.NewProc(t) + defer proc.Free() + baseFS, err := colexec.GetSharedFSFromProc(proc) + require.NoError(t, err) + writer, name := newCloneTombstoneObjectForTest(t, proc, baseFS) + expiredCtx, cancel := context.WithDeadline(context.Background(), time.Now().Add(-time.Second)) + defer cancel() + reader := &TableMetaReader{ctx: expiredCtx, pendingTombstoneWriter: writer} + + require.Error(t, reader.Close(), "the original request deadline prevents cleanup") + _, err = baseFS.StatFile(context.Background(), name) + require.NoError(t, err) + require.NoError(t, reader.CloseWithCleanup(context.Background())) + _, err = baseFS.StatFile(context.Background(), name) + require.True(t, moerr.IsMoErrCode(err, moerr.ErrFileNotFound)) +} + +type failOnceCloneDeleteFS struct { + fileservice.FileService + failErr error + failed bool + retryStarted chan struct{} + retryRelease <-chan struct{} +} + +func (fs *failOnceCloneDeleteFS) Delete(ctx context.Context, names ...string) error { + if !fs.failed { + fs.failed = true + return fs.failErr + } + if fs.retryStarted != nil { + close(fs.retryStarted) + <-fs.retryRelease + fs.retryStarted = nil + } + return fs.FileService.Delete(ctx, names...) +} + +func newCloneTombstoneObjectForTest( + t *testing.T, + proc *process.Process, + fs fileservice.FileService, +) (*colexec.CNS3Writer, string) { + t.Helper() + writer := colexec.NewCNS3TombstoneWriter(proc.Mp(), fs, types.T_int64.ToType(), 1) + bat := batch.NewWithSize(2) + bat.SetAttributes([]string{objectio.PhysicalAddr_Attr, "pk"}) + bat.Vecs[0] = vector.NewVec(types.T_Rowid.ToType()) + bat.Vecs[1] = vector.NewVec(types.T_int64.ToType()) + require.NoError(t, vector.AppendFixed(bat.Vecs[0], types.RandomRowid(), false, proc.Mp())) + require.NoError(t, vector.AppendFixed(bat.Vecs[1], int64(1), false, proc.Mp())) + bat.SetRowCount(1) + defer bat.Clean(proc.Mp()) + require.NoError(t, writer.Write(proc.Ctx, bat)) + blockInfo, err := writer.SyncAndFillBlockInfoBat(proc.Ctx) + require.NoError(t, err) + require.Equal(t, 1, blockInfo.RowCount()) + data, area := vector.MustVarlenaRawData(blockInfo.Vecs[0]) + stats := objectio.ObjectStats(data[0].GetByteSlice(area)) + return writer, stats.ObjectName().String() +} + func TestNewImmutableTableMetaReader(t *testing.T) { tbl := newTxnTableForTest() tbl.getTxn().proc = testutil.NewProcess(t) diff --git a/pkg/vm/engine/disttae/transfer.go b/pkg/vm/engine/disttae/transfer.go index 87ff75dc4311d..7cf103414f5c7 100644 --- a/pkg/vm/engine/disttae/transfer.go +++ b/pkg/vm/engine/disttae/transfer.go @@ -16,6 +16,7 @@ package disttae import ( "context" + "errors" "fmt" "strings" "time" @@ -101,7 +102,7 @@ func transferTombstoneObjects( return txn.forEachTableHasDeletesLocked( true, - func(tbl *txnTable) error { + func(tbl *txnTable) (retErr error) { now := time.Now() if flow, logs, err = ConstructCNTombstoneObjectsTransferFlow( ctx, start, end, tbl, txn, txn.proc.Mp(), fs); err != nil { @@ -111,8 +112,9 @@ func transferTombstoneObjects( return nil } + registered := false defer func() { - err = flow.Close() + retErr = txn.closeTransferFlowWithError(ctx, flow, !registered, retErr) }() if err = flow.Process(ctx); err != nil { @@ -124,6 +126,18 @@ func transferTombstoneObjects( logutil.Fatal("tombstone sinker tail size is not zero", zap.Int("tail", len(tail))) } + if len(slist) > 0 { + names := make([]string, 0, len(slist)) + for i := range slist { + names = append(names, slist[i].ObjectName().String()) + } + owner, ownerErr := colexec.NewUnpublishedS3ObjectOwnerForService(txn.engine.service, flow.fs, names...) + if ownerErr != nil { + return ownerErr + } + txn.RetainUnpublishedS3ObjectOwner(owner) + flow.ownerTransferred = true + } bat := colexec.AllocCNS3ResultBat(true) if err = bat.PreExtend(txn.proc.Mp(), len(slist)); err != nil { @@ -158,6 +172,7 @@ func transferTombstoneObjects( return err } } + registered = true logs = append(logs, zap.String("txn-id", txn.op.Txn().DebugString()), @@ -176,6 +191,23 @@ func transferTombstoneObjects( }) } +func (txn *Transaction) closeTransferFlowWithError( + ctx context.Context, + flow *TransferFlow, + failed bool, + originalErr error, +) error { + cleanupErr := txn.closeTransferFlow(ctx, flow, failed) + if cleanupErr == nil { + return originalErr + } + if originalErr != nil { + logutil.Warn("tombstone transfer cleanup retained by transaction", zap.Error(cleanupErr)) + return originalErr + } + return errors.Join(originalErr, cleanupErr) +} + func transferTombstones( ctx context.Context, table *txnTable, diff --git a/pkg/vm/engine/disttae/transfer_util.go b/pkg/vm/engine/disttae/transfer_util.go index 2cf1b2971ce44..3f5872a0c0303 100644 --- a/pkg/vm/engine/disttae/transfer_util.go +++ b/pkg/vm/engine/disttae/transfer_util.go @@ -16,6 +16,7 @@ package disttae import ( "context" + "errors" "github.com/matrixorigin/matrixone/pkg/common/mpool" "github.com/matrixorigin/matrixone/pkg/container/batch" @@ -25,6 +26,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/objectio" "github.com/matrixorigin/matrixone/pkg/objectio/ioutil" "github.com/matrixorigin/matrixone/pkg/objectio/mergeutil" + "github.com/matrixorigin/matrixone/pkg/sql/colexec" plan2 "github.com/matrixorigin/matrixone/pkg/sql/plan" "github.com/matrixorigin/matrixone/pkg/vm/engine" "github.com/matrixorigin/matrixone/pkg/vm/engine/readutil" @@ -199,6 +201,10 @@ type TransferFlow struct { buffer *containers.OneSchemaBatchBuffer staged *batch.Batch sinker *ioutil.Sinker + cleanupPending bool + ownerTransferred bool + pendingNames []string + serviceID string mp *mpool.MPool fs fileservice.FileService @@ -222,15 +228,22 @@ func (flow *TransferFlow) fillDefaults() { ) } if flow.sinker == nil { + opts := []ioutil.SinkerOption{ + ioutil.WithBuffer(flow.buffer, false), + ioutil.WithMemorySizeThreshold(mpool.MB * 16), + ioutil.WithTailSizeCap(0), + } + if flow.table != nil { + flow.serviceID = flow.table.getTxn().engine.service + opts = append(opts, colexec.UnpublishedS3AdmissionOption(flow.serviceID)) + } flow.sinker = ioutil.NewTombstoneSinker( flow.hiddenSelection, pkType, flow.mp, flow.fs, - ioutil.WithBuffer(flow.buffer, false), - ioutil.WithMemorySizeThreshold(mpool.MB*16), - ioutil.WithTailSizeCap(0), - //readutil.WithAllMergeSorted(), + opts..., + //readutil.WithAllMergeSorted(), ) } @@ -359,24 +372,75 @@ func (flow *TransferFlow) GetResult() ([]objectio.ObjectStats, []*batch.Batch) { } func (flow *TransferFlow) Close() error { - if flow.sourcer != nil { - flow.sourcer.Close() - flow.sourcer = nil + return flow.CloseWithCleanup(context.Background(), false) +} + +// CloseWithCleanup discards persisted objects if the flow failed before its +// result metadata was registered on the transaction. A failed delete retains +// only object names and the file service for the next attempt. +func (flow *TransferFlow) CloseWithCleanup(ctx context.Context, failed bool) error { + var ( + errs []error + ) + if (failed || flow.cleanupPending) && flow.sinker != nil && !flow.ownerTransferred { + if names, err := cleanupUnpublishedTransferSinker(ctx, flow.sinker); err != nil { + errs = append(errs, err) + flow.pendingNames = append(flow.pendingNames, names...) + flow.cleanupPending = true + } else { + flow.cleanupPending = false + } } - if flow.sinker != nil { - flow.sinker.Close() - flow.sinker = nil + if flow.sinker == nil && len(flow.pendingNames) > 0 { + cleanupCtx, cancel := colexec.UnpublishedS3CleanupContext(ctx) + completed, remaining, err := ioutil.DeleteUnpublishedObjectsWithProgress( + cleanupCtx, flow.fs, flow.pendingNames...) + cancel() + for _, name := range completed { + colexec.ReleaseUnpublishedS3Name(flow.serviceID, name) + } + flow.pendingNames = remaining + if err != nil { + errs = append(errs, err) + } else { + flow.cleanupPending = false + } } - if flow.buffer != nil { - flow.buffer.Close(flow.mp) - flow.buffer = nil + if flow.sourcer != nil { + if err := flow.sourcer.Close(); err != nil { + errs = append(errs, err) + } + flow.sourcer = nil } if flow.staged != nil { flow.staged.Clean(flow.mp) flow.staged = nil } - flow.mp = nil + if flow.sinker != nil { + if err := flow.sinker.Close(); err != nil { + errs = append(errs, err) + } + flow.sinker = nil + } + if flow.sinker == nil { + if flow.buffer != nil { + flow.buffer.Close(flow.mp) + flow.buffer = nil + } + flow.mp = nil + } flow.table = nil + flow.newDataObjects = nil + flow.isObjectDeletedFn = nil + if len(flow.pendingNames) == 0 { + flow.fs = nil + } flow.transferred.objDetails = nil - return nil + return errors.Join(errs...) +} + +func cleanupUnpublishedTransferSinker(ctx context.Context, sinker *ioutil.Sinker) ([]string, error) { + cleanupCtx, cancel := colexec.UnpublishedS3CleanupContext(ctx) + defer cancel() + return sinker.DeletePersisted(cleanupCtx) } diff --git a/pkg/vm/engine/disttae/transfer_util_test.go b/pkg/vm/engine/disttae/transfer_util_test.go index 4147585f1285c..287e12754e81a 100644 --- a/pkg/vm/engine/disttae/transfer_util_test.go +++ b/pkg/vm/engine/disttae/transfer_util_test.go @@ -16,16 +16,24 @@ package disttae import ( "context" + "errors" "fmt" "runtime" "testing" + "github.com/matrixorigin/matrixone/pkg/common/moerr" "github.com/matrixorigin/matrixone/pkg/common/mpool" "github.com/matrixorigin/matrixone/pkg/container/batch" "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/matrixorigin/matrixone/pkg/container/vector" + "github.com/matrixorigin/matrixone/pkg/defines" + "github.com/matrixorigin/matrixone/pkg/fileservice" "github.com/matrixorigin/matrixone/pkg/objectio" + "github.com/matrixorigin/matrixone/pkg/objectio/ioutil" + "github.com/matrixorigin/matrixone/pkg/testutil" "github.com/matrixorigin/matrixone/pkg/vm/engine/disttae/logtailreplay" + "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/containers" + "github.com/matrixorigin/matrixone/pkg/vm/process" "github.com/stretchr/testify/require" ) @@ -130,6 +138,182 @@ func TestTransferFlowBatchesAcrossObjectBlocks(t *testing.T) { require.Equal(t, rowCount*2, flow.transferred.rowCnt) } +func TestTransferFlowRetainsUnpublishedSinkerForCleanupRetry(t *testing.T) { + proc := testutil.NewProc(t) + defer proc.Free() + baseFS, err := fileservice.Get[fileservice.FileService]( + proc.Base.FileService, defines.SharedFileServiceName, + ) + require.NoError(t, err) + + deleteErr := errors.New("injected transfer cleanup failure") + fs := &failOnceTransferDeleteFS{FileService: baseFS, failErr: deleteErr, failCount: 2} + flow, objectName := newTransferFlowWithPersistedObject(t, proc, fs) + txn := &Transaction{} + + err = txn.closeTransferFlow(proc.Ctx, flow, true) + require.ErrorIs(t, err, deleteErr) + require.Nil(t, flow.sinker, "failed cleanup must release the sinker") + require.Nil(t, flow.buffer, "failed cleanup must release the flow buffer") + require.Equal(t, []string{objectName}, flow.pendingNames) + require.Len(t, txn.unpublishedS3Cleanup, 1, "transaction must retain the failed cleanup owner") + _, err = baseFS.StatFile(proc.Ctx, objectName) + require.NoError(t, err, "the object must remain available for cleanup retry") + require.ErrorIs(t, txn.closeTransferFlow(proc.Ctx, flow, false), deleteErr, + "a later success-shaped callback must still retry the abort cleanup") + require.Equal(t, []string{objectName}, flow.pendingNames) + + require.NoError(t, txn.CleanupUnpublishedS3Objects(proc.Ctx)) + require.Nil(t, flow.sinker) + require.Nil(t, flow.buffer) + _, err = baseFS.StatFile(proc.Ctx, objectName) + require.True(t, moerr.IsMoErrCode(err, moerr.ErrFileNotFound), "unpublished object should be deleted, got %v", err) +} + +type failSecondTransferDeleteBatchFS struct { + fileservice.FileService + batchSizes []int +} + +func (fs *failSecondTransferDeleteBatchFS) Delete(_ context.Context, names ...string) error { + fs.batchSizes = append(fs.batchSizes, len(names)) + if len(fs.batchSizes) == 2 { + return errors.New("second transfer batch unavailable") + } + return nil +} + +func TestTransferFlowRetryKeepsOnlyUnconfirmedDeleteBatch(t *testing.T) { + const objectCount = 1001 + names := make([]string, objectCount) + for i := range names { + names[i] = fmt.Sprintf("transfer-unpublished-%d", i) + } + fs := &failSecondTransferDeleteBatchFS{} + flow := &TransferFlow{fs: fs, pendingNames: names, cleanupPending: true} + + require.ErrorContains(t, flow.CloseWithCleanup(context.Background(), true), "second transfer batch unavailable") + require.Len(t, flow.pendingNames, 1) + require.Equal(t, []int{1000, 1}, fs.batchSizes) + require.Nil(t, flow.sinker) + require.Nil(t, flow.mp) + require.True(t, flow.cleanupPending) + + require.NoError(t, flow.CloseWithCleanup(context.Background(), true)) + require.Empty(t, flow.pendingNames) + require.Nil(t, flow.fs) + require.False(t, flow.cleanupPending) + require.Equal(t, []int{1000, 1, 1}, fs.batchSizes) +} + +func TestTransferFlowCleanupPreservesOriginalSQLError(t *testing.T) { + proc := testutil.NewProc(t) + defer proc.Free() + fs, err := fileservice.Get[fileservice.FileService]( + proc.Base.FileService, defines.SharedFileServiceName, + ) + require.NoError(t, err) + flow, objectName := newTransferFlowWithPersistedObject(t, proc, fs) + txn := &Transaction{} + originalErr := moerr.NewDuplicateEntryNoCtx("value", "key") + + err = txn.closeTransferFlowWithError(proc.Ctx, flow, true, originalErr) + require.Same(t, originalErr, err) + require.Same(t, originalErr, moerr.ConvertGoError(proc.Ctx, err)) + _, statErr := fs.StatFile(proc.Ctx, objectName) + require.True(t, moerr.IsMoErrCode(statErr, moerr.ErrFileNotFound), "%v", statErr) +} + +func TestTransferFlowFailedCleanupPreservesOriginalSQLError(t *testing.T) { + proc := testutil.NewProc(t) + defer proc.Free() + baseFS, err := fileservice.Get[fileservice.FileService]( + proc.Base.FileService, defines.SharedFileServiceName, + ) + require.NoError(t, err) + deleteErr := errors.New("temporary delete failure") + fs := &failOnceTransferDeleteFS{FileService: baseFS, failErr: deleteErr, failCount: 1} + flow, objectName := newTransferFlowWithPersistedObject(t, proc, fs) + txn := &Transaction{} + originalErr := moerr.NewDuplicateEntryNoCtx("value", "key") + + err = txn.closeTransferFlowWithError(proc.Ctx, flow, true, originalErr) + require.Same(t, originalErr, err) + require.Same(t, originalErr, moerr.ConvertGoError(proc.Ctx, err)) + require.Len(t, txn.unpublishedS3Cleanup, 1) + _, statErr := baseFS.StatFile(proc.Ctx, objectName) + require.NoError(t, statErr) + require.NoError(t, txn.CleanupUnpublishedS3Objects(proc.Ctx)) + _, statErr = baseFS.StatFile(proc.Ctx, objectName) + require.True(t, moerr.IsMoErrCode(statErr, moerr.ErrFileNotFound), "%v", statErr) +} + +func TestTransferFlowClosePreservesRegisteredObjects(t *testing.T) { + proc := testutil.NewProc(t) + defer proc.Free() + fs, err := fileservice.Get[fileservice.FileService]( + proc.Base.FileService, defines.SharedFileServiceName, + ) + require.NoError(t, err) + flow, objectName := newTransferFlowWithPersistedObject(t, proc, fs) + + require.NoError(t, flow.CloseWithCleanup(proc.Ctx, false)) + _, err = fs.StatFile(proc.Ctx, objectName) + require.NoError(t, err, "registered tombstone object must survive flow close") + require.NoError(t, fs.Delete(proc.Ctx, objectName)) +} + +func newTransferFlowWithPersistedObject( + t *testing.T, + proc *process.Process, + fs fileservice.FileService, +) (*TransferFlow, string) { + t.Helper() + pkType := types.T_int64.ToType() + attrs, attrTypes := objectio.GetTombstoneSchema(pkType, objectio.HiddenColumnSelection_None) + buffer := containers.NewOneSchemaBatchBuffer(mpool.MB*8, attrs, attrTypes, false) + sinker := ioutil.NewTombstoneSinker( + objectio.HiddenColumnSelection_None, + pkType, + proc.Mp(), + fs, + ioutil.WithBuffer(buffer, false), + ioutil.WithMemorySizeThreshold(mpool.MB*16), + ioutil.WithTailSizeCap(0), + ) + flow := &TransferFlow{buffer: buffer, sinker: sinker, mp: proc.Mp(), fs: fs} + bat := batch.NewWithSize(len(attrs)) + bat.SetAttributes(attrs) + for i := range attrTypes { + bat.Vecs[i] = vector.NewVec(attrTypes[i]) + } + rowID := types.RandomRowid() + require.NoError(t, vector.AppendFixed(bat.Vecs[0], rowID, false, proc.Mp())) + require.NoError(t, vector.AppendFixed(bat.Vecs[1], int64(1), false, proc.Mp())) + bat.SetRowCount(1) + require.NoError(t, sinker.Write(proc.Ctx, bat)) + bat.Clean(proc.Mp()) + require.NoError(t, sinker.Sync(proc.Ctx)) + stats, tail := sinker.GetResult() + require.Len(t, stats, 1) + require.Empty(t, tail) + return flow, stats[0].ObjectName().String() +} + +type failOnceTransferDeleteFS struct { + fileservice.FileService + failErr error + failCount int +} + +func (fs *failOnceTransferDeleteFS) Delete(ctx context.Context, names ...string) error { + if fs.failCount > 0 { + fs.failCount-- + return fs.failErr + } + return fs.FileService.Delete(ctx, names...) +} + func makeRepeated[T any](value T, count int) []T { values := make([]T, count) for i := range values { diff --git a/pkg/vm/engine/disttae/txn.go b/pkg/vm/engine/disttae/txn.go index 33747331ae7bb..4f9cc17c6f016 100644 --- a/pkg/vm/engine/disttae/txn.go +++ b/pkg/vm/engine/disttae/txn.go @@ -18,6 +18,7 @@ import ( "cmp" "context" "encoding/hex" + "errors" "fmt" "math" "runtime" @@ -139,6 +140,46 @@ func (txn *Transaction) unaccountWorkspaceEntryLocked(entry *Entry) { func (txn *Transaction) appendWorkspaceEntryLocked(entry Entry) { txn.accountWorkspaceEntryLocked(&entry) txn.writes = append(txn.writes, entry) + txn.unpublishedS3OwnersMu.Lock() + if len(txn.unpublishedS3ObjectOwnersByName) != 0 { + txn.acceptUnpublishedS3ObjectNamesLocked(unpublishedS3ObjectNames(entry)) + } + txn.unpublishedS3OwnersMu.Unlock() +} + +func unpublishedS3ObjectNames(entry Entry) []string { + if entry.bat == nil || len(entry.bat.Attrs) == 0 { + return nil + } + + var statsVec *vector.Vector + switch { + case entry.typ == INSERT && len(entry.bat.Attrs) > 1 && + entry.bat.Attrs[0] == catalog.BlockMeta_BlockInfo && + entry.bat.Attrs[1] == catalog.ObjectMeta_ObjectStats: + statsVec = entry.bat.Vecs[1] + case entry.typ == DELETE && entry.bat.Attrs[0] == catalog.ObjectMeta_ObjectStats: + statsVec = entry.bat.Vecs[0] + } + if statsVec == nil { + return nil + } + + names := make([]string, 0, statsVec.Length()) + seen := make(map[string]struct{}, statsVec.Length()) + for i := 0; i < statsVec.Length(); i++ { + stats := objectio.ObjectStats(statsVec.GetBytesAt(i)) + name := stats.ObjectName().String() + if name == "" { + continue + } + if _, ok := seen[name]; ok { + continue + } + seen[name] = struct{}{} + names = append(names, name) + } + return names } func (txn *Transaction) releaseWorkspaceEntryBatchLocked(idx int) { @@ -1140,7 +1181,10 @@ func (txn *Transaction) dumpInsertBatchLocked( defer func() { if s3Writer != nil { - s3Writer.Close() + if closeErr := s3Writer.CloseWithCleanup(ctx, true); closeErr != nil { + logutil.Warn("failed to clean unpublished insert object", zap.Error(closeErr)) + txn.retainUnpublishedS3Writer(s3Writer) + } } }() @@ -1191,9 +1235,11 @@ func (txn *Transaction) dumpInsertBatchLocked( return err } - s3Writer.Close() - + writer := s3Writer s3Writer = nil + if err = writer.Close(); err != nil { + return err + } } return nil @@ -1283,7 +1329,10 @@ func (txn *Transaction) dumpDeleteBatchLocked( defer func() { if s3Writer != nil { - s3Writer.Close() + if closeErr := s3Writer.CloseWithCleanup(ctx, true); closeErr != nil { + logutil.Warn("failed to clean unpublished delete object", zap.Error(closeErr)) + txn.retainUnpublishedS3Writer(s3Writer) + } } }() @@ -1335,7 +1384,9 @@ func (txn *Transaction) dumpDeleteBatchLocked( return err } - if err = s3Writer.Close(); err != nil { + writer := s3Writer + s3Writer = nil + if err = writer.Close(); err != nil { return err } } @@ -1343,6 +1394,211 @@ func (txn *Transaction) dumpDeleteBatchLocked( return nil } +func (txn *Transaction) retainUnpublishedS3Writer(writer *colexec.CNS3Writer) { + if writer == nil { + return + } + txn.RetainUnpublishedS3Cleanup(func(ctx context.Context) error { + return writer.CloseWithCleanup(ctx, true) + }) +} + +func (txn *Transaction) retainUnpublishedTransferFlow(flow *TransferFlow) { + if flow == nil || (flow.sinker == nil && len(flow.pendingNames) == 0) { + return + } + txn.RetainUnpublishedS3Cleanup(func(ctx context.Context) error { + return flow.CloseWithCleanup(ctx, true) + }) +} + +// RetainUnpublishedS3Cleanup transfers an unpublished object's retry owner to +// the transaction workspace before an execution-local owner is released. +func (txn *Transaction) RetainUnpublishedS3Cleanup(cleanup func(context.Context) error) { + if cleanup == nil { + return + } + txn.unpublishedS3OwnersMu.Lock() + txn.unpublishedS3Cleanup = append(txn.unpublishedS3Cleanup, cleanup) + txn.unpublishedS3OwnersMu.Unlock() +} + +// RetainUnpublishedS3ObjectOwner keeps the names of synced objects in the +// workspace until their corresponding entries are appended or the statement +// is aborted. Unlike a generic cleanup callback, this owner can be reduced as +// individual objects cross the registration boundary. +func (txn *Transaction) RetainUnpublishedS3ObjectOwner( + owner *colexec.UnpublishedS3ObjectOwner, +) { + if owner == nil || !owner.Pending() { + return + } + txn.unpublishedS3OwnersMu.Lock() + txn.retainUnpublishedS3ObjectOwnerLocked(owner) + txn.unpublishedS3OwnersMu.Unlock() +} + +func (txn *Transaction) retainUnpublishedS3ObjectOwnerLocked( + owner *colexec.UnpublishedS3ObjectOwner, +) { + if txn.unpublishedS3ObjectOwners == nil { + txn.unpublishedS3ObjectOwners = make(map[*colexec.UnpublishedS3ObjectOwner]struct{}) + } + if txn.unpublishedS3ObjectOwnersByName == nil { + txn.unpublishedS3ObjectOwnersByName = make(map[string]*colexec.UnpublishedS3ObjectOwner) + } + if _, exists := txn.unpublishedS3ObjectOwners[owner]; exists { + return + } + for _, name := range owner.Names() { + if _, exists := txn.unpublishedS3ObjectOwnersByName[name]; exists { + // The first owner remains responsible until this name is registered. + // Keeping a second owner would delete an accepted object on rollback. + owner.Accept(name) + continue + } + txn.unpublishedS3ObjectOwnersByName[name] = owner + } + if owner.Pending() { + txn.unpublishedS3ObjectOwners[owner] = struct{}{} + } +} + +func (txn *Transaction) AcceptUnpublishedS3ObjectNames(names ...string) { + txn.unpublishedS3OwnersMu.Lock() + defer txn.unpublishedS3OwnersMu.Unlock() + txn.acceptUnpublishedS3ObjectNamesLocked(names) +} + +func (txn *Transaction) acceptUnpublishedS3ObjectNamesLocked(names []string) { + for _, name := range names { + owner := txn.unpublishedS3ObjectOwnersByName[name] + if owner == nil { + continue + } + owner.Accept(name) + delete(txn.unpublishedS3ObjectOwnersByName, name) + if !owner.Pending() { + delete(txn.unpublishedS3ObjectOwners, owner) + } + } +} + +func (txn *Transaction) HasUnpublishedS3ObjectOwners() bool { + txn.unpublishedS3OwnersMu.Lock() + defer txn.unpublishedS3OwnersMu.Unlock() + return len(txn.unpublishedS3ObjectOwners) != 0 +} + +// AcceptAllUnpublishedS3ObjectOwners is used on a remote executor only after +// the coordinator has acknowledged receipt of every output batch and taken +// cleanup ownership into its own workspace. +func (txn *Transaction) AcceptAllUnpublishedS3ObjectOwners() { + txn.unpublishedS3OwnersMu.Lock() + for owner := range txn.unpublishedS3ObjectOwners { + owner.AcceptAll() + } + txn.unpublishedS3ObjectOwners = nil + txn.unpublishedS3ObjectOwnersByName = nil + txn.unpublishedS3OwnersMu.Unlock() +} + +func (txn *Transaction) closeTransferFlow( + ctx context.Context, + flow *TransferFlow, + failed bool, +) error { + err := flow.CloseWithCleanup(ctx, failed) + if flow.sinker != nil || len(flow.pendingNames) != 0 { + txn.retainUnpublishedTransferFlow(flow) + } + return err +} + +// QueueUnpublishedS3Cleanup transfers terminal cleanup records instead of +// retaining a bound transaction method. Fallback callbacks retain the resources +// they still need; name-only owners do not retain the transaction or process. +// Call only after execution has stopped; no further object registration occurs. +func (txn *Transaction) QueueUnpublishedS3Cleanup(server *colexec.Server) error { + txn.unpublishedS3OwnersMu.Lock() + pending := txn.unpublishedS3Cleanup + for owner := range txn.unpublishedS3ObjectOwners { + pending = append(pending, owner.Cleanup) + } + txn.unpublishedS3Cleanup = nil + txn.unpublishedS3ObjectOwners = nil + txn.unpublishedS3ObjectOwnersByName = nil + txn.unpublishedS3OwnersMu.Unlock() + if len(pending) == 0 { + return nil + } + cleanup := func(ctx context.Context) error { + failed := pending[:0] + var errs []error + for _, task := range pending { + if err := task(ctx); err != nil { + failed = append(failed, task) + errs = append(errs, err) + } + } + clear(pending[len(failed):]) + pending = failed + if len(pending) == 0 { + pending = nil + } + return errors.Join(errs...) + } + if err := server.RetryUnpublishedS3Cleanup(cleanup); err != nil { + txn.RetainUnpublishedS3Cleanup(cleanup) + return err + } + return nil +} + +// CleanupUnpublishedS3Objects retries cleanup for objects that never crossed +// the workspace-registration boundary. Detach the owners during I/O so no +// file-service call runs while holding the transaction mutex; failed cleanup +// keeps the exact owner on the transaction for the next lifecycle callback. +func (txn *Transaction) CleanupUnpublishedS3Objects(ctx context.Context) error { + txn.unpublishedS3OwnersMu.Lock() + cleanups := txn.unpublishedS3Cleanup + txn.unpublishedS3Cleanup = nil + owners := make([]*colexec.UnpublishedS3ObjectOwner, 0, len(txn.unpublishedS3ObjectOwners)) + for owner := range txn.unpublishedS3ObjectOwners { + owners = append(owners, owner) + } + txn.unpublishedS3ObjectOwners = nil + txn.unpublishedS3ObjectOwnersByName = nil + txn.unpublishedS3OwnersMu.Unlock() + + var ( + pending []func(context.Context) error + pendingOwners []*colexec.UnpublishedS3ObjectOwner + errs []error + ) + for _, cleanup := range cleanups { + if err := cleanup(ctx); err != nil { + pending = append(pending, cleanup) + errs = append(errs, err) + } + } + for _, owner := range owners { + if err := owner.Cleanup(ctx); err != nil { + pendingOwners = append(pendingOwners, owner) + errs = append(errs, err) + } + } + if len(pending) != 0 || len(pendingOwners) != 0 { + txn.unpublishedS3OwnersMu.Lock() + txn.unpublishedS3Cleanup = append(txn.unpublishedS3Cleanup, pending...) + for _, owner := range pendingOwners { + txn.retainUnpublishedS3ObjectOwnerLocked(owner) + } + txn.unpublishedS3OwnersMu.Unlock() + } + return errors.Join(errs...) +} + // resolveDumpTablesLocked resolves the engine.Relation of every table whose // workspace entries (of the given type, not yet flushed to a file) a dump is // about to flush. It must run BEFORE the dump mutates txn.writes: getTable @@ -2368,8 +2624,17 @@ func (txn *Transaction) compactDeletionOnObjsLocked(ctx context.Context) error { tbl.ensureSeqnumsAndTypesExpectRowid() locker.Unlock() - bat, fileName, err := tbl.rewriteObjectByDeletion(ctx, stats, objBlkDeletion[*objId]) + bat, fileName, s3Writer, err := tbl.rewriteObjectByDeletion(ctx, stats, objBlkDeletion[*objId]) if err != nil { + if bat != nil { + bat.Clean(txn.proc.Mp()) + } + if s3Writer != nil { + if cleanupErr := s3Writer.CloseWithCleanup(ctx, true); cleanupErr != nil { + logutil.Warn("failed to clean rewritten object", zap.Error(cleanupErr)) + txn.retainUnpublishedS3Writer(s3Writer) + } + } panicWhenFailed(err, "rewrite object by deletion failed") } @@ -2388,11 +2653,18 @@ func (txn *Transaction) compactDeletionOnObjsLocked(ctx context.Context) error { tbKey.autoIncrEpoch, tbKey.autoIncrEpochKnown, ); err != nil { + if cleanupErr := s3Writer.CloseWithCleanup(ctx, true); cleanupErr != nil { + logutil.Warn("failed to clean rewritten object after registration failure", zap.Error(cleanupErr)) + txn.retainUnpublishedS3Writer(s3Writer) + } bat.Clean(txn.proc.Mp()) panicWhenFailed(err, "write txn file failed") } bat.Clean(txn.proc.Mp()) + if closeErr := s3Writer.Close(); closeErr != nil { + panicWhenFailed(closeErr, "close rewritten object writer failed") + } } dirtyObject := make([]objectio.ObjectStats, 0, 1) @@ -2684,6 +2956,16 @@ func (txn *Transaction) getCachedTableByKey( } func (txn *Transaction) Commit(ctx context.Context) (reqs []txn.TxnRequest, err error) { + cleanupCtx, cancelCleanup := colexec.TerminalUnpublishedS3CleanupContext(ctx) + cleanupErr := txn.CleanupUnpublishedS3Objects(cleanupCtx) + cancelCleanup() + if cleanupErr != nil { + if queueErr := txn.QueueUnpublishedS3Cleanup(colexec.MustGetServer(txn.engine.service)); queueErr != nil { + return nil, errors.Join(cleanupErr, queueErr) + } + logutil.Warn("commit unpublished S3 cleanup transferred to CN retry worker", zap.Error(cleanupErr)) + } + common.DoIfDebugEnabled(func() { logutil.Debug( "Transaction.Commit", @@ -2958,6 +3240,18 @@ func skipTransfer(ctx context.Context, txn *Transaction) bool { } func (txn *Transaction) Rollback(ctx context.Context) error { + cleanupCtx, cancelCleanup := colexec.TerminalUnpublishedS3CleanupContext(ctx) + unpublishedCleanupErr := txn.CleanupUnpublishedS3Objects(cleanupCtx) + cancelCleanup() + if unpublishedCleanupErr != nil { + // The workspace is retired below even when rollback reports an error. + // Transfer its remaining cleanup obligation to the CN before that point. + queueErr := txn.QueueUnpublishedS3Cleanup(colexec.MustGetServer(txn.engine.service)) + if queueErr == nil { + logutil.Warn("rollback unpublished S3 cleanup transferred to CN retry worker", zap.Error(unpublishedCleanupErr)) + } + unpublishedCleanupErr = queueErr + } if !txn.ReadOnly() && len(txn.writes) > 0 { logutil.Info( "Transaction.Rollback", @@ -2979,7 +3273,7 @@ func (txn *Transaction) Rollback(ctx context.Context) error { } } txn.delTransaction() - return loadCleanupErr + return errors.Join(unpublishedCleanupErr, loadCleanupErr) } func (txn *Transaction) delTransaction() { diff --git a/pkg/vm/engine/disttae/txn_table.go b/pkg/vm/engine/disttae/txn_table.go index ec3452e168297..cf3ba7b9ba6cd 100644 --- a/pkg/vm/engine/disttae/txn_table.go +++ b/pkg/vm/engine/disttae/txn_table.go @@ -2111,7 +2111,7 @@ func (tbl *txnTable) rewriteObjectByDeletion( ctx context.Context, obj objectio.ObjectStats, blockDeletes map[objectio.Blockid][]int64, -) (*batch.Batch, string, error) { +) (*batch.Batch, string, *colexec.CNS3Writer, error) { proc := tbl.proc.Load() @@ -2121,15 +2121,13 @@ func (tbl *txnTable) rewriteObjectByDeletion( ) if fs, err = colexec.GetSharedFSFromProc(proc); err != nil { - return nil, "", err + return nil, "", nil, err } s3Writer := colexec.NewCNS3DataWriterForService( proc.GetService(), proc.Mp(), fs, tbl.tableDef, -1, false, ) - defer func() { s3Writer.Close() }() - var ( bat *batch.Batch stats []objectio.ObjectStats @@ -2180,15 +2178,15 @@ func (tbl *txnTable) rewriteObjectByDeletion( ) if err != nil { - return nil, fileName, err + return nil, fileName, s3Writer, err } if stats, err = s3Writer.Sync(ctx); err != nil { - return nil, fileName, err + return nil, fileName, s3Writer, err } if bat, err = s3Writer.FillBlockInfoBat(); err != nil { - return nil, fileName, err + return nil, fileName, s3Writer, err } if len(stats) != 0 { @@ -2196,8 +2194,7 @@ func (tbl *txnTable) rewriteObjectByDeletion( } ret, err := bat.Dup(proc.Mp()) - - return ret, fileName, err + return ret, fileName, s3Writer, err } func (tbl *txnTable) Delete( diff --git a/pkg/vm/engine/disttae/txn_test.go b/pkg/vm/engine/disttae/txn_test.go index 3365689491b92..2c6c8a1b3f9f7 100644 --- a/pkg/vm/engine/disttae/txn_test.go +++ b/pkg/vm/engine/disttae/txn_test.go @@ -30,6 +30,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/container/batch" "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/matrixorigin/matrixone/pkg/container/vector" + "github.com/matrixorigin/matrixone/pkg/fileservice" mock_frontend "github.com/matrixorigin/matrixone/pkg/frontend/test" "github.com/matrixorigin/matrixone/pkg/objectio" "github.com/matrixorigin/matrixone/pkg/pb/api" @@ -1157,6 +1158,281 @@ func TestIssue25589DumpInsertRestoresWorkspaceAccounting(t *testing.T) { } } +func TestIssue29257DumpRegistrationFailureDeletesUnpublishedObjects(t *testing.T) { + for _, tc := range []struct { + name string + typ int + }{ + {name: "insert", typ: INSERT}, + {name: "delete", typ: DELETE}, + } { + t.Run(tc.name, func(t *testing.T) { + colexec.NewServer("") + txn := newTransactionWithActivePKTableForTest(t, "pk") + defer txn.proc.Free() + txn.op = &unsupportedAutoIncrEpochTxnOperator{TxnOperator: txn.op} + tbl := txn.tableOps.existAndActive(genTableKey(1, "tbl", 7, "db")) + require.NotNil(t, tbl) + tbl.tableDef.Cols[0].Typ = pbplan.Type{Id: int32(types.T_int64)} + txn.tnStores = []DNStore{{}} + txn.batchSelectList = make(map[*batch.Batch][]int64) + txn.tablesInVain = make(map[uint64]int) + txn.deletedBlocks = &deletedBlocks{offsets: make(map[types.Blockid][]int64)} + txn.cnObjsSummary = make(map[types.Objectid]Summary) + txn.currentRowId.SetSegment(colexec.TxnWorkspaceSegment) + + var bat *batch.Batch + if tc.typ == INSERT { + bat = newInsertBatchWithRowIDForTest(t, txn.proc, []int64{1, 2, 3}) + } else { + bat = newDeleteBatchForTest(t, txn.proc, []int64{1, 2, 3}) + } + txn.appendWorkspaceEntryLocked(Entry{ + typ: tc.typ, + accountId: 1, + databaseId: 7, + tableId: 42, + databaseName: "db", + tableName: "tbl", + bat: bat, + autoIncrEpoch: 1, + autoIncrEpochKnown: true, + }) + + baseFS, err := colexec.GetSharedFSFromProc(txn.proc) + require.NoError(t, err) + fs := &recordingObjectFileService{ + FileService: baseFS, + failDeleteCount: 1, + deleteErr: moerr.NewInternalErrorNoCtx("injected cleanup failure"), + } + txn.Lock() + if tc.typ == INSERT { + var pkCount int + err = txn.dumpInsertBatchLocked(context.Background(), fs, 0, &pkCount) + } else { + err = txn.dumpDeleteBatchLocked(context.Background(), fs, 0) + } + txn.Unlock() + require.True(t, moerr.IsMoErrCode(err, moerr.ErrNotSupported), "registration should reject unsupported epoch fence: %v", err) + txn.unpublishedS3OwnersMu.Lock() + pendingWriterCount := len(txn.unpublishedS3Cleanup) + txn.unpublishedS3OwnersMu.Unlock() + require.Equal(t, 1, pendingWriterCount, "failed immediate cleanup must remain transaction-owned") + require.NoError(t, txn.CleanupUnpublishedS3Objects(context.Background())) + + written, deleted := fs.paths() + require.NotEmpty(t, written) + require.NotEmpty(t, deleted, "failed workspace registration must delete the uploaded object") + for _, name := range written { + _, statErr := baseFS.StatFile(context.Background(), name) + require.True(t, moerr.IsMoErrCode(statErr, moerr.ErrFileNotFound), "unpublished file %q leaked: %v", name, statErr) + } + }) + } +} + +func TestTransactionRetriesUnpublishedS3CleanupOwners(t *testing.T) { + txn := &Transaction{} + cleanupErr := moerr.NewInternalErrorNoCtx("injected cleanup failure") + attempts := 0 + txn.RetainUnpublishedS3Cleanup(func(context.Context) error { + attempts++ + if attempts == 1 { + return cleanupErr + } + return nil + }) + + require.ErrorIs(t, txn.CleanupUnpublishedS3Objects(context.Background()), cleanupErr) + require.Len(t, txn.unpublishedS3Cleanup, 1, "failed cleanup must preserve its retry owner") + require.NoError(t, txn.CleanupUnpublishedS3Objects(context.Background())) + require.Empty(t, txn.unpublishedS3Cleanup) + require.Equal(t, 2, attempts) +} + +func TestRollbackRetriesUnpublishedS3CleanupAfterWorkspaceRemoval(t *testing.T) { + server := colexec.NewServer("") + retryStarted := make(chan struct{}) + retryRelease := make(chan struct{}) + var releaseOnce sync.Once + t.Cleanup(func() { + releaseOnce.Do(func() { close(retryRelease) }) + require.NoError(t, server.CloseUnpublishedS3Cleanup(context.Background())) + }) + txn := newTransactionWithActivePKTableForTest(t, "pk") + defer txn.proc.Free() + baseFS, err := colexec.GetSharedFSFromProc(txn.proc) + require.NoError(t, err) + const name = "rollback-unpublished-retry-object" + require.NoError(t, baseFS.Write(context.Background(), fileservice.IOVector{ + FilePath: name, + Entries: []fileservice.IOEntry{{Size: 1, Data: []byte("x")}}, + Policy: fileservice.SkipAllCache, + })) + deleteErr := moerr.NewInternalErrorNoCtx("temporary S3 delete failure") + fs := &failOnceCloneDeleteFS{ + FileService: baseFS, failErr: deleteErr, + retryStarted: retryStarted, retryRelease: retryRelease, + } + owner, err := colexec.NewUnpublishedS3ObjectOwner(fs, name) + require.NoError(t, err) + txn.RetainUnpublishedS3ObjectOwner(owner) + + err = txn.Rollback(context.Background()) + require.NoError(t, err, "CN retry ownership should not turn rollback into a failure") + require.True(t, txn.removed, "terminal rollback must retire the workspace") + require.False(t, txn.HasUnpublishedS3ObjectOwners(), "the CN must own detached cleanup records") + require.Empty(t, txn.unpublishedS3Cleanup) + select { + case <-retryStarted: + case <-time.After(5 * time.Second): + t.Fatal("CN retry worker did not attempt the detached cleanup") + } + _, err = baseFS.StatFile(context.Background(), name) + require.NoError(t, err, "the first Delete failed and the retry is blocked") + releaseOnce.Do(func() { close(retryRelease) }) + require.Eventually(t, func() bool { + _, statErr := baseFS.StatFile(context.Background(), name) + return moerr.IsMoErrCode(statErr, moerr.ErrFileNotFound) + }, 5*time.Second, 10*time.Millisecond) + require.False(t, owner.Pending()) +} + +func TestQueueUnpublishedCleanupRetainsRecordsWhenCNRejectsHandoff(t *testing.T) { + server := &colexec.Server{} + require.NoError(t, server.CloseUnpublishedS3Cleanup(context.Background())) + txn := &Transaction{} + calls := 0 + txn.RetainUnpublishedS3Cleanup(func(context.Context) error { + calls++ + return nil + }) + require.Error(t, txn.QueueUnpublishedS3Cleanup(server)) + require.NoError(t, txn.CleanupUnpublishedS3Objects(context.Background())) + require.Equal(t, 1, calls, "rejected handoff must keep the cleanup obligation") +} + +func TestWorkspaceAppendAcceptsUnpublishedS3ObjectNames(t *testing.T) { + proc := testutil.NewProc(t) + defer proc.Free() + fs, err := colexec.GetSharedFSFromProc(proc) + require.NoError(t, err) + + stats := objectio.NewObjectStatsWithObjectID( + objectio.NewObjectidWithSegmentIDAndNum(objectio.NewSegmentid(), 7), + false, + false, + true, + ) + owner, err := colexec.NewUnpublishedS3ObjectOwner(fs, stats.ObjectName().String()) + require.NoError(t, err) + txn := &Transaction{} + txn.RetainUnpublishedS3ObjectOwner(owner) + require.True(t, txn.HasUnpublishedS3ObjectOwners()) + + bat := batch.NewWithSize(2) + bat.Attrs = []string{catalog.BlockMeta_BlockInfo, catalog.ObjectMeta_ObjectStats} + bat.Vecs[0] = vector.NewVec(types.T_text.ToType()) + bat.Vecs[1] = vector.NewVec(types.T_binary.ToType()) + require.NoError(t, vector.AppendBytes(bat.Vecs[1], stats.Marshal(), false, proc.Mp())) + bat.SetRowCount(1) + defer bat.Clean(proc.Mp()) + + // Producer completion alone is not acceptance. The workspace retires the + // owner only after the metadata-bearing Entry has actually been appended. + require.True(t, owner.Pending()) + txn.appendWorkspaceEntryLocked(Entry{ + typ: INSERT, + fileName: stats.ObjectLocation().String(), + bat: bat, + }) + require.Len(t, txn.writes, 1) + require.False(t, owner.Pending()) + require.False(t, txn.HasUnpublishedS3ObjectOwners()) + + deleteStats := objectio.NewObjectStatsWithObjectID( + objectio.NewObjectidWithSegmentIDAndNum(objectio.NewSegmentid(), 8), + false, + false, + true, + ) + deleteOwner, err := colexec.NewUnpublishedS3ObjectOwner(fs, deleteStats.ObjectName().String()) + require.NoError(t, err) + txn.RetainUnpublishedS3ObjectOwner(deleteOwner) + deleteBat := batch.NewWithSize(1) + deleteBat.Attrs = []string{catalog.ObjectMeta_ObjectStats} + deleteBat.Vecs[0] = vector.NewVec(types.T_binary.ToType()) + require.NoError(t, vector.AppendBytes(deleteBat.Vecs[0], deleteStats.Marshal(), false, proc.Mp())) + deleteBat.SetRowCount(1) + defer deleteBat.Clean(proc.Mp()) + txn.appendWorkspaceEntryLocked(Entry{ + typ: DELETE, + bat: deleteBat, + }) + require.False(t, deleteOwner.Pending(), "DELETE metadata must cross the same registration boundary") + require.NoError(t, txn.CleanupUnpublishedS3Objects(context.Background())) +} + +func TestRetainUnpublishedS3ObjectOwnerDeduplicatesNames(t *testing.T) { + proc := testutil.NewProc(t) + defer proc.Free() + fs, err := colexec.GetSharedFSFromProc(proc) + require.NoError(t, err) + + first, err := colexec.NewUnpublishedS3ObjectOwner(fs, "shared") + require.NoError(t, err) + second, err := colexec.NewUnpublishedS3ObjectOwner(fs, "shared", "other") + require.NoError(t, err) + txn := &Transaction{} + txn.RetainUnpublishedS3ObjectOwner(first) + txn.RetainUnpublishedS3ObjectOwner(second) + + require.Equal(t, first, txn.unpublishedS3ObjectOwnersByName["shared"]) + require.Equal(t, second, txn.unpublishedS3ObjectOwnersByName["other"]) + require.Equal(t, []string{"other"}, second.Names()) + txn.AcceptUnpublishedS3ObjectNames("shared") + require.False(t, first.Pending()) + require.True(t, second.Pending()) + txn.AcceptUnpublishedS3ObjectNames("other") + require.False(t, txn.HasUnpublishedS3ObjectOwners()) +} + +type recordingObjectFileService struct { + fileservice.FileService + mu sync.Mutex + written []string + deleted []string + failDeleteCount int + deleteErr error +} + +func (fs *recordingObjectFileService) Write(ctx context.Context, vector fileservice.IOVector) error { + fs.mu.Lock() + fs.written = append(fs.written, vector.FilePath) + fs.mu.Unlock() + return fs.FileService.Write(ctx, vector) +} + +func (fs *recordingObjectFileService) Delete(ctx context.Context, names ...string) error { + fs.mu.Lock() + fs.deleted = append(fs.deleted, names...) + if fs.failDeleteCount > 0 { + fs.failDeleteCount-- + err := fs.deleteErr + fs.mu.Unlock() + return err + } + fs.mu.Unlock() + return fs.FileService.Delete(ctx, names...) +} + +func (fs *recordingObjectFileService) paths() (written, deleted []string) { + fs.mu.Lock() + defer fs.mu.Unlock() + return append([]string(nil), fs.written...), append([]string(nil), fs.deleted...) +} + func TestIssue25589SoftDeleteObjectUsesWorkspaceAccounting(t *testing.T) { proc := testutil.NewProc(t) txn := &Transaction{ diff --git a/pkg/vm/engine/disttae/types.go b/pkg/vm/engine/disttae/types.go index b3538baf90278..8dfb05705d069 100644 --- a/pkg/vm/engine/disttae/types.go +++ b/pkg/vm/engine/disttae/types.go @@ -366,6 +366,13 @@ type Transaction struct { // writes cache stores any writes done by txn writes []Entry + // CN-created S3 owners whose cleanup failed and outlived their + // execution-local lifecycle. Retry them during statement/transaction + // teardown rather than dropping ownership. + unpublishedS3OwnersMu sync.Mutex + unpublishedS3Cleanup []func(context.Context) error + unpublishedS3ObjectOwners map[*colexec.UnpublishedS3ObjectOwner]struct{} + unpublishedS3ObjectOwnersByName map[string]*colexec.UnpublishedS3ObjectOwner // txn workspace size, includes in memory entries and persisted entries. workspaceSize uint64 // the approximation of total size for insert entries @@ -1194,6 +1201,7 @@ func (txn *Transaction) RollbackLastStatement(ctx context.Context) error { ) }) }() + unpublishedCleanupErr := txn.CleanupUnpublishedS3Objects(ctx) deletedLoadFiles, loadCleanupErr := txn.deleteLoadFiles(ctx, &txn.statementID) txn.Lock() @@ -1253,7 +1261,7 @@ func (txn *Transaction) RollbackLastStatement(ctx context.Context) error { // current statement has been rolled back, make can call IncrStatementID again. txn.incrStatementCalled = false txn.removeLoadFileProtectionsLocked(deletedLoadFiles) - return loadCleanupErr + return errors.Join(unpublishedCleanupErr, loadCleanupErr) } func (txn *Transaction) IncrSQLCount() { diff --git a/pkg/vm/pipeline/types.go b/pkg/vm/pipeline/types.go index 72f4d2f49ea6b..c06897cc95624 100644 --- a/pkg/vm/pipeline/types.go +++ b/pkg/vm/pipeline/types.go @@ -149,6 +149,7 @@ func (p *Pipeline) CleanRootOperator(proc *process.Process, pipelineFailed bool, if p.rootOp == nil { return } + process.BeginPipelineCleanup(proc.Ctx) p.rootOp.Reset(proc, pipelineFailed, err) cleanupDeferredSpool(p.rootOp) if !isPrepare { @@ -159,6 +160,7 @@ func (p *Pipeline) CleanRootOperator(proc *process.Process, pipelineFailed bool, // Cleanup do memory release work for whole pipeline. // we deliver the error because some operator may need to know what the error it is. func (p *Pipeline) Cleanup(proc *process.Process, pipelineFailed bool, isPrepare bool, err error) { + process.BeginPipelineCleanup(proc.Ctx) // cancel the context to stop its pre-pipelines. proc.Cancel(err) resetDone := make(map[vm.Operator]struct{}) diff --git a/pkg/vm/pipeline/types_test.go b/pkg/vm/pipeline/types_test.go index 945b01d54aaf9..835c6b1cd2365 100644 --- a/pkg/vm/pipeline/types_test.go +++ b/pkg/vm/pipeline/types_test.go @@ -16,17 +16,112 @@ package pipeline import ( "context" + "errors" "testing" "time" "github.com/matrixorigin/matrixone/pkg/common/mpool" + "github.com/matrixorigin/matrixone/pkg/container/batch" + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/container/vector" + "github.com/matrixorigin/matrixone/pkg/fileservice" + "github.com/matrixorigin/matrixone/pkg/pb/plan" "github.com/matrixorigin/matrixone/pkg/sql/colexec" "github.com/matrixorigin/matrixone/pkg/sql/colexec/connector" "github.com/matrixorigin/matrixone/pkg/sql/colexec/merge" + "github.com/matrixorigin/matrixone/pkg/testutil" "github.com/matrixorigin/matrixone/pkg/vm" "github.com/matrixorigin/matrixone/pkg/vm/process" + "github.com/stretchr/testify/require" ) +type deadlineDeleteFS struct { + fileservice.FileService + deadlines []time.Time + recovered bool + deadlineCeiling time.Time +} + +func (fs *deadlineDeleteFS) Delete(ctx context.Context, names ...string) error { + if fs.recovered { + return fs.FileService.Delete(ctx, names...) + } + d, _ := ctx.Deadline() + fs.deadlines = append(fs.deadlines, d) + // Fail quickly if a regression reintroduces a ten-minute writer budget. + if d.After(fs.deadlineCeiling) { + return errors.New("writer restarted cleanup budget") + } + <-ctx.Done() + return ctx.Err() +} + +type cleanupWritersOperator struct { + *colexec.MockOperator + writers []*colexec.CNS3Writer +} + +func (op *cleanupWritersOperator) Reset(proc *process.Process, _ bool, _ error) { + for _, writer := range op.writers { + _ = writer.ResetWithCleanup(proc.Ctx, true) + } +} + +func (op *cleanupWritersOperator) Free(proc *process.Process, _ bool, _ error) { + for _, writer := range op.writers { + _ = writer.CloseWithCleanup(proc.Ctx, true) + } +} + +func TestPipelineCleanupSharesBudgetAcrossWritersAndFree(t *testing.T) { + old := process.PipelineCleanupTimeout + process.PipelineCleanupTimeout = 40 * time.Millisecond + t.Cleanup(func() { process.PipelineCleanupTimeout = old }) + for _, rootOnly := range []bool{false, true} { + proc := testutil.NewProc(t) + proc.BuildPipelineContext(proc.Ctx) + _, hasDeadline := proc.Ctx.Deadline() + require.False(t, hasDeadline, "no request deadline may mask budget restarts") + baseFS, err := colexec.GetSharedFSFromProc(proc) + require.NoError(t, err) + fs := &deadlineDeleteFS{FileService: baseFS} + op := &cleanupWritersOperator{MockOperator: colexec.NewMockOperator()} + bat := &batch.Batch{Attrs: []string{"a"}, Vecs: []*vector.Vector{testutil.MakeInt64Vector([]int64{1}, nil, proc.Mp())}} + bat.SetRowCount(1) + for i := 0; i < 4; i++ { + writer := colexec.NewCNS3DataWriter(proc.Mp(), fs, &plan.TableDef{Pkey: &plan.PrimaryKeyDef{}, Cols: []*plan.ColDef{{Name: "a", Typ: plan.Type{Id: int32(types.T_int64)}}}}, 1, false) + require.NoError(t, writer.Write(proc.Ctx, bat)) + info, err := writer.SyncAndFillBlockInfoBat(proc.Ctx) + require.NoError(t, err) + info.Clean(proc.Mp()) + op.writers = append(op.writers, writer) + } + p := New(0, nil, op) + start := time.Now() + fs.deadlineCeiling = start.Add(time.Second) + if rootOnly { + p.CleanRootOperator(proc, true, false, context.Canceled) + } else { + p.Cleanup(proc, true, false, context.Canceled) + } + require.Less(t, time.Since(start), time.Second) + d, ok := process.PipelineCleanupDeadline(proc.Ctx) + require.True(t, ok) + require.NotEmpty(t, fs.deadlines) + for _, deadline := range fs.deadlines { + require.Equal(t, d, deadline, "Reset and Free must never restart the budget") + } + // Teardown leaves obligations intact. A new background attempt gets its + // own budget rather than inheriting the expired execution deadline. + fs.recovered = true + for _, writer := range op.writers { + require.NoError(t, writer.CloseWithCleanup(context.Background(), true)) + } + bat.Clean(proc.Mp()) + proc.Free() + } +} + type resetOrderOperator struct { *colexec.MockOperator name string diff --git a/pkg/vm/process/cleanup.go b/pkg/vm/process/cleanup.go new file mode 100644 index 0000000000000..1a871bfc7f422 --- /dev/null +++ b/pkg/vm/process/cleanup.go @@ -0,0 +1,57 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package process + +import ( + "context" + "sync" + "time" +) + +// PipelineCleanupTimeout bounds synchronous object cleanup across Reset, Free +// and remote finalization. Tests may shorten it before starting an execution. +var PipelineCleanupTimeout = 10 * time.Minute + +type cleanupBudgetKey struct{} +type cleanupBudget struct { + sync.Mutex + deadline time.Time +} + +func withCleanupBudget(ctx context.Context) context.Context { + return context.WithValue(ctx, cleanupBudgetKey{}, &cleanupBudget{}) +} + +// BeginPipelineCleanup starts the execution's shared budget once, before any +// operator teardown. It does not change execution cancellation. A new query +// context gives prepared executions a fresh budget; retries use a fresh context. +func BeginPipelineCleanup(ctx context.Context) { + if budget, ok := ctx.Value(cleanupBudgetKey{}).(*cleanupBudget); ok { + budget.Lock() + defer budget.Unlock() + if budget.deadline.IsZero() { + budget.deadline = time.Now().Add(PipelineCleanupTimeout) + } + } +} + +func PipelineCleanupDeadline(ctx context.Context) (time.Time, bool) { + if budget, ok := ctx.Value(cleanupBudgetKey{}).(*cleanupBudget); ok { + budget.Lock() + defer budget.Unlock() + return budget.deadline, !budget.deadline.IsZero() + } + return time.Time{}, false +} diff --git a/pkg/vm/process/process2.go b/pkg/vm/process/process2.go index 367b319b55f74..ae7b924113343 100644 --- a/pkg/vm/process/process2.go +++ b/pkg/vm/process/process2.go @@ -200,8 +200,8 @@ func (proc *Process) SaveToTopContext(key, value any) context.Context { // // Everyone should be careful to call this method. func (proc *Process) doPrepareForRunningWithoutPipeline() { - proc.Base.sqlContext.queryContext = proc.Base.sqlContext.outerContext - proc.Ctx = proc.Base.sqlContext.outerContext + proc.Base.sqlContext.queryContext = withCleanupBudget(proc.Base.sqlContext.outerContext) + proc.Ctx = proc.Base.sqlContext.queryContext } // GetQueryCtxFromProc returns the query context and its cancel function. @@ -307,7 +307,7 @@ func (qbCtx *QueryBaseContext) DoSpecialCleanUp(isMergeCTE bool) bool { // BuildQueryCtx refreshes the query context and cancellation method after the outer context was ready to run the query. func (qbCtx *QueryBaseContext) BuildQueryCtx(parent context.Context) context.Context { - qbCtx.queryContext, qbCtx.queryCancel = context.WithCancel(parent) + qbCtx.queryContext, qbCtx.queryCancel = context.WithCancel(withCleanupBudget(parent)) qbCtx.Lock() qbCtx.pipelineLoopBreak = false diff --git a/pkg/vm/process/process2_test.go b/pkg/vm/process/process2_test.go index 71f40e52ce0fd..7f41505cb9301 100644 --- a/pkg/vm/process/process2_test.go +++ b/pkg/vm/process/process2_test.go @@ -16,6 +16,7 @@ package process import ( "context" + "sync" "testing" "github.com/matrixorigin/matrixone/pkg/common/mpool" @@ -28,6 +29,34 @@ import ( "github.com/stretchr/testify/require" ) +func TestCleanupBudgetSurvivesCancellationButNotPreparedReuse(t *testing.T) { + proc := NewTopProcess(context.Background(), mpool.MustNewZero(), nil, nil, nil, nil, nil, nil, nil, nil, nil) + defer proc.Free() + ctx := proc.Base.GetContextBase().BuildQueryCtx(proc.GetTopContext()) + proc.BuildPipelineContext(ctx) + _, ok := PipelineCleanupDeadline(ctx) + require.False(t, ok, "execution time must not consume teardown time") + var wg sync.WaitGroup + for i := 0; i < 8; i++ { + wg.Add(1) + go func() { defer wg.Done(); BeginPipelineCleanup(proc.Ctx) }() + } + wg.Wait() + d, ok := PipelineCleanupDeadline(ctx) + require.True(t, ok) + proc.Cancel(context.Canceled) + BeginPipelineCleanup(proc.Ctx) + d2, ok := PipelineCleanupDeadline(context.WithoutCancel(proc.Ctx)) + require.True(t, ok) + require.Equal(t, d, d2) + proc.ResetQueryContext() + ctx = proc.Base.GetContextBase().BuildQueryCtx(proc.GetTopContext()) + _, ok = PipelineCleanupDeadline(ctx) + require.False(t, ok, "a later EXECUTE must not inherit the old deadline") + _, ok = PipelineCleanupDeadline(context.Background()) + require.False(t, ok, "CN retries use an independent attempt budget") +} + type childProcessSession struct{} func (*childProcessSession) GetTempTable(string, string) (string, bool) { return "", false } diff --git a/proto/pipeline.proto b/proto/pipeline.proto index bd5c1fdc6a738..a35fde9d96221 100644 --- a/proto/pipeline.proto +++ b/proto/pipeline.proto @@ -73,6 +73,9 @@ message Message { uint64 accepted_batch_credit_bytes = 16; uint64 batch_sequence = 17; uint64 batch_ack_sequence = 18; + // ACK only: the receiving workspace retained cleanup ownership for this + // batch's S3 objects. This receipt is not cumulative like batch credits. + bool batch_ack_s3_ownership_retained = 19; } message Connector {