diff --git a/.github/workflows/move-tables-tests.yml b/.github/workflows/move-tables-tests.yml new file mode 100644 index 000000000..e028dbb5a --- /dev/null +++ b/.github/workflows/move-tables-tests.yml @@ -0,0 +1,46 @@ +name: move-tables tests +permissions: + contents: read + +on: [pull_request] + +jobs: + docker-tests: + runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + image: + # - 'mysql/mysql-server:5.7.41' # metadata locks not supported by default? (https://github.com/github/gh-ost/actions/runs/27841216224/job/82401716601?pr=1714) + - 'mysql:8.0.41' + - 'mysql:8.4.3' + - 'percona/percona-server:8.0.41-32' + env: + TEST_MYSQL_IMAGE: ${{ matrix.image }} + + steps: + - uses: actions/checkout@v4 + + - name: Setup environment + run: script/docker-gh-ost-move-tables-tests up + + - name: Run tests + run: script/docker-gh-ost-move-tables-tests run + + - name: Set artifact name + if: failure() + run: | + ARTIFACT_NAME=$(echo "${{ matrix.image }}" | tr '/:' '-') + echo "ARTIFACT_NAME=test-move-tables-logs-${ARTIFACT_NAME}" >> $GITHUB_ENV + + - name: Upload test logs on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: ${{ env.ARTIFACT_NAME }} + path: /tmp/gh-ost-test.* + retention-days: 7 + + - name: Teardown environment + if: always() + run: script/docker-gh-ost-move-tables-tests down diff --git a/.gitignore b/.gitignore index 5b4676a24..bdc5b754d 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,7 @@ /.vendor/ .idea/ *.tmp + +tools/ +*__failpoint_binding__.go +*.go__failpoint_stash__ diff --git a/doc/hooks.md b/doc/hooks.md index 03ce26db0..d9d4ffa46 100644 --- a/doc/hooks.md +++ b/doc/hooks.md @@ -73,15 +73,20 @@ The following variables are available on all hooks: - `GH_OST_MIGRATED_HOST` - `GH_OST_INSPECTED_HOST` - `GH_OST_EXECUTING_HOST` +- `GH_OST_TARGET_HOST` - the target/applier hostname (in `--move-tables`, this is the target cluster host) - `GH_OST_HOOKS_HINT` - copy of `--hooks-hint` value - `GH_OST_HOOKS_HINT_OWNER` - copy of `--hooks-hint-owner` value - `GH_OST_HOOKS_HINT_TOKEN` - copy of `--hooks-hint-token` value - `GH_OST_DRY_RUN` - whether or not the `gh-ost` run is a dry run - `GH_OST_REVERT` - whether or not `gh-ost` is running in revert mode +- `GH_OST_MOVE_TABLES` - whether or not `gh-ost` is running in `--move-tables` mode +- `GH_OST_TARGET_DATABASE_NAME` - operation target database name (in `--move-tables`, this is the explicit target database) +- `GH_OST_TARGET_TABLE_NAME` - operation target table name (mode-dependent) -The following variable are available on particular hooks: +The following variables are available on particular hooks: - `GH_OST_INSTANT_DDL` is only available in `gh-ost-on-success`. The value is `true` if instant DDL was successful, and `false` if it was not. +- `GH_OST_DRAIN_GTID` is only available in `gh-ost-on-success` and only in `--move-tables` mode. It contains the source `@@gtid_executed` captured immediately after the source `RENAME TABLE` (cutover) and represents the drain target the applier waits for. - `GH_OST_COMMAND` is only available in `gh-ost-on-interactive-command` - `GH_OST_STATUS` is only available in `gh-ost-on-status` - `GH_OST_LAST_BATCH_COPY_ERROR` is only available in `gh-ost-on-batch-copy-retry` diff --git a/go.mod b/go.mod index af30097c3..52121f925 100644 --- a/go.mod +++ b/go.mod @@ -56,6 +56,7 @@ require ( github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect github.com/pingcap/errors v0.11.5-0.20260310054046-9c8b3586e4b2 // indirect + github.com/pingcap/failpoint v0.0.0-20260521055755-e7642935314f // indirect github.com/pingcap/log v1.1.1-0.20260227082333-572e590d08f1 // indirect github.com/pingcap/tidb/pkg/parser v0.0.0-20260504140133-511dba1dbe17 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect diff --git a/go.sum b/go.sum index 709d499e2..c4df33263 100644 --- a/go.sum +++ b/go.sum @@ -101,6 +101,8 @@ github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJw github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= github.com/pingcap/errors v0.11.5-0.20260310054046-9c8b3586e4b2 h1:cLgCk5mwDG9lDH+dPK8TmEliTjyGJwwKN0qevWAl8IY= github.com/pingcap/errors v0.11.5-0.20260310054046-9c8b3586e4b2/go.mod h1:ktAJCA9lxrHHjVyVl2pKJFvzBnq2eZbb+CUOjBRPlXo= +github.com/pingcap/failpoint v0.0.0-20260521055755-e7642935314f h1:cDo4qNgaQc2POMWTXjNrMA7yySdIF/d1AaW8kOA7qOs= +github.com/pingcap/failpoint v0.0.0-20260521055755-e7642935314f/go.mod h1:jimwlLpI/XtwQdlZML15HS+j4rirvwZM0GLY07wwgOo= github.com/pingcap/log v1.1.1-0.20260227082333-572e590d08f1 h1:A2bEfgSb7hLwR9mxDszgGKweF+xY9YoTDG+8RjdFjDE= github.com/pingcap/log v1.1.1-0.20260227082333-572e590d08f1/go.mod h1:pxfz2oJfAuhwrb3/rcLqD//GS/5gRP4gD022iP3cEO0= github.com/pingcap/tidb/pkg/parser v0.0.0-20260504140133-511dba1dbe17 h1:cfAVPis6GP6lxQgm1WGaNGi4rVXTB4KDvYf96LjqRCM= diff --git a/go/base/context.go b/go/base/context.go index b2ec8ff1e..dc012279c 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -7,10 +7,13 @@ package base import ( "context" + "crypto/sha256" + "encoding/hex" "fmt" "math" "os" "regexp" + "sort" "strings" "sync" "sync/atomic" @@ -22,9 +25,10 @@ import ( "github.com/github/gh-ost/go/metrics" "github.com/github/gh-ost/go/mysql" "github.com/github/gh-ost/go/sql" - "github.com/openark/golib/log" "github.com/go-ini/ini" + "github.com/openark/golib/log" + "github.com/pingcap/failpoint" ) // RowsEstimateMethod is the type of row number estimation @@ -75,6 +79,139 @@ func NewThrottleCheckResult(throttle bool, reason string, reasonHint ThrottleRea } } +// MoveTable holds the per-table runtime state for a single table within a +// move-tables run. In move-tables mode the surrounding plumbing (one binlog +// stream, one applier connection, one throttler, one hooks executor) stays +// singular, but every migrated table carries its own schema, unique key, +// iteration progress, and counters keyed by table name. +// +// The range/iteration fields are guarded by the per-table rangeMutex. The +// applier-wide "current applied source coordinates" mutex stays single — there +// is one applied stream feeding all tables. +type MoveTable struct { + // Identity. + SourceDatabaseName string + SourceTableName string + TargetDatabaseName string + TargetTableName string + + // CreateTableStatement is the captured `SHOW CREATE TABLE` from the source, + // used to (re)create the table on the target. + CreateTableStatement string + + // Schema, captured from the source (or from the target, on resume). In + // move-tables mode source and target schemas match, so shared columns are the + // original columns minus generated columns, which MySQL recomputes on the target. + OriginalTableColumns *sql.ColumnList + OriginalTableVirtualColumns *sql.ColumnList + OriginalTableUniqueKeys [](*sql.UniqueKey) + UniqueKey *sql.UniqueKey + SharedColumns *sql.ColumnList + MappedSharedColumns *sql.ColumnList + + // RowsEstimate is the estimated row count for this table. + RowsEstimate int64 + + // Iteration / range state. Guarded by rangeMutex (except Iteration, which is + // accessed atomically so status readers don't need the lock). + MigrationRangeMinValues *sql.ColumnValues + MigrationRangeMaxValues *sql.ColumnValues + MigrationIterationRangeMinValues *sql.ColumnValues + MigrationIterationRangeMaxValues *sql.ColumnValues + Iteration int64 + + // LastIterationRange* record the last successfully-copied chunk range, used + // for checkpointing. Guarded by rangeMutex. + LastIterationRangeMinValues *sql.ColumnValues + LastIterationRangeMaxValues *sql.ColumnValues + + // RowsCopied is the number of rows copied for this table (accessed atomically). + RowsCopied int64 + + // rowCopyComplete is set (1) once this table's row copy finishes. The + // on-row-copy-complete hook and the cutover only proceed once every table is + // complete. Accessed atomically. + rowCopyComplete int64 + + // rangeMutex guards this table's range/iteration fields. + rangeMutex sync.Mutex +} + +// GetIteration returns the table's current iteration counter. +func (mt *MoveTable) GetIteration() int64 { + return atomic.LoadInt64(&mt.Iteration) +} + +// IncrementIteration advances the table's iteration counter by one. +func (mt *MoveTable) IncrementIteration() { + atomic.AddInt64(&mt.Iteration, 1) +} + +// SetNextIterationRangeMinValues advances the iteration window: the next chunk's +// min becomes the previous chunk's max (or the table min for the first chunk). +func (mt *MoveTable) SetNextIterationRangeMinValues() { + mt.rangeMutex.Lock() + defer mt.rangeMutex.Unlock() + mt.MigrationIterationRangeMinValues = mt.MigrationIterationRangeMaxValues + if mt.MigrationIterationRangeMinValues == nil { + mt.MigrationIterationRangeMinValues = mt.MigrationRangeMinValues + } +} + +// IsRowCopyComplete reports whether this table has finished its row copy. +func (mt *MoveTable) IsRowCopyComplete() bool { + return atomic.LoadInt64(&mt.rowCopyComplete) > 0 +} + +// SetRowCopyComplete marks this table's row copy as finished. +func (mt *MoveTable) SetRowCopyComplete() { + atomic.StoreInt64(&mt.rowCopyComplete, 1) +} + +// RecordLastIterationRange stores the last successfully-copied chunk range for +// checkpointing. +func (mt *MoveTable) RecordLastIterationRange() { + mt.rangeMutex.Lock() + defer mt.rangeMutex.Unlock() + if mt.MigrationIterationRangeMinValues != nil && mt.MigrationIterationRangeMaxValues != nil { + mt.LastIterationRangeMinValues = mt.MigrationIterationRangeMinValues.Clone() + mt.LastIterationRangeMaxValues = mt.MigrationIterationRangeMaxValues.Clone() + } +} + +// GetLastIterationRange returns clones of the last successfully-copied chunk +// range for checkpointing. Either value may be nil if no chunk has completed. +func (mt *MoveTable) GetLastIterationRange() (minValues, maxValues *sql.ColumnValues) { + mt.rangeMutex.Lock() + defer mt.rangeMutex.Unlock() + if mt.LastIterationRangeMinValues != nil { + minValues = mt.LastIterationRangeMinValues.Clone() + } + if mt.LastIterationRangeMaxValues != nil { + maxValues = mt.LastIterationRangeMaxValues.Clone() + } + return minValues, maxValues +} + +// GetRowsCopied returns the number of rows copied for this table. +func (mt *MoveTable) GetRowsCopied() int64 { + return atomic.LoadInt64(&mt.RowsCopied) +} + +// RestoreFromCheckpoint rehydrates this table's row-copy state from a resumed +// checkpoint: the next chunk starts at the last-copied range, and the iteration +// counter and rows-copied total are restored. +func (mt *MoveTable) RestoreFromCheckpoint(rangeMin, rangeMax *sql.ColumnValues, iteration, rowsCopied int64) { + mt.rangeMutex.Lock() + mt.MigrationIterationRangeMinValues = rangeMin + mt.MigrationIterationRangeMaxValues = rangeMax + mt.LastIterationRangeMinValues = rangeMin + mt.LastIterationRangeMaxValues = rangeMax + mt.rangeMutex.Unlock() + atomic.StoreInt64(&mt.Iteration, iteration) + atomic.StoreInt64(&mt.RowsCopied, rowsCopied) +} + // MigrationContext has the general, global state of migration. It is used by // all components throughout the migration process. type MigrationContext struct { @@ -177,31 +314,38 @@ type MigrationContext struct { CutOverType CutOver ReplicaServerId uint - Hostname string - AssumeMasterHostname string - ApplierTimeZone string - ApplierWaitTimeout int64 - TableEngine string - RowsEstimate int64 - RowsDeltaEstimate int64 - UsedRowsEstimateMethod RowsEstimateMethod - HasSuperPrivilege bool - OriginalBinlogFormat string - OriginalBinlogRowImage string - InspectorConnectionConfig *mysql.ConnectionConfig - InspectorMySQLVersion string - ApplierConnectionConfig *mysql.ConnectionConfig - ApplierMySQLVersion string - StartTime time.Time - RowCopyStartTime time.Time - RowCopyEndTime time.Time - LockTablesStartTime time.Time - RenameTablesStartTime time.Time - RenameTablesEndTime time.Time - pointOfInterestTime time.Time - pointOfInterestTimeMutex *sync.Mutex - lastHeartbeatOnChangelogTime time.Time - lastHeartbeatOnChangelogMutex *sync.Mutex + Hostname string + AssumeMasterHostname string + ApplierTimeZone string + ApplierWaitTimeout int64 + TableEngine string + RowsEstimate int64 + RowsDeltaEstimate int64 + UsedRowsEstimateMethod RowsEstimateMethod + HasSuperPrivilege bool + OriginalBinlogFormat string + OriginalBinlogRowImage string + InspectorConnectionConfig *mysql.ConnectionConfig + InspectorMySQLVersion string + ApplierConnectionConfig *mysql.ConnectionConfig + ApplierMySQLVersion string + StartTime time.Time + RowCopyStartTime time.Time + RowCopyEndTime time.Time + LockTablesStartTime time.Time + RenameTablesStartTime time.Time + RenameTablesEndTime time.Time + pointOfInterestTime time.Time + pointOfInterestTimeMutex *sync.Mutex + lastHeartbeatOnChangelogTime time.Time + lastHeartbeatOnChangelogMutex *sync.Mutex + // lastAppliedBinlogEventTime is the binlog-header timestamp of the last event + // applied to the target in move-tables mode. lastBinlogEventStreamedTime is the + // wall-clock time the streamer last delivered an event for the moved table. + // Together they drive the move-tables writer-lag metric (see GetBinlogWriterLag). + lastAppliedBinlogEventTime time.Time + lastBinlogEventStreamedTime time.Time + binlogWriterLagMutex *sync.Mutex CurrentLag int64 currentProgress uint64 etaNanoseonds int64 @@ -226,6 +370,7 @@ type MigrationContext struct { UserCommandedUnpostponeFlag int64 CutOverCompleteFlag int64 InCutOverCriticalSectionFlag int64 + MoveTablesSourceRenamedFlag int64 PanicAbort chan error // Context for cancellation signaling across all goroutines @@ -280,6 +425,41 @@ type MigrationContext struct { SkipMetadataLockCheck bool IsOpenMetadataLockInstruments bool + // move tables: + MoveTables struct { + TableNames []string // Ordered list of table names to be moved (order from --move-tables). Iteration is deterministic over this slice, never over the Tables map. + // Tables holds the per-table runtime state, keyed by source table name. + // Populated by InitMoveTableContainers() once per-table schema is known. + Tables map[string]*MoveTable + TargetHost string // Target hostname for the move. This must be a primary/writable host. + TargetPort int // Target MySQL port for the move. + TargetUser string // Target username for the move. If not specified, it will default to the source user. + TargetPass string // Target password for the move. If not specified, it will default to the source password. + TargetDatabase string // Target database name for the move. If not specified, it will default to the source database name. + + // AllowOnSourcePrimary opts in to running the move-tables read path (schema + // inspection, the full row copy, binlog streaming) directly against the + // source cluster's primary. By default gh-ost stops early when --host is the + // primary, since reading the whole table copy from the primary is the load + // move-tables is meant to avoid; the operator should point --host at a replica. + AllowOnSourcePrimary bool + + ConnectionConfig *mysql.ConnectionConfig + + // SourcePrimaryConnectionConfig is the detected source-cluster primary. All + // source reads (schema inspection, row copy, binlog streaming) go through the + // inspector config (InspectorConnectionConfig), which may point at a read + // replica to take load off the primary. The cutover RENAME + drain-GTID + // capture and the source `__del` DROP must run on a writable primary, so they + // use this dedicated config. When the source --host is itself the primary (no + // replica topology), detection returns the inspector key and the two coincide. + SourcePrimaryConnectionConfig *mysql.ConnectionConfig + + DrainGTID mysql.BinlogCoordinates // Source @@gtid_executed captured immediately after the source RENAME TABLE; the applier drains until it reaches this coordinate (move-tables only). + } + + UnsafeFailPointsEnabled bool + Log Logger } @@ -333,6 +513,7 @@ func NewMigrationContext() *MigrationContext { configMutex: &sync.Mutex{}, pointOfInterestTimeMutex: &sync.Mutex{}, lastHeartbeatOnChangelogMutex: &sync.Mutex{}, + binlogWriterLagMutex: &sync.Mutex{}, ColumnRenameMap: make(map[string]string), PanicAbort: make(chan error), ctx: ctx, @@ -352,6 +533,9 @@ func (mctx *MigrationContext) SetConnectionConfig(storageEngine string) error { } mctx.InspectorConnectionConfig.TransactionIsolation = transactionIsolation mctx.ApplierConnectionConfig.TransactionIsolation = transactionIsolation + if mctx.MoveTables.ConnectionConfig != nil { + mctx.MoveTables.ConnectionConfig.TransactionIsolation = transactionIsolation + } return nil } @@ -362,6 +546,9 @@ func (mctx *MigrationContext) SetConnectionCharset(charset string) { mctx.InspectorConnectionConfig.Charset = charset mctx.ApplierConnectionConfig.Charset = charset + if mctx.MoveTables.ConnectionConfig != nil { + mctx.MoveTables.ConnectionConfig.Charset = charset + } } func getSafeTableName(baseName string, suffix string) string { @@ -376,6 +563,9 @@ func getSafeTableName(baseName string, suffix string) string { // GetGhostTableName generates the name of ghost table, based on original table name // or a given table name func (mctx *MigrationContext) GetGhostTableName() string { + if mctx.IsMoveTablesMode() { + panic("GetGhostTableName() must not be called in move-tables mode; there is no ghost table (the target keeps each migrated table's name)") + } if mctx.Revert { // When reverting the "ghost" table is the _del table from the original migration. return mctx.OldTableName @@ -387,8 +577,20 @@ func (mctx *MigrationContext) GetGhostTableName() string { } } +// GetTargetDatabaseName fetches the name of the target database, which defaults to the original +// database name unless we're in move-tables mode. +func (mctx *MigrationContext) GetTargetDatabaseName() string { + if mctx.IsMoveTablesMode() { + return mctx.MoveTables.TargetDatabase + } + return mctx.DatabaseName +} + // GetOldTableName generates the name of the "old" table, into which the original table is renamed. func (mctx *MigrationContext) GetOldTableName() string { + if mctx.IsMoveTablesMode() { + panic("GetOldTableName() must not be called in move-tables mode; use MoveTableDelName(tableName) for each migrated table's `__del` rollback handle") + } var tableName string if mctx.ForceTmpTableName != "" { tableName = mctx.ForceTmpTableName @@ -410,9 +612,29 @@ func (mctx *MigrationContext) GetOldTableName() string { return getSafeTableName(tableName, suffix) } +// MoveTableDelName returns the `_
_del` rollback-handle table name for a +// specific migrated table in move-tables mode. It mirrors GetOldTableName but +// for an explicit table name, so a multi-table cutover can rename every source +// table in one atomic RENAME. Revert is disallowed in move-tables mode, so the +// suffix is always "del". +func (mctx *MigrationContext) MoveTableDelName(tableName string) string { + suffix := "del" + if mctx.TimestampOldTable { + t := mctx.StartTime + timestamp := fmt.Sprintf("%d%02d%02d%02d%02d%02d", + t.Year(), t.Month(), t.Day(), + t.Hour(), t.Minute(), t.Second()) + return getSafeTableName(tableName, fmt.Sprintf("%s_%s", timestamp, suffix)) + } + return getSafeTableName(tableName, suffix) +} + // GetChangelogTableName generates the name of changelog table, based on original table name // or a given table name. func (mctx *MigrationContext) GetChangelogTableName() string { + if mctx.IsMoveTablesMode() { + panic("GetChangelogTableName() must not be called in move-tables mode; there is no changelog table (§1.2)") + } if mctx.ForceTmpTableName != "" { return getSafeTableName(mctx.ForceTmpTableName, "ghc") } else { @@ -424,15 +646,13 @@ func (mctx *MigrationContext) GetChangelogTableName() string { func (mctx *MigrationContext) GetCheckpointTableName() string { if mctx.ForceTmpTableName != "" { return getSafeTableName(mctx.ForceTmpTableName, "ghk") - } else { - return getSafeTableName(mctx.OriginalTableName, "ghk") } -} - -// GetVoluntaryLockName returns a name of a voluntary lock to be used throughout -// the swap-tables process. -func (mctx *MigrationContext) GetVoluntaryLockName() string { - return fmt.Sprintf("%s.%s.lock", mctx.DatabaseName, mctx.OriginalTableName) + if mctx.IsMoveTablesMode() { + // One checkpoint table per run, named from the set-derived run token so it + // does not depend on any single migrated table and is stable across resume. + return getSafeTableName("gho_"+mctx.MoveTablesRunToken(), "ghk") + } + return getSafeTableName(mctx.OriginalTableName, "ghk") } // RequiresBinlogFormatChange is `true` when the original binlog format isn't `ROW` @@ -462,6 +682,18 @@ func (mctx *MigrationContext) GetInspectorHostname() string { return mctx.InspectorConnectionConfig.ImpliedKey.Hostname } +// GetTargetHostname is a safe access method to the target hostname. +// In move-tables mode, this is the hostname of the target database, +// otherwise it's the same as the applier hostname. +func (mctx *MigrationContext) GetTargetHostname() string { + if mctx.IsMoveTablesMode() && + mctx.MoveTables.ConnectionConfig != nil && + mctx.MoveTables.ConnectionConfig.ImpliedKey != nil { + return mctx.MoveTables.ConnectionConfig.ImpliedKey.Hostname + } + return mctx.GetApplierHostname() +} + // InspectorIsAlsoApplier is `true` when the both inspector and applier are the // same database instance. This would be true when running directly on master or when // testing on replica. @@ -479,8 +711,12 @@ func (mctx *MigrationContext) SetCutOverLockTimeoutSeconds(timeoutSeconds int64) if timeoutSeconds < 1 { return fmt.Errorf("minimal timeout is 1sec. Timeout remains at %d", mctx.CutOverLockTimeoutSeconds) } - if timeoutSeconds > 10 { - return fmt.Errorf("maximal timeout is 10sec. Timeout remains at %d", mctx.CutOverLockTimeoutSeconds) + maxTimeout := int64(10) + if mctx.IsMoveTablesMode() { + maxTimeout = 60 + } + if timeoutSeconds > maxTimeout { + return fmt.Errorf("maximal timeout is %dsec. Timeout remains at %d", maxTimeout, mctx.CutOverLockTimeoutSeconds) } mctx.CutOverLockTimeoutSeconds = timeoutSeconds return nil @@ -672,6 +908,54 @@ func (mctx *MigrationContext) GetLastHeartbeatOnChangelogTime() time.Time { return mctx.lastHeartbeatOnChangelogTime } +// UpdateLastAppliedBinlogEventTime records the binlog-header timestamp of the +// last event successfully applied to the target. Used by move-tables mode to +// derive writer lag. +func (mctx *MigrationContext) UpdateLastAppliedBinlogEventTime(t time.Time) { + mctx.binlogWriterLagMutex.Lock() + defer mctx.binlogWriterLagMutex.Unlock() + + mctx.lastAppliedBinlogEventTime = t +} + +// MarkBinlogEventStreamed records that the streamer just delivered an event for +// the moved table. Used to distinguish "falling behind" from "source is idle". +func (mctx *MigrationContext) MarkBinlogEventStreamed() { + mctx.binlogWriterLagMutex.Lock() + defer mctx.binlogWriterLagMutex.Unlock() + + mctx.lastBinlogEventStreamedTime = time.Now() +} + +// BumpBinlogWriterLagIfIdle treats prolonged streamer silence as "caught up": +// if no event has been streamed for the moved table within idleThreshold, the +// last-applied timestamp is advanced to now so writer lag does not climb forever +// while the source is quiet. +func (mctx *MigrationContext) BumpBinlogWriterLagIfIdle(idleThreshold time.Duration) { + mctx.binlogWriterLagMutex.Lock() + defer mctx.binlogWriterLagMutex.Unlock() + + if mctx.lastBinlogEventStreamedTime.IsZero() || time.Since(mctx.lastBinlogEventStreamedTime) >= idleThreshold { + mctx.lastAppliedBinlogEventTime = time.Now() + } +} + +// GetBinlogWriterLag returns now - last applied event timestamp, the move-tables +// writer lag. It returns 0 before any event has been applied. +func (mctx *MigrationContext) GetBinlogWriterLag() time.Duration { + mctx.binlogWriterLagMutex.Lock() + defer mctx.binlogWriterLagMutex.Unlock() + + if mctx.lastAppliedBinlogEventTime.IsZero() { + return 0 + } + lag := time.Since(mctx.lastAppliedBinlogEventTime) + if lag < 0 { + return 0 + } + return lag +} + func (mctx *MigrationContext) SetHeartbeatIntervalMilliseconds(heartbeatIntervalMilliseconds int64) { if heartbeatIntervalMilliseconds < 100 { heartbeatIntervalMilliseconds = 100 @@ -928,11 +1212,33 @@ func (mctx *MigrationContext) ApplyCredentials() { // Override mctx.InspectorConnectionConfig.Password = mctx.CliPassword } + + if mctx.IsMoveTablesMode() { + // Derive the applier config from the inspector config, but point it at + // the target host and override credentials from the target CLI args. + mctx.MoveTables.ConnectionConfig = mctx.InspectorConnectionConfig.DuplicateCredentials(mysql.InstanceKey{ + Hostname: mctx.MoveTables.TargetHost, + Port: mctx.MoveTables.TargetPort, + }) + if mctx.MoveTables.TargetUser != "" { + // Override + mctx.MoveTables.ConnectionConfig.User = mctx.MoveTables.TargetUser + } + if mctx.MoveTables.TargetPass != "" { + // Override + mctx.MoveTables.ConnectionConfig.Password = mctx.MoveTables.TargetPass + } + } } func (mctx *MigrationContext) SetupTLS() error { if mctx.UseTLS { - return mctx.InspectorConnectionConfig.UseTLS(mctx.TLSCACertificate, mctx.TLSCertificate, mctx.TLSKey, mctx.TLSAllowInsecure) + if err := mctx.InspectorConnectionConfig.UseTLS(mctx.TLSCACertificate, mctx.TLSCertificate, mctx.TLSKey, mctx.TLSAllowInsecure); err != nil { + return err + } + if mctx.IsMoveTablesMode() && mctx.MoveTables.ConnectionConfig != nil { + return mctx.MoveTables.ConnectionConfig.UseTLS(mctx.TLSCACertificate, mctx.TLSCertificate, mctx.TLSKey, mctx.TLSAllowInsecure) + } } return nil } @@ -1038,6 +1344,89 @@ func (mctx *MigrationContext) CancelContext() { } } +// IsMoveTablesMode returns true if gh-ost should be used for moving tables instead of running a schema migration. +func (mctx *MigrationContext) IsMoveTablesMode() bool { + return len(mctx.MoveTables.TableNames) > 0 +} + +// InitMoveTableContainers builds (or rebuilds) the per-table runtime containers +// from the ordered MoveTables.TableNames list. It is idempotent: tables already +// present in the map keep their existing container so callers may invoke it +// after partially populating state. Source and target table names match in +// move-tables mode; only the database may differ. +func (mctx *MigrationContext) InitMoveTableContainers() { + if mctx.MoveTables.Tables == nil { + mctx.MoveTables.Tables = make(map[string]*MoveTable, len(mctx.MoveTables.TableNames)) + } + for _, tableName := range mctx.MoveTables.TableNames { + if _, ok := mctx.MoveTables.Tables[tableName]; ok { + continue + } + mctx.MoveTables.Tables[tableName] = &MoveTable{ + SourceDatabaseName: mctx.DatabaseName, + SourceTableName: tableName, + TargetDatabaseName: mctx.GetTargetDatabaseName(), + TargetTableName: tableName, + } + } +} + +// GetMoveTable returns the per-table container for the given source table name, +// or nil if it has not been initialized. +func (mctx *MigrationContext) GetMoveTable(tableName string) *MoveTable { + if mctx.MoveTables.Tables == nil { + return nil + } + return mctx.MoveTables.Tables[tableName] +} + +// OrderedMoveTables returns the per-table containers in --move-tables order. +// Iteration must always use this deterministic order, never the Tables map's +// (random) iteration order. +func (mctx *MigrationContext) OrderedMoveTables() []*MoveTable { + tables := make([]*MoveTable, 0, len(mctx.MoveTables.TableNames)) + for _, tableName := range mctx.MoveTables.TableNames { + if mt := mctx.GetMoveTable(tableName); mt != nil { + tables = append(tables, mt) + } + } + return tables +} + +// MoveTablesRunToken returns a short, stable identifier for a move-tables run, +// derived from the (sorted) set of migrated table names. It is: +// - deterministic: the same table set always yields the same token, so a +// resumed run finds the same run-wide artifacts (e.g. the checkpoint table). +// - order-independent: --move-tables=a,b and --move-tables=b,a match. +// - fixed-length: independent of how many tables are moved (so it never blows +// past identifier length limits the way a concatenation of names would). +// +// It is used to name run-wide singular artifacts (checkpoint table, applier +// advisory lock, serve socket) so they never depend on any single migrated +// table name. Returns "" outside move-tables mode. +func (mctx *MigrationContext) MoveTablesRunToken() string { + if !mctx.IsMoveTablesMode() { + return "" + } + names := append([]string(nil), mctx.MoveTables.TableNames...) + sort.Strings(names) + // NUL separator: table names cannot contain it, so the join is unambiguous. + sum := sha256.Sum256([]byte(strings.Join(names, "\x00"))) + return hex.EncodeToString(sum[:6]) // 12 hex chars / 48 bits +} + +// AllMoveTablesRowCopyComplete reports whether every migrated table has finished +// its row copy. The on-row-copy-complete hook and the cutover only proceed once +// this is true. +func (mctx *MigrationContext) AllMoveTablesRowCopyComplete() bool { + for _, mt := range mctx.OrderedMoveTables() { + if !mt.IsRowCopyComplete() { + return false + } + } + return true +} + // SendWithContext attempts to send a value to a channel, but returns early // if the context is cancelled. This prevents goroutine deadlocks when the // channel receiver has exited due to an error. @@ -1058,3 +1447,33 @@ func SendWithContext[T any](ctx context.Context, ch chan<- T, val T) error { return ctx.Err() } } + +type failPointOpts struct { + wait time.Duration +} + +type FailPointOpt func(*failPointOpts) + +// WithFailPointWait sets the time for a fail point to wait before exiting. +func WithFailPointWait(wait time.Duration) FailPointOpt { + return func(opts *failPointOpts) { + opts.wait = wait + } +} + +func (mctx *MigrationContext) NewFailPoint(name string, opts ...FailPointOpt) { + if mctx.UnsafeFailPointsEnabled { + var fpo failPointOpts + for _, opt := range opts { + opt(&fpo) + } + + failpoint.Inject(name, func(_ failpoint.Value) { + mctx.Log.Debugf("[TEST] Encountered fail point: '%s'", name) + if fpo.wait > 0 { + time.Sleep(fpo.wait) + } + panic(fmt.Sprintf("[TEST] Encountered fail point: '%s'", name)) + }) + } +} diff --git a/go/base/context_test.go b/go/base/context_test.go index a9f62150d..b0331d075 100644 --- a/go/base/context_test.go +++ b/go/base/context_test.go @@ -61,6 +61,44 @@ func TestGetTableNames(t *testing.T) { } } +func TestMoveTableDelName(t *testing.T) { + context := NewMigrationContext() + // Per-table `_
_del` rollback handle, independent of any other table. + require.Equal(t, "_some_table_del", context.MoveTableDelName("some_table")) + require.Equal(t, "_other_del", context.MoveTableDelName("other")) + + // Honors --timestamp-old-table like the single-table GetOldTableName does. + context.TimestampOldTable = true + longForm := "Jan 2, 2006 at 3:04pm (MST)" + context.StartTime, _ = time.Parse(longForm, "Feb 3, 2013 at 7:54pm (PST)") + require.Equal(t, "_some_table_20130203195400_del", context.MoveTableDelName("some_table")) +} + +func TestMoveTablesRunToken(t *testing.T) { + // Empty outside move-tables mode. + require.Equal(t, "", NewMigrationContext().MoveTablesRunToken()) + + context := NewMigrationContext() + context.MoveTables.TableNames = []string{"a", "b", "c"} + token := context.MoveTablesRunToken() + // Fixed length, lowercase hex (12 chars / 48 bits). + require.Len(t, token, 12) + require.Regexp(t, "^[0-9a-f]{12}$", token) + // Deterministic: the same set always yields the same token (so a resumed run + // finds the same run-wide artifacts). + require.Equal(t, token, context.MoveTablesRunToken()) + + // Order-independent: --move-tables=a,b,c and =c,b,a match. + reordered := NewMigrationContext() + reordered.MoveTables.TableNames = []string{"c", "b", "a"} + require.Equal(t, token, reordered.MoveTablesRunToken()) + + // A different set yields a different token. + different := NewMigrationContext() + different.MoveTables.TableNames = []string{"a", "b", "d"} + require.NotEqual(t, token, different.MoveTablesRunToken()) +} + func TestGetTriggerNames(t *testing.T) { { context := NewMigrationContext() @@ -216,6 +254,62 @@ func TestReadConfigFile(t *testing.T) { } } +func TestApplyCredentialsMoveTablesDerivesConnectionConfig(t *testing.T) { + ctx := NewMigrationContext() + ctx.MoveTables.TableNames = []string{"some_table"} + ctx.MoveTables.TargetHost = "target-host" + ctx.MoveTables.TargetPort = 3307 + ctx.MoveTables.TargetUser = "target-user" + ctx.MoveTables.TargetPass = "target-pass" + + ctx.InspectorConnectionConfig.Key.Hostname = "source-host" + ctx.InspectorConnectionConfig.Key.Port = 3306 + ctx.InspectorConnectionConfig.User = "source-user" + ctx.InspectorConnectionConfig.Password = "source-pass" + ctx.InspectorConnectionConfig.Timeout = 12.5 + ctx.InspectorConnectionConfig.TransactionIsolation = "REPEATABLE-READ" + ctx.InspectorConnectionConfig.Charset = "utf8mb4" + + ctx.ApplyCredentials() + + got := ctx.MoveTables.ConnectionConfig + require.NotNil(t, got) + require.Equal(t, "target-host", got.Key.Hostname) + require.Equal(t, 3307, got.Key.Port) + require.Equal(t, "target-user", got.User) + require.Equal(t, "target-pass", got.Password) + require.Equal(t, 12.5, got.Timeout) + require.Equal(t, "REPEATABLE-READ", got.TransactionIsolation) + require.Equal(t, "utf8mb4", got.Charset) + require.NotNil(t, got.ImpliedKey) + require.Equal(t, "target-host", got.ImpliedKey.Hostname) + require.Equal(t, 3307, got.ImpliedKey.Port) +} + +func TestSetupTLSAppliesToMoveTablesConfig(t *testing.T) { + ctx := NewMigrationContext() + ctx.UseTLS = true + ctx.TLSAllowInsecure = true + ctx.MoveTables.TableNames = []string{"some_table"} + ctx.MoveTables.TargetHost = "target-host" + ctx.MoveTables.TargetPort = 3307 + ctx.MoveTables.TargetUser = "target-user" + ctx.MoveTables.TargetPass = "target-pass" + + ctx.InspectorConnectionConfig.Key.Hostname = "source-host" + ctx.InspectorConnectionConfig.Key.Port = 3306 + ctx.InspectorConnectionConfig.User = "source-user" + ctx.InspectorConnectionConfig.Password = "source-pass" + + ctx.ApplyCredentials() + require.NoError(t, ctx.SetupTLS()) + + require.NotNil(t, ctx.InspectorConnectionConfig.TLSConfig()) + require.NotNil(t, ctx.MoveTables.ConnectionConfig.TLSConfig()) + require.Equal(t, "source-host", ctx.InspectorConnectionConfig.TLSConfig().ServerName) + require.Equal(t, "target-host", ctx.MoveTables.ConnectionConfig.TLSConfig().ServerName) +} + func TestSetAbortError_StoresFirstError(t *testing.T) { ctx := NewMigrationContext() @@ -270,3 +364,22 @@ func TestSetAbortError_ThreadSafe(t *testing.T) { t.Errorf("Stored error %v not in list of sent errors", got) } } + +func TestSetCutOverLockTimeoutSecondsRangeByMode(t *testing.T) { + { + ctx := NewMigrationContext() + require.NoError(t, ctx.SetCutOverLockTimeoutSeconds(10)) + err := ctx.SetCutOverLockTimeoutSeconds(11) + require.Error(t, err) + require.Contains(t, err.Error(), "maximal timeout is 10sec") + } + + { + ctx := NewMigrationContext() + ctx.MoveTables.TableNames = []string{"tbl"} + require.NoError(t, ctx.SetCutOverLockTimeoutSeconds(60)) + err := ctx.SetCutOverLockTimeoutSeconds(61) + require.Error(t, err) + require.Contains(t, err.Error(), "maximal timeout is 60sec") + } +} diff --git a/go/binlog/binlog_entry.go b/go/binlog/binlog_entry.go index 7620281d2..c1216054c 100644 --- a/go/binlog/binlog_entry.go +++ b/go/binlog/binlog_entry.go @@ -7,6 +7,7 @@ package binlog import ( "fmt" + "time" "github.com/github/gh-ost/go/mysql" ) @@ -14,7 +15,11 @@ import ( // BinlogEntry describes an entry in the binary log type BinlogEntry struct { Coordinates mysql.BinlogCoordinates - DmlEvent *BinlogDMLEvent + // Timestamp is the wall-clock time recorded in the binlog event header of the + // event that produced this entry. It is used in move-tables mode to measure + // writer lag (now - last applied event timestamp). + Timestamp time.Time + DmlEvent *BinlogDMLEvent } // NewBinlogEntryAt creates an empty, ready to go BinlogEntry object diff --git a/go/binlog/gomysql_reader.go b/go/binlog/gomysql_reader.go index e139726fd..ee50b0d1a 100644 --- a/go/binlog/gomysql_reader.go +++ b/go/binlog/gomysql_reader.go @@ -122,6 +122,7 @@ func (gmr *GoMySQLReader) handleRowsEvent(ev *replication.BinlogEvent, rowsEvent continue } binlogEntry := NewBinlogEntryAt(currentCoords) + binlogEntry.Timestamp = time.Unix(int64(ev.Header.Timestamp), 0) binlogEntry.DmlEvent = NewBinlogDMLEvent( string(rowsEvent.Table.Schema), string(rowsEvent.Table.Table), diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index cd1f5993f..61abcad99 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -12,12 +12,15 @@ import ( "os" "os/signal" "regexp" + "slices" + "strings" "syscall" "time" "github.com/github/gh-ost/go/base" "github.com/github/gh-ost/go/logic" "github.com/github/gh-ost/go/metrics" + "github.com/github/gh-ost/go/mysql" "github.com/github/gh-ost/go/sql" _ "github.com/go-sql-driver/mysql" "github.com/openark/golib/log" @@ -131,7 +134,7 @@ func main() { maxLagMillis := flag.Int64("max-lag-millis", 1500, "replication lag at which to throttle operation") replicationLagQuery := flag.String("replication-lag-query", "", "Deprecated. gh-ost uses an internal, subsecond resolution query") - throttleControlReplicas := flag.String("throttle-control-replicas", "", "List of replicas on which to check for lag; comma delimited. Example: myhost1.com:3306,myhost2.com,myhost3.com:3307") + throttleControlReplicas := flag.String("throttle-control-replicas", "", "List of replicas on which to check for lag; comma delimited. Example: myhost1.com:3306,myhost2.com,myhost3.com:3307. In move-tables mode, these replicas are expected to be in the target cluster. Specified target credentials will be used for the connection.") throttleQuery := flag.String("throttle-query", "", "when given, issued (every second) to check if operation should throttle. Expecting to return zero for no-throttle, >0 for throttle. Query is issued on the migrated server. Make sure this query is lightweight") throttleHTTP := flag.String("throttle-http", "", "when given, gh-ost checks given URL via HEAD request; any response code other than 200 (OK) causes throttling; make sure it has low latency response") flag.Int64Var(&migrationContext.ThrottleHTTPIntervalMillis, "throttle-http-interval-millis", 100, "Number of milliseconds to wait before triggering another HTTP throttle check") @@ -185,9 +188,27 @@ func main() { version := flag.Bool("version", false, "Print version & exit") checkFlag := flag.Bool("check-flag", false, "Check if another flag exists/supported. This allows for cross-version scripting. Exits with 0 when all additional provided flags exist, nonzero otherwise. You must provide (dummy) values for flags that require a value. Example: gh-ost --check-flag --cut-over-lock-timeout-seconds --nice-ratio 0") flag.StringVar(&migrationContext.ForceTmpTableName, "force-table-names", "", "table name prefix to be used on the temporary tables") - flag.CommandLine.SetOutput(os.Stdout) + // move tables flags + moveTables := flag.String("move-tables", "", "Comma delimited list of tables to move. e.g. 'table1,table2,table3'. This is a special mode that allows you to move tables between database clusters. This mode is mutually exclusive with --alter, --table, --test-on-replica, --migrate-on-replica and --revert.") + flag.StringVar(&migrationContext.MoveTables.TargetHost, "target-host", "", "Target MySQL hostname for --move-tables mode. Must be specified if --move-tables is specified.") + flag.IntVar(&migrationContext.MoveTables.TargetPort, "target-port", 3306, "Target MySQL port for --move-tables mode. Defaults to 3306.") + flag.StringVar(&migrationContext.MoveTables.TargetUser, "target-user", "", "Target MySQL username for --move-tables mode. If not provided, uses the same user as the source connection") + flag.StringVar(&migrationContext.MoveTables.TargetPass, "target-password", "", "Target MySQL password for --move-tables mode. If not provided, uses the same password as the source connection") + flag.StringVar(&migrationContext.MoveTables.TargetDatabase, "target-database", "", "Target MySQL database name for --move-tables mode. If not provided, uses the same database name as the source connection") + flag.BoolVar(&migrationContext.MoveTables.AllowOnSourcePrimary, "allow-on-source-primary", false, "allow --move-tables to read (schema, row copy, binlog) from the source cluster's primary. By default gh-ost stops if --host is the primary; prefer pointing --host at a replica to spare the primary the copy load.") + + // unsafe fail points, for integration testing purposes + flag.BoolVar(&migrationContext.UnsafeFailPointsEnabled, "unsafe-fail-points-enabled", false, "UNSAFE: Enable fail points for integration testing purposes. Do not use in production.") + + flag.CommandLine.SetOutput(os.Stdout) flag.Parse() + cutOverLockTimeoutUserSpecified := false + flag.Visit(func(f *flag.Flag) { + if f.Name == "cut-over-lock-timeout-seconds" { + cutOverLockTimeoutUserSpecified = true + } + }) if *checkFlag { return @@ -224,14 +245,8 @@ func main() { migrationContext.Log.SetLevel(log.ERROR) } - if err := migrationContext.SetConnectionConfig(*storageEngine); err != nil { - migrationContext.Log.Fatale(err) - } - - migrationContext.SetConnectionCharset(*charset) - - if migrationContext.AlterStatement == "" && !migrationContext.Revert { - log.Fatal("--alter must be provided and statement must not be empty") + if migrationContext.AlterStatement == "" && !migrationContext.Revert && *moveTables == "" { + log.Fatal("--alter must be provided and statement must not be empty, or --revert must be used, or --move-tables must be used") } parser := sql.NewParserFromAlterStatement(migrationContext.AlterStatement) migrationContext.AlterStatementOptions = parser.GetAlterStatementOptions() @@ -271,7 +286,7 @@ func main() { migrationContext.Log.Fatale(err) } - if migrationContext.OriginalTableName == "" { + if migrationContext.OriginalTableName == "" && *moveTables == "" { if parser.HasExplicitTable() { migrationContext.OriginalTableName = parser.GetExplicitTable() } else { @@ -334,13 +349,77 @@ func main() { if *storageEngine == "rocksdb" { migrationContext.Log.Warning("RocksDB storage engine support is experimental") } - if migrationContext.CheckpointIntervalSeconds < 10 { + // ignore low checkpoint intervals in unsafe mode as frequent checkpoints are required to reliably + // reduce test duration + if migrationContext.CheckpointIntervalSeconds < 10 && !migrationContext.UnsafeFailPointsEnabled { migrationContext.Log.Fatalf("--checkpoint-seconds should be >=10") } if migrationContext.CountTableRows && migrationContext.PanicOnWarnings { migrationContext.Log.Warning("--exact-rowcount with --panic-on-warnings: row counts cannot be exact due to warning detection") } + if *moveTables != "" { + if migrationContext.AlterStatement != "" { + log.Fatal("--move-tables is mutually exclusive with --alter") + } + if migrationContext.OriginalTableName != "" { + log.Fatal("--move-tables is mutually exclusive with --table") + } + if migrationContext.TestOnReplica { + log.Fatal("--move-tables is mutually exclusive with --test-on-replica") + } + if migrationContext.MigrateOnReplica { + log.Fatal("--move-tables is mutually exclusive with --migrate-on-replica") + } + if migrationContext.Revert { + log.Fatal("--move-tables is mutually exclusive with --revert") + } + if migrationContext.MoveTables.TargetHost == "" { + log.Fatal("--target-host must be specified when using --move-tables") + } + if migrationContext.PostponeCutOverFlagFile == "" { + log.Fatal("--postpone-cut-over-flag-file must be specified when using --move-tables") + } + if !migrationContext.Checkpoint { + log.Infof("--move-tables requires checkpointing; enabling --checkpoint") + migrationContext.Checkpoint = true + } + if !migrationContext.UseGTIDs { + log.Infof("--move-tables requires GTID coordinates for cutover drain checks; enabling --gtid") + migrationContext.UseGTIDs = true + } + migrationContext.MoveTables.TableNames = strings.Split(*moveTables, ",") + for i := range migrationContext.MoveTables.TableNames { + migrationContext.MoveTables.TableNames[i] = strings.TrimSpace(migrationContext.MoveTables.TableNames[i]) + } + migrationContext.MoveTables.TableNames = slices.DeleteFunc(migrationContext.MoveTables.TableNames, func(s string) bool { return s == "" }) + if len(migrationContext.MoveTables.TableNames) == 0 { + log.Fatal("--move-tables requires at least one table") + } + // Reject duplicate table names: a table listed twice would register two + // listeners and two row-copy loops for the same data. + seenMoveTables := make(map[string]bool, len(migrationContext.MoveTables.TableNames)) + for _, tableName := range migrationContext.MoveTables.TableNames { + if seenMoveTables[tableName] { + log.Fatalf("--move-tables lists table %q more than once", tableName) + } + seenMoveTables[tableName] = true + } + if migrationContext.MoveTables.TargetDatabase == "" { + migrationContext.MoveTables.TargetDatabase = migrationContext.DatabaseName + } + if !cutOverLockTimeoutUserSpecified { + *cutOverLockTimeoutSeconds = 60 + } + migrationContext.MoveTables.ConnectionConfig = mysql.NewConnectionConfig() + } + + if err := migrationContext.SetConnectionConfig(*storageEngine); err != nil { + migrationContext.Log.Fatale(err) + } + + migrationContext.SetConnectionCharset(*charset) + switch *cutOver { case "atomic", "default", "": migrationContext.CutOverType = base.CutOverAtomic @@ -362,7 +441,14 @@ func main() { migrationContext.Log.Fatale(err) } if migrationContext.ServeSocketFile == "" { - migrationContext.ServeSocketFile = fmt.Sprintf("/tmp/gh-ost.%s.%s.sock", migrationContext.DatabaseName, migrationContext.OriginalTableName) + if migrationContext.IsMoveTablesMode() { + // OriginalTableName is not set until MoveTables() runs and there is no + // single "primary" table, so name the socket from the set-derived run + // token (avoids an empty path component like /tmp/gh-ost.test..sock). + migrationContext.ServeSocketFile = fmt.Sprintf("/tmp/gh-ost.%s.movetables-%s.sock", migrationContext.DatabaseName, migrationContext.MoveTablesRunToken()) + } else { + migrationContext.ServeSocketFile = fmt.Sprintf("/tmp/gh-ost.%s.%s.sock", migrationContext.DatabaseName, migrationContext.OriginalTableName) + } } if *askPass { fmt.Println("Password:") @@ -411,6 +497,8 @@ func main() { var err error if migrationContext.Revert { err = migrator.Revert() + } else if migrationContext.IsMoveTablesMode() { + err = migrator.MoveTables() } else { err = migrator.Migrate() } diff --git a/go/logic/applier.go b/go/logic/applier.go index 3f401c598..7f705baff 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -92,6 +92,29 @@ type Applier struct { migrationLockName string migrationLockStop chan struct{} migrationLockDone chan struct{} + + moveTablesTargetDB *gosql.DB + moveTablesConnectionConfig *mysql.ConnectionConfig + + // moveTablesBuilders holds the per-table query builders, keyed by source + // table name. In move-tables mode there is one entry per migrated table; DML + // is routed to the right set at apply time using the TableName already on each + // binlog DML event. Empty in standard (single-table) mode. + moveTablesBuilders map[string]*moveTableBuilders +} + +// moveTableBuilders holds the query builders and schema needed to copy and apply +// DML for a single migrated table in move-tables mode. One instance exists per +// table; the applier selects the right instance by source table name. +type moveTableBuilders struct { + uniqueKey *sql.UniqueKey + originalTableColumns *sql.ColumnList + dmlDeleteQueryBuilder *sql.DMLDeleteQueryBuilder + dmlInsertQueryBuilder *sql.DMLInsertQueryBuilder + dmlUpdateQueryBuilder *sql.DMLUpdateQueryBuilder + copySelectFirstQueryBuilder *sql.MoveTableCopySelectQueryBuilder + copySelectNextQueryBuilder *sql.MoveTableCopySelectQueryBuilder + copyInsertQueryBuilder *sql.MoveTableCopyInsertQueryBuilder } func NewApplier(migrationContext *base.MigrationContext) *Applier { @@ -100,7 +123,38 @@ func NewApplier(migrationContext *base.MigrationContext) *Applier { migrationContext: migrationContext, finishedMigrating: 0, name: "applier", + + moveTablesConnectionConfig: migrationContext.MoveTables.ConnectionConfig, + } +} + +func (apl *Applier) checkpointDB() *gosql.DB { + if apl.migrationContext.IsMoveTablesMode() && apl.moveTablesTargetDB != nil { + return apl.moveTablesTargetDB + } + return apl.db +} + +func (apl *Applier) checkpointDatabaseName() string { + if apl.migrationContext.IsMoveTablesMode() { + return apl.migrationContext.GetTargetDatabaseName() + } + return apl.migrationContext.DatabaseName +} + +func (apl *Applier) checkpointDrainGTIDString(chk *Checkpoint) string { + if chk == nil || chk.MoveTablesCutOverDrainGTID == nil || chk.MoveTablesCutOverDrainGTID.IsEmpty() { + return "" } + return chk.MoveTablesCutOverDrainGTID.String() +} + +func (apl *Applier) checkpointRangeColumnNames() (minColumnNames []string, maxColumnNames []string) { + for _, col := range apl.migrationContext.UniqueKey.Columns.Columns() { + minColumnNames = append(minColumnNames, sql.TruncateColumnName(col.Name, sql.MaxColumnNameLength-4)+"_min") + maxColumnNames = append(maxColumnNames, sql.TruncateColumnName(col.Name, sql.MaxColumnNameLength-4)+"_max") + } + return minColumnNames, maxColumnNames } // compileMigrationKeyWarningRegex compiles a regex pattern that matches duplicate key warnings @@ -108,8 +162,44 @@ func NewApplier(migrationContext *base.MigrationContext) *Applier { // hence the optional table name prefix. Metacharacters in table/index names are escaped to avoid // regex syntax errors. func (apl *Applier) compileMigrationKeyWarningRegex() (*regexp.Regexp, error) { - escapedTable := regexp.QuoteMeta(apl.migrationContext.GetGhostTableName()) - escapedKey := regexp.QuoteMeta(apl.migrationContext.UniqueKey.NameInGhostTable) + if apl.migrationContext.IsMoveTablesMode() { + return apl.compileMoveTablesKeyWarningRegex() + } + return compileKeyWarningRegex(apl.migrationContext.GetGhostTableName(), apl.migrationContext.UniqueKey.NameInGhostTable) +} + +// compileMoveTablesKeyWarningRegex builds one duplicate-key warning regex +// covering every migrated table's unique key. A duplicate on any migrated +// table's key is an expected artifact of binlog replay after bulk copy, so a +// combined alternation is sufficient and avoids singling out a representative +// table (a DML batch may interleave statements for several tables). +func (apl *Applier) compileMoveTablesKeyWarningRegex() (*regexp.Regexp, error) { + var alternatives []string + for _, mt := range apl.migrationContext.OrderedMoveTables() { + if mt.UniqueKey == nil { + continue + } + escapedTable := regexp.QuoteMeta(mt.TargetTableName) + escapedKey := regexp.QuoteMeta(mt.UniqueKey.NameInGhostTable) + alternatives = append(alternatives, fmt.Sprintf(`(%s\.)?%s`, escapedTable, escapedKey)) + } + if len(alternatives) == 0 { + return regexp.Compile(`$.^`) // matches nothing + } + pattern := fmt.Sprintf(`for key '(%s)'`, strings.Join(alternatives, "|")) + migrationKeyRegex, err := regexp.Compile(pattern) + if err != nil { + return nil, fmt.Errorf("failed to compile move-tables key pattern: %w", err) + } + return migrationKeyRegex, nil +} + +// compileKeyWarningRegex compiles the duplicate-key warning regex for a specific +// target table + unique key name. In move-tables mode each table has its own +// unique key, so the duplicate-key filter must be compiled per table. +func compileKeyWarningRegex(targetTableName, uniqueKeyName string) (*regexp.Regexp, error) { + escapedTable := regexp.QuoteMeta(targetTableName) + escapedKey := regexp.QuoteMeta(uniqueKeyName) migrationUniqueKeyPattern := fmt.Sprintf(`for key '(%s\.)?%s'`, escapedTable, escapedKey) migrationKeyRegex, err := regexp.Compile(migrationUniqueKeyPattern) if err != nil { @@ -119,7 +209,7 @@ func (apl *Applier) compileMigrationKeyWarningRegex() (*regexp.Regexp, error) { } func (apl *Applier) InitDBConnections() (err error) { - applierUri := apl.connectionConfig.GetDBUri(apl.migrationContext.DatabaseName) + applierUri := apl.connectionConfig.GetDBUri(apl.migrationContext.GetTargetDatabaseName()) uriWithMulti := fmt.Sprintf("%s&multiStatements=true", applierUri) if apl.db, _, err = mysql.GetDB(apl.migrationContext.Uuid, uriWithMulti); err != nil { return err @@ -147,8 +237,27 @@ func (apl *Applier) InitDBConnections() (err error) { apl.connectionConfig.ImpliedKey = impliedKey } } - if err := apl.readTableColumns(); err != nil { - return err + if !apl.migrationContext.IsMoveTablesMode() { + // read target table columns from applier + if err := apl.readTableColumns(); err != nil { + return err + } + } + if apl.moveTablesConnectionConfig != nil { + moveTablesURI := apl.moveTablesConnectionConfig.GetDBUri(apl.migrationContext.GetTargetDatabaseName()) + "&multiStatements=true" + if apl.moveTablesTargetDB, _, err = mysql.GetDB(apl.migrationContext.Uuid, moveTablesURI); err != nil { + return err + } + if _, err := base.ValidateConnection(apl.moveTablesTargetDB, apl.moveTablesConnectionConfig, apl.migrationContext, apl.name); err != nil { + return err + } + // Fail fast if the move-tables target is not a writable primary. All target + // work (table create, row-copy INSERT, checkpoint writes, checkpoint DROP) + // requires a writable host; catching read_only here turns a confusing + // mid-run write failure into a clear startup error. + if err := assertConnectionWritable(apl.moveTablesTargetDB, apl.moveTablesConnectionConfig.Key, "target"); err != nil { + return err + } } apl.migrationContext.Log.Infof("Applier initiated on %+v, version %+v", apl.connectionConfig.ImpliedKey, apl.migrationContext.ApplierMySQLVersion) return nil @@ -169,11 +278,24 @@ func buildMigrationLockName(db, table string) string { // preventing two gh-ost processes from migrating the same table concurrently // on the same MySQL server. func (apl *Applier) AcquireMigrationLock(ctx context.Context) error { - lockName := buildMigrationLockName(apl.migrationContext.DatabaseName, apl.migrationContext.OriginalTableName) + // One advisory lock per run. In move-tables mode it is keyed on the + // set-derived run token (not any single table) so two processes moving the + // same set of tables collide, while a single-table run keeps its table-keyed + // lock name. lockSubject is a human-readable description used in contention + // errors; neither branch consults the representative table accessor. + var lockTable, lockSubject string + if apl.migrationContext.IsMoveTablesMode() { + lockTable = "movetables." + apl.migrationContext.MoveTablesRunToken() + lockSubject = fmt.Sprintf("tables %v", apl.migrationContext.MoveTables.TableNames) + } else { + lockTable = apl.originalTableName() + lockSubject = fmt.Sprintf("`%s`.`%s`", apl.migrationContext.DatabaseName, apl.originalTableName()) + } + lockName := buildMigrationLockName(apl.migrationContext.GetTargetDatabaseName(), lockTable) // Use a dedicated *sql.DB so the pinned connection does not consume a // slot in apl.db's small pool (mysql.MaxDBPoolConnections). - lockURI := apl.connectionConfig.GetDBUri(apl.migrationContext.DatabaseName) + lockURI := apl.connectionConfig.GetDBUri(apl.migrationContext.GetTargetDatabaseName()) lockDB, err := mysql.OpenDB(lockURI) if err != nil { return fmt.Errorf("failed to open migration lock DB: %w", err) @@ -206,11 +328,11 @@ func (apl *Applier) AcquireMigrationLock(ctx context.Context) error { conn.Close() lockDB.Close() if holderID.Valid { - return fmt.Errorf("another gh-ost process is already migrating `%s`.`%s`: migration lock %s held by connection id %d", - apl.migrationContext.DatabaseName, apl.migrationContext.OriginalTableName, lockName, holderID.Int64) + return fmt.Errorf("another gh-ost process is already migrating %s: migration lock %s held by connection id %d", + lockSubject, lockName, holderID.Int64) } - return fmt.Errorf("another gh-ost process is already migrating `%s`.`%s`: migration lock %s is held", - apl.migrationContext.DatabaseName, apl.migrationContext.OriginalTableName, lockName) + return fmt.Errorf("another gh-ost process is already migrating %s: migration lock %s is held", + lockSubject, lockName) } apl.migrationLockConn = conn @@ -298,41 +420,119 @@ func (apl *Applier) releaseMigrationLock() { } func (apl *Applier) prepareQueries() (err error) { - if apl.dmlDeleteQueryBuilder, err = sql.NewDMLDeleteQueryBuilder( - apl.migrationContext.DatabaseName, - apl.migrationContext.GetGhostTableName(), - apl.migrationContext.OriginalTableColumns, - &apl.migrationContext.UniqueKey.Columns, - ); err != nil { - return err - } - if apl.dmlInsertQueryBuilder, err = sql.NewDMLInsertQueryBuilder( - apl.migrationContext.DatabaseName, - apl.migrationContext.GetGhostTableName(), - apl.migrationContext.OriginalTableColumns, - apl.migrationContext.SharedColumns, - apl.migrationContext.MappedSharedColumns, - ); err != nil { - return err - } - if apl.dmlUpdateQueryBuilder, err = sql.NewDMLUpdateQueryBuilder( - apl.migrationContext.DatabaseName, - apl.migrationContext.GetGhostTableName(), - apl.migrationContext.OriginalTableColumns, - apl.migrationContext.SharedColumns, - apl.migrationContext.MappedSharedColumns, - &apl.migrationContext.UniqueKey.Columns, - ); err != nil { - return err - } - if apl.migrationContext.Checkpoint { - if apl.checkpointInsertQueryBuilder, err = sql.NewCheckpointQueryBuilder( - apl.migrationContext.DatabaseName, - apl.migrationContext.GetCheckpointTableName(), + targetDatabaseName := apl.migrationContext.GetTargetDatabaseName() + + if !apl.migrationContext.IsMoveTablesMode() { + targetTableName := apl.migrationContext.GetGhostTableName() + if apl.dmlDeleteQueryBuilder, err = sql.NewDMLDeleteQueryBuilder( + targetDatabaseName, + targetTableName, + apl.migrationContext.OriginalTableColumns, + &apl.migrationContext.UniqueKey.Columns, + ); err != nil { + return err + } + if apl.dmlInsertQueryBuilder, err = sql.NewDMLInsertQueryBuilder( + targetDatabaseName, + targetTableName, + apl.migrationContext.OriginalTableColumns, + apl.migrationContext.SharedColumns, + apl.migrationContext.MappedSharedColumns, + ); err != nil { + return err + } + if apl.dmlUpdateQueryBuilder, err = sql.NewDMLUpdateQueryBuilder( + targetDatabaseName, + targetTableName, + apl.migrationContext.OriginalTableColumns, + apl.migrationContext.SharedColumns, + apl.migrationContext.MappedSharedColumns, &apl.migrationContext.UniqueKey.Columns, ); err != nil { return err } + if apl.migrationContext.Checkpoint { + if apl.checkpointInsertQueryBuilder, err = sql.NewCheckpointQueryBuilder( + apl.checkpointDatabaseName(), + apl.migrationContext.GetCheckpointTableName(), + &apl.migrationContext.UniqueKey.Columns, + false, + ); err != nil { + return err + } + } + return nil + } + + // Move-tables mode: build one set of query builders per migrated table. DML is + // routed to the right set at apply time by source table name (§2.1). There is + // no representative/primary table: every table is handled identically through + // its own builders, and the checkpoint uses a table-agnostic schema written by + // WriteMoveTableCheckpoints (no checkpointInsertQueryBuilder). + apl.moveTablesBuilders = make(map[string]*moveTableBuilders, len(apl.migrationContext.MoveTables.TableNames)) + for _, mt := range apl.migrationContext.OrderedMoveTables() { + if mt.UniqueKey == nil { + return fmt.Errorf("move-table %s.%s has no unique key; cannot prepare queries", mt.SourceDatabaseName, mt.SourceTableName) + } + b := &moveTableBuilders{ + uniqueKey: mt.UniqueKey, + originalTableColumns: mt.OriginalTableColumns, + } + if b.dmlDeleteQueryBuilder, err = sql.NewDMLDeleteQueryBuilder( + mt.TargetDatabaseName, + mt.TargetTableName, + mt.OriginalTableColumns, + &mt.UniqueKey.Columns, + ); err != nil { + return err + } + if b.dmlInsertQueryBuilder, err = sql.NewDMLInsertQueryBuilder( + mt.TargetDatabaseName, + mt.TargetTableName, + mt.OriginalTableColumns, + mt.SharedColumns, + mt.MappedSharedColumns, + ); err != nil { + return err + } + if b.dmlUpdateQueryBuilder, err = sql.NewDMLUpdateQueryBuilder( + mt.TargetDatabaseName, + mt.TargetTableName, + mt.OriginalTableColumns, + mt.SharedColumns, + mt.MappedSharedColumns, + &mt.UniqueKey.Columns, + ); err != nil { + return err + } + if b.copySelectFirstQueryBuilder, err = sql.NewMoveTableCopySelectQueryBuilder( + mt.SourceDatabaseName, + mt.SourceTableName, + mt.SharedColumns, + mt.UniqueKey.Name, + &mt.UniqueKey.Columns, + true, // <-- include start range values for first select query + ); err != nil { + return err + } + if b.copySelectNextQueryBuilder, err = sql.NewMoveTableCopySelectQueryBuilder( + mt.SourceDatabaseName, + mt.SourceTableName, + mt.SharedColumns, + mt.UniqueKey.Name, + &mt.UniqueKey.Columns, + false, + ); err != nil { + return err + } + if b.copyInsertQueryBuilder, err = sql.NewMoveTableCopyInsertQueryBuilder( + mt.TargetDatabaseName, + mt.TargetTableName, + mt.SharedColumns, + ); err != nil { + return err + } + apl.moveTablesBuilders[mt.SourceTableName] = b } return nil } @@ -373,7 +573,7 @@ func (apl *Applier) generateSqlModeQuery() string { func (apl *Applier) generateInstantDDLQuery() string { return fmt.Sprintf(`ALTER /* gh-ost */ TABLE %s.%s %s, ALGORITHM=INSTANT`, sql.EscapeName(apl.migrationContext.DatabaseName), - sql.EscapeName(apl.migrationContext.OriginalTableName), + sql.EscapeName(apl.originalTableName()), apl.migrationContext.AlterStatementOptions, ) } @@ -381,7 +581,7 @@ func (apl *Applier) generateInstantDDLQuery() string { // readTableColumns reads table columns on applier func (apl *Applier) readTableColumns() (err error) { apl.migrationContext.Log.Infof("Examining table structure on applier") - apl.migrationContext.OriginalTableColumnsOnApplier, _, err = mysql.GetTableColumns(apl.db, apl.migrationContext.DatabaseName, apl.migrationContext.OriginalTableName) + apl.migrationContext.OriginalTableColumnsOnApplier, _, err = mysql.GetTableColumns(apl.db, apl.migrationContext.DatabaseName, apl.originalTableName()) if err != nil { return err } @@ -404,6 +604,17 @@ func (apl *Applier) tableExists(tableName string) (tableFound bool) { return (m != nil) } +// originalTableName returns the single migrated table. It is a representative +// accessor that has no meaning in move-tables mode (every table is handled +// through its own MoveTable container), so calling it there is a programmer +// error and panics to fail fast rather than silently operate on the wrong table. +func (apl *Applier) originalTableName() string { + if apl.migrationContext.IsMoveTablesMode() { + panic("applier.originalTableName() must not be called in move-tables mode; use the per-table MoveTable container instead") + } + return apl.migrationContext.OriginalTableName +} + // ValidateOrDropExistingTables verifies ghost and changelog tables do not exist, // or attempts to drop them if instructed to. func (apl *Applier) ValidateOrDropExistingTables() error { @@ -485,17 +696,19 @@ func retryOnLockWaitTimeout(operation func() error, maxRetries int64, logger bas return err } -// CreateGhostTable creates the ghost table on the applier host -func (apl *Applier) CreateGhostTable() error { +// createTargetTable creates the table on the applier host to which the applier will +// apply changes. +func (apl *Applier) createTargetTable(targetTableName string) error { + targetDatabase := apl.migrationContext.GetTargetDatabaseName() query := fmt.Sprintf(`create /* gh-ost */ table %s.%s like %s.%s`, + sql.EscapeName(targetDatabase), + sql.EscapeName(targetTableName), sql.EscapeName(apl.migrationContext.DatabaseName), - sql.EscapeName(apl.migrationContext.GetGhostTableName()), - sql.EscapeName(apl.migrationContext.DatabaseName), - sql.EscapeName(apl.migrationContext.OriginalTableName), + sql.EscapeName(apl.originalTableName()), ) - apl.migrationContext.Log.Infof("Creating ghost table %s.%s", - sql.EscapeName(apl.migrationContext.DatabaseName), - sql.EscapeName(apl.migrationContext.GetGhostTableName()), + apl.migrationContext.Log.Infof("Creating target table %s.%s", + sql.EscapeName(targetDatabase), + sql.EscapeName(targetTableName), ) err := func() error { @@ -514,7 +727,7 @@ func (apl *Applier) CreateGhostTable() error { if _, err := tx.Exec(query); err != nil { return err } - apl.migrationContext.Log.Infof("Ghost table created") + apl.migrationContext.Log.Infof("Target table created") if err := tx.Commit(); err != nil { // Neither SET SESSION nor ALTER are really transactional, so strictly speaking // there's no need to commit; but let's do this the legit way anyway. @@ -598,6 +811,120 @@ func (apl *Applier) AnalyzeGhostTable() error { return nil } +// createTargetTableFromStatement creates the table on the applier host to which the applier will +// apply changes. In move-tables mode this executes on moveTablesTargetDB (the target cluster); +// in standard mode it executes on apl.db (the source/applier host). +func (apl *Applier) createTargetTableFromStatement(targetTableName, createStatement string) error { + targetDatabase := apl.migrationContext.GetTargetDatabaseName() + apl.migrationContext.Log.Infof("Creating target table %s.%s", + sql.EscapeName(targetDatabase), + sql.EscapeName(targetTableName), + ) + + db := apl.db + if apl.migrationContext.IsMoveTablesMode() { + db = apl.moveTablesTargetDB + } + + err := func() error { + tx, err := db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + + sessionQuery := fmt.Sprintf(`SET SESSION time_zone = '%s'`, apl.migrationContext.ApplierTimeZone) + sessionQuery = fmt.Sprintf("%s, %s", sessionQuery, apl.generateSqlModeQuery()) + + if _, err := tx.Exec(sessionQuery); err != nil { + return err + } + if _, err := tx.Exec(createStatement); err != nil { + return err + } + apl.migrationContext.Log.Infof("Target table created") + if err := tx.Commit(); err != nil { + // Neither SET SESSION nor ALTER are really transactional, so strictly speaking + // there's no need to commit; but let's do this the legit way anyway. + return err + } + return nil + }() + + return err +} + +// CreateGhostTable creates the ghost table on the applier host +func (apl *Applier) CreateGhostTable() error { + if apl.migrationContext.IsMoveTablesMode() { + return errors.New("CreateGhostTable is not available in MoveTables mode") + } + return apl.createTargetTable(apl.migrationContext.GetGhostTableName()) +} + +// targetTableExists reports whether the named table already exists on the +// move-tables target database. +func (apl *Applier) targetTableExists(targetTableName string) (bool, error) { + var count int + if err := apl.moveTablesTargetDB.QueryRow( + "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema=? AND table_name=?", + apl.migrationContext.GetTargetDatabaseName(), targetTableName, + ).Scan(&count); err != nil { + return false, fmt.Errorf("failed to check for existing target table %s: %w", sql.EscapeName(targetTableName), err) + } + return count > 0, nil +} + +// CreateTargetTableForName creates the named target table on the target host +// from the given CREATE statement. In multi-table move-tables mode it is called +// once per migrated table. +func (apl *Applier) CreateTargetTableForName(targetTableName, createStatement string) error { + if !apl.migrationContext.IsMoveTablesMode() { + return errors.New("CreateTargetTableForName is only available in MoveTables mode") + } + targetDatabase := apl.migrationContext.GetTargetDatabaseName() + + // Explicit pre-check: abort before any data is copied if the target table + // already exists. The CREATE TABLE would also fail (MySQL ERROR 1050), but + // this gives operators a clear gh-ost error message explaining what to do. + exists, err := apl.targetTableExists(targetTableName) + if err != nil { + return err + } + if exists { + return fmt.Errorf("target table %s.%s already exists on the target cluster. Aborting to prevent writing into a table with unrelated data. Drop the table manually if this is intentional", + sql.EscapeName(targetDatabase), sql.EscapeName(targetTableName)) + } + + return apl.createTargetTableFromStatement(targetTableName, createStatement) +} + +// ValidateMoveTablesTargetsAbsent verifies that none of the migrated tables +// already exist on the target cluster, before any of them are created. This +// makes a collision abort cleanly up front rather than after partially creating +// the earlier tables in the set. +func (apl *Applier) ValidateMoveTablesTargetsAbsent() error { + if !apl.migrationContext.IsMoveTablesMode() { + return errors.New("ValidateMoveTablesTargetsAbsent is only available in MoveTables mode") + } + targetDatabase := apl.migrationContext.GetTargetDatabaseName() + var existing []string + for _, mt := range apl.migrationContext.OrderedMoveTables() { + exists, err := apl.targetTableExists(mt.TargetTableName) + if err != nil { + return err + } + if exists { + existing = append(existing, fmt.Sprintf("%s.%s", sql.EscapeName(targetDatabase), sql.EscapeName(mt.TargetTableName))) + } + } + if len(existing) > 0 { + return fmt.Errorf("the following target table(s) already exist on the target cluster: %s. Aborting before creating any tables to avoid leaving partial state; drop them manually if this is intentional", + strings.Join(existing, ", ")) + } + return nil +} + // AlterGhost applies `alter` statement on ghost table func (apl *Applier) AlterGhost() error { query := fmt.Sprintf(`alter /* gh-ost */ table %s.%s %s`, @@ -693,6 +1020,9 @@ func (apl *Applier) CreateCheckpointTable() error { if err := apl.DropCheckpointTable(); err != nil { return err } + if apl.migrationContext.IsMoveTablesMode() { + return apl.createMoveTablesCheckpointTable() + } colDefs := []string{ "`gh_ost_chk_id` bigint auto_increment primary key", "`gh_ost_chk_timestamp` bigint", @@ -718,12 +1048,45 @@ func (apl *Applier) CreateCheckpointTable() error { } query := fmt.Sprintf("create /* gh-ost */ table %s.%s (\n %s\n)", - sql.EscapeName(apl.migrationContext.DatabaseName), + sql.EscapeName(apl.checkpointDatabaseName()), sql.EscapeName(apl.migrationContext.GetCheckpointTableName()), strings.Join(colDefs, ",\n "), ) apl.migrationContext.Log.Infof("Created checkpoint table") - if _, err := sqlutils.ExecNoPrepare(apl.db, query); err != nil { + if _, err := sqlutils.ExecNoPrepare(apl.checkpointDB(), query); err != nil { + return err + } + return nil +} + +// createMoveTablesCheckpointTable creates the move-tables checkpoint table. It +// holds one row per migrated table, with the per-table iteration range stored +// in a table-agnostic, serialized text form (gh_ost_chk_range_min/max) so a +// single checkpoint table can serve tables with heterogeneous unique keys. The +// run-wide state (coords, totals, cutover markers, drain GTID) is replicated on +// every row, so the latest row carries the freshest run-wide state. +func (apl *Applier) createMoveTablesCheckpointTable() error { + colDefs := []string{ + "`gh_ost_chk_id` bigint auto_increment primary key", + "`gh_ost_chk_timestamp` bigint", + "`gh_ost_chk_table_name` varchar(320) charset utf8mb4 collate utf8mb4_bin", + "`gh_ost_chk_coords` text charset ascii", + "`gh_ost_chk_iteration` bigint", + "`gh_ost_rows_copied` bigint", + "`gh_ost_dml_applied` bigint", + "`gh_ost_is_cutover` tinyint(1) DEFAULT '0'", + "`gh_ost_move_tables_cutover_started` tinyint(1) DEFAULT '0'", + "`gh_ost_move_tables_drain_gtid` text charset ascii", + "`gh_ost_chk_range_min` text charset ascii", + "`gh_ost_chk_range_max` text charset ascii", + } + query := fmt.Sprintf("create /* gh-ost */ table %s.%s (\n %s\n)", + sql.EscapeName(apl.checkpointDatabaseName()), + sql.EscapeName(apl.migrationContext.GetCheckpointTableName()), + strings.Join(colDefs, ",\n "), + ) + apl.migrationContext.Log.Infof("Created move-tables checkpoint table") + if _, err := sqlutils.ExecNoPrepare(apl.checkpointDB(), query); err != nil { return err } return nil @@ -732,14 +1095,14 @@ func (apl *Applier) CreateCheckpointTable() error { // dropTable drops a given table on the applied host func (apl *Applier) dropTable(tableName string) error { query := fmt.Sprintf(`drop /* gh-ost */ table if exists %s.%s`, - sql.EscapeName(apl.migrationContext.DatabaseName), + sql.EscapeName(apl.checkpointDatabaseName()), sql.EscapeName(tableName), ) apl.migrationContext.Log.Infof("Dropping table %s.%s", - sql.EscapeName(apl.migrationContext.DatabaseName), + sql.EscapeName(apl.checkpointDatabaseName()), sql.EscapeName(tableName), ) - if _, err := sqlutils.ExecNoPrepare(apl.db, query); err != nil { + if _, err := sqlutils.ExecNoPrepare(apl.checkpointDB(), query); err != nil { return err } apl.migrationContext.Log.Infof("Table dropped") @@ -804,7 +1167,7 @@ func (apl *Applier) createTriggers(tableName string) error { sql.EscapeName(tableName), trigger.Statement, ) - apl.migrationContext.Log.Infof("Createing trigger %s on %s.%s", + apl.migrationContext.Log.Infof("Creating trigger %s on %s.%s", sql.EscapeName(triggerName), sql.EscapeName(apl.migrationContext.DatabaseName), sql.EscapeName(tableName), @@ -820,8 +1183,10 @@ func (apl *Applier) createTriggers(tableName string) error { // CreateTriggers creates the original triggers on applier host func (apl *Applier) CreateTriggersOnGhost() error { - err := apl.createTriggers(apl.migrationContext.GetGhostTableName()) - return err + if err := apl.createTriggers(apl.migrationContext.GetGhostTableName()); err != nil { + return fmt.Errorf("error creating triggers on ghost table: %w", err) + } + return nil } // DropChangelogTable drops the changelog table on the applier host @@ -841,12 +1206,22 @@ func (apl *Applier) DropOldTable() error { // DropGhostTable drops the ghost table on the applier host func (apl *Applier) DropGhostTable() error { - return apl.dropTable(apl.migrationContext.GetGhostTableName()) + if err := apl.dropTable(apl.migrationContext.GetGhostTableName()); err != nil { + return fmt.Errorf("error dropping ghost table: %w", err) + } + return nil } // WriteChangelog writes a value to the changelog table. -// It returns the hint as given, for convenience +// It returns the hint (or an empty string in move-tables mode), for convenience func (apl *Applier) WriteChangelog(hint, value string) (string, error) { + // In move-tables mode, there is no changelog table (§1.2). All changelog + // writes are no-ops. This is a single chokepoint rather than per-caller + // guards to prevent drift when new callers are added. + if apl.migrationContext.IsMoveTablesMode() { + return "", nil + } + explicitId := 0 switch hint { case "heartbeat": @@ -882,8 +1257,13 @@ func (apl *Applier) WriteChangelogState(value string) (string, error) { return apl.WriteAndLogChangelog("state", value) } -// WriteCheckpoints writes a checkpoint to the _ghk table. +// WriteCheckpoint writes a standard-mode checkpoint row to the _ghk table. In +// move-tables mode use WriteMoveTableCheckpoints instead; calling this there is a +// programmer error (the checkpoint schema and query builder are standard-only). func (apl *Applier) WriteCheckpoint(chk *Checkpoint) (int64, error) { + if apl.migrationContext.IsMoveTablesMode() { + panic("WriteCheckpoint() must not be called in move-tables mode; use WriteMoveTableCheckpoints") + } var insertId int64 uniqueKeyArgs := sqlutils.Args(chk.IterationRangeMin.AbstractValues()...) uniqueKeyArgs = append(uniqueKeyArgs, chk.IterationRangeMax.AbstractValues()...) @@ -893,15 +1273,166 @@ func (apl *Applier) WriteCheckpoint(chk *Checkpoint) (int64, error) { } args := sqlutils.Args(chk.LastTrxCoords.String(), chk.Iteration, chk.RowsCopied, chk.DMLApplied, chk.IsCutover) args = append(args, uniqueKeyArgs...) - res, err := apl.db.Exec(query, args...) + res, err := apl.checkpointDB().Exec(query, args...) if err != nil { return insertId, err } return res.LastInsertId() } +// moveTablesCheckpointColumns lists the columns of the move-tables checkpoint +// table, in insert order. Run-wide columns are replicated on every per-table row. +var moveTablesCheckpointColumns = []string{ + "gh_ost_chk_timestamp", + "gh_ost_chk_table_name", + "gh_ost_chk_coords", + "gh_ost_chk_iteration", + "gh_ost_rows_copied", + "gh_ost_dml_applied", + "gh_ost_is_cutover", + "gh_ost_move_tables_cutover_started", + "gh_ost_move_tables_drain_gtid", + "gh_ost_chk_range_min", + "gh_ost_chk_range_max", +} + +// WriteMoveTableCheckpoints writes one checkpoint row per migrated table. All +// rows of a single call share the run-wide state (coords, totals, cutover +// markers, drain GTID); each row carries its own table name, iteration, +// rows-copied, and serialized iteration range. The latest row therefore always +// reflects the freshest run-wide state. +func (apl *Applier) WriteMoveTableCheckpoints(rows []*Checkpoint) error { + if len(rows) == 0 { + return nil + } + escaped := make([]string, len(moveTablesCheckpointColumns)) + for i, c := range moveTablesCheckpointColumns { + escaped[i] = sql.EscapeName(c) + } + placeholders := "(" + strings.TrimSuffix(strings.Repeat("?, ", len(moveTablesCheckpointColumns)), ", ") + ")" + query := fmt.Sprintf("insert /* gh-ost */ into %s.%s (%s) values %s", + sql.EscapeName(apl.checkpointDatabaseName()), + sql.EscapeName(apl.migrationContext.GetCheckpointTableName()), + strings.Join(escaped, ", "), + placeholders, + ) + now := time.Now().Unix() + for _, chk := range rows { + coordStr := "" + if chk.LastTrxCoords != nil { + coordStr = chk.LastTrxCoords.String() + } + args := sqlutils.Args( + now, + chk.TableName, + coordStr, + chk.Iteration, + chk.RowsCopied, + chk.DMLApplied, + chk.IsCutover, + chk.MoveTablesCutOverStarted, + apl.checkpointDrainGTIDString(chk), + serializeRangeValues(chk.IterationRangeMin), + serializeRangeValues(chk.IterationRangeMax), + ) + if _, err := apl.checkpointDB().Exec(query, args...); err != nil { + return err + } + } + return nil +} + +// ReadMoveTableCheckpoints returns the latest checkpoint row per migrated table, +// keyed by table name. The per-table iteration range is deserialized using each +// table's unique-key arity (taken from its container), so the move-table +// containers must be populated before calling this. +func (apl *Applier) ReadMoveTableCheckpoints() (map[string]*Checkpoint, error) { + dbName := sql.EscapeName(apl.checkpointDatabaseName()) + tableName := sql.EscapeName(apl.migrationContext.GetCheckpointTableName()) + query := fmt.Sprintf(`select /* gh-ost */ c.gh_ost_chk_id, c.gh_ost_chk_timestamp, c.gh_ost_chk_table_name, c.gh_ost_chk_coords, c.gh_ost_chk_iteration, c.gh_ost_rows_copied, c.gh_ost_dml_applied, c.gh_ost_is_cutover, c.gh_ost_move_tables_cutover_started, c.gh_ost_move_tables_drain_gtid, c.gh_ost_chk_range_min, c.gh_ost_chk_range_max from %s.%s c inner join (select gh_ost_chk_table_name, max(gh_ost_chk_id) as max_id from %s.%s group by gh_ost_chk_table_name) latest on c.gh_ost_chk_table_name = latest.gh_ost_chk_table_name and c.gh_ost_chk_id = latest.max_id`, + dbName, tableName, dbName, tableName) + rows, err := apl.checkpointDB().Query(query) + if err != nil { + return nil, err + } + defer rows.Close() + result := make(map[string]*Checkpoint) + for rows.Next() { + chk := &Checkpoint{} + var tableNameBytes []byte + var coordStr, drainGTIDStr, rangeMinStr, rangeMaxStr string + var timestamp int64 + if err := rows.Scan(&chk.Id, ×tamp, &tableNameBytes, &coordStr, &chk.Iteration, &chk.RowsCopied, &chk.DMLApplied, &chk.IsCutover, &chk.MoveTablesCutOverStarted, &drainGTIDStr, &rangeMinStr, &rangeMaxStr); err != nil { + return nil, err + } + chk.TableName = string(tableNameBytes) + chk.Timestamp = time.Unix(timestamp, 0) + if coordStr != "" { + coords, err := apl.parseCheckpointCoordinates(coordStr) + if err != nil { + return nil, err + } + chk.LastTrxCoords = coords + } + if drainGTIDStr != "" { + drainGTID, err := mysql.NewGTIDBinlogCoordinates(mysql.FlavorFor(apl.migrationContext.InspectorMySQLVersion), drainGTIDStr) + if err != nil { + return nil, err + } + chk.MoveTablesCutOverDrainGTID = drainGTID + } + arity := 0 + if mt := apl.migrationContext.GetMoveTable(chk.TableName); mt != nil && mt.UniqueKey != nil { + arity = mt.UniqueKey.Columns.Len() + } + chk.IterationRangeMin = deserializeRangeValues(rangeMinStr, arity) + chk.IterationRangeMax = deserializeRangeValues(rangeMaxStr, arity) + result[chk.TableName] = chk + } + if err := rows.Err(); err != nil { + return nil, err + } + if len(result) == 0 { + return nil, ErrNoCheckpointFound + } + return result, nil +} + +// parseCheckpointCoordinates parses a stored coordinate string into the binlog +// coordinate family configured for this migration. +func (apl *Applier) parseCheckpointCoordinates(coordStr string) (mysql.BinlogCoordinates, error) { + if apl.migrationContext.UseGTIDs { + return mysql.NewGTIDBinlogCoordinates(mysql.FlavorFor(apl.migrationContext.ApplierMySQLVersion), coordStr) + } + return mysql.ParseFileBinlogCoordinates(coordStr) +} + +// ReadLastCheckpoint reads the most recent standard-mode checkpoint row. In +// move-tables mode use ReadMoveTableCheckpoints instead; calling this there is a +// programmer error (the checkpoint schema is standard-only). func (apl *Applier) ReadLastCheckpoint() (*Checkpoint, error) { - row := apl.db.QueryRow(fmt.Sprintf(`select /* gh-ost */ * from %s.%s order by gh_ost_chk_id desc limit 1`, sql.EscapeName(apl.migrationContext.DatabaseName), sql.EscapeName(apl.migrationContext.GetCheckpointTableName()))) + if apl.migrationContext.IsMoveTablesMode() { + panic("ReadLastCheckpoint() must not be called in move-tables mode; use ReadMoveTableCheckpoints") + } + minColumnNames, maxColumnNames := apl.checkpointRangeColumnNames() + selectColumns := []string{ + "gh_ost_chk_id", + "gh_ost_chk_timestamp", + "gh_ost_chk_coords", + "gh_ost_chk_iteration", + "gh_ost_rows_copied", + "gh_ost_dml_applied", + "gh_ost_is_cutover", + } + selectColumns = append(selectColumns, minColumnNames...) + selectColumns = append(selectColumns, maxColumnNames...) + + row := apl.checkpointDB().QueryRow(fmt.Sprintf( + `select /* gh-ost */ %s from %s.%s order by gh_ost_chk_id desc limit 1`, + strings.Join(selectColumns, ", "), + sql.EscapeName(apl.checkpointDatabaseName()), + sql.EscapeName(apl.migrationContext.GetCheckpointTableName()), + )) chk := &Checkpoint{ IterationRangeMin: sql.NewColumnValues(apl.migrationContext.UniqueKey.Columns.Len()), IterationRangeMax: sql.NewColumnValues(apl.migrationContext.UniqueKey.Columns.Len()), @@ -920,18 +1451,48 @@ func (apl *Applier) ReadLastCheckpoint() (*Checkpoint, error) { return nil, err } chk.Timestamp = time.Unix(timestamp, 0) - if apl.migrationContext.UseGTIDs { - gtidCoords, err := mysql.NewGTIDBinlogCoordinates(mysql.FlavorFor(apl.migrationContext.ApplierMySQLVersion), coordStr) - if err != nil { - return nil, err + coords, err := apl.parseCheckpointCoordinates(coordStr) + if err != nil { + return nil, err + } + chk.LastTrxCoords = coords + return chk, nil +} + +func (apl *Applier) ReadMoveTablesCutOverCheckpoint() (*Checkpoint, error) { + row := apl.checkpointDB().QueryRow(fmt.Sprintf(`select /* gh-ost */ gh_ost_chk_id, gh_ost_chk_timestamp, gh_ost_chk_coords, gh_ost_chk_iteration, gh_ost_rows_copied, gh_ost_dml_applied, gh_ost_is_cutover, gh_ost_move_tables_cutover_started, gh_ost_move_tables_drain_gtid from %s.%s where gh_ost_move_tables_cutover_started = 1 and gh_ost_move_tables_drain_gtid is not null and gh_ost_move_tables_drain_gtid != '' order by gh_ost_chk_id desc limit 1`, sql.EscapeName(apl.checkpointDatabaseName()), sql.EscapeName(apl.migrationContext.GetCheckpointTableName()))) + chk := &Checkpoint{} + var coordStr, drainGTIDStr string + var timestamp int64 + err := row.Scan(&chk.Id, ×tamp, &coordStr, &chk.Iteration, &chk.RowsCopied, &chk.DMLApplied, &chk.IsCutover, &chk.MoveTablesCutOverStarted, &drainGTIDStr) + if err != nil { + if errors.Is(err, gosql.ErrNoRows) { + return nil, ErrNoCheckpointFound } - chk.LastTrxCoords = gtidCoords - } else { - fileCoords, err := mysql.ParseFileBinlogCoordinates(coordStr) + return nil, err + } + chk.Timestamp = time.Unix(timestamp, 0) + if coordStr != "" { + if apl.migrationContext.UseGTIDs { + coords, err := mysql.NewGTIDBinlogCoordinates(mysql.FlavorFor(apl.migrationContext.InspectorMySQLVersion), coordStr) + if err != nil { + return nil, err + } + chk.LastTrxCoords = coords + } else { + coords, err := mysql.ParseFileBinlogCoordinates(coordStr) + if err != nil { + return nil, err + } + chk.LastTrxCoords = coords + } + } + if drainGTIDStr != "" { + drainGTID, err := mysql.NewGTIDBinlogCoordinates(mysql.FlavorFor(apl.migrationContext.InspectorMySQLVersion), drainGTIDStr) if err != nil { return nil, err } - chk.LastTrxCoords = fileCoords + chk.MoveTablesCutOverDrainGTID = drainGTID } return chk, nil } @@ -939,6 +1500,11 @@ func (apl *Applier) ReadLastCheckpoint() (*Checkpoint, error) { // InitiateHeartbeat creates a heartbeat cycle, writing to the changelog table. // Apl is done asynchronously func (apl *Applier) InitiateHeartbeat() { + // In move-tables mode, there is no heartbeat table (§1.2). + if apl.migrationContext.IsMoveTablesMode() { + return + } + var numSuccessiveFailures int64 injectHeartbeat := func() error { if atomic.LoadInt64(&apl.migrationContext.HibernateUntil) > 0 { @@ -1007,8 +1573,7 @@ func (apl *Applier) ExecuteThrottleQuery() (int64, error) { // readMigrationMinValues returns the minimum values to be iterated on rowcopy func (apl *Applier) readMigrationMinValues(tx *gosql.Tx, uniqueKey *sql.UniqueKey) error { - apl.migrationContext.Log.Debugf("Reading migration range according to key: %s", uniqueKey.Name) - query, err := sql.BuildUniqueKeyMinValuesPreparedQuery(apl.migrationContext.DatabaseName, apl.migrationContext.OriginalTableName, uniqueKey) + query, err := sql.BuildUniqueKeyMinValuesPreparedQuery(apl.migrationContext.DatabaseName, apl.originalTableName(), uniqueKey) if err != nil { return err } @@ -1033,7 +1598,7 @@ func (apl *Applier) readMigrationMinValues(tx *gosql.Tx, uniqueKey *sql.UniqueKe // readMigrationMaxValues returns the maximum values to be iterated on rowcopy func (apl *Applier) readMigrationMaxValues(tx *gosql.Tx, uniqueKey *sql.UniqueKey) error { apl.migrationContext.Log.Debugf("Reading migration range according to key: %s", uniqueKey.Name) - query, err := sql.BuildUniqueKeyMaxValuesPreparedQuery(apl.migrationContext.DatabaseName, apl.migrationContext.OriginalTableName, uniqueKey) + query, err := sql.BuildUniqueKeyMaxValuesPreparedQuery(apl.migrationContext.DatabaseName, apl.originalTableName(), uniqueKey) if err != nil { return err } @@ -1072,12 +1637,17 @@ Detail description of the lost data in mysql two-phase commit issue by @Fanduzi: will not be run. When the changelog writes successfully, the ReadMigrationRangeValues will read the newly inserted data, thus Avoiding data loss due to the above problem. */ -func (apl *Applier) ReadMigrationRangeValues() error { +func (apl *Applier) ReadMigrationRangeValues(db *gosql.DB) error { if _, err := apl.WriteChangelogState(string(ReadMigrationRangeValues)); err != nil { return err } - tx, err := apl.db.Begin() + if db == nil { + // default to reading from applier database + db = apl.db + } + + tx, err := db.Begin() if err != nil { return err } @@ -1097,7 +1667,7 @@ func (apl *Applier) ReadMigrationRangeValues() error { // which will be used for copying the next chunk of rows. Ir returns "false" if there is // no further chunk to work through, i.e. we're past the last chunk and are done with // iterating the range (and thus done with copying row chunks) -func (apl *Applier) CalculateNextIterationRangeEndValues() (hasFurtherRange bool, err error) { +func (apl *Applier) CalculateNextIterationRangeEndValues(db *gosql.DB) (hasFurtherRange bool, err error) { for i := 0; i < 2; i++ { buildFunc := sql.BuildUniqueKeyRangeEndPreparedQueryViaOffset if i == 1 { @@ -1105,7 +1675,7 @@ func (apl *Applier) CalculateNextIterationRangeEndValues() (hasFurtherRange bool } query, explodedArgs, err := buildFunc( apl.migrationContext.DatabaseName, - apl.migrationContext.OriginalTableName, + apl.originalTableName(), &apl.migrationContext.UniqueKey.Columns, apl.migrationContext.MigrationIterationRangeMinValues.AbstractValues(), apl.migrationContext.MigrationRangeMaxValues.AbstractValues(), @@ -1118,7 +1688,12 @@ func (apl *Applier) CalculateNextIterationRangeEndValues() (hasFurtherRange bool } queryStartTime := time.Now() - rows, err := apl.db.Query(query, explodedArgs...) + if db == nil { + // default to applier database if not provided + db = apl.db + } + + rows, err := db.Query(query, explodedArgs...) if err != nil { metrics.RecordQueryDuration(apl.migrationContext.Metrics, "source", "range_select", time.Since(queryStartTime), err) return hasFurtherRange, err @@ -1147,15 +1722,122 @@ func (apl *Applier) CalculateNextIterationRangeEndValues() (hasFurtherRange bool return hasFurtherRange, nil } -// ApplyIterationInsertQuery issues a chunk-INSERT query on the ghost table. It is where -// data actually gets copied from original table. +// ReadMoveTableMigrationRangeValues reads the min/max unique-key values for a +// single migrated table into its per-table container. It is the move-tables +// analogue of ReadMigrationRangeValues; each table has its own range. +func (apl *Applier) ReadMoveTableMigrationRangeValues(db *gosql.DB, mt *base.MoveTable) error { + if db == nil { + db = apl.db + } + tx, err := db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + + minQuery, err := sql.BuildUniqueKeyMinValuesPreparedQuery(mt.SourceDatabaseName, mt.SourceTableName, mt.UniqueKey) + if err != nil { + return err + } + if mt.MigrationRangeMinValues, err = apl.scanMoveTableRangeBoundary(tx, minQuery, mt.UniqueKey.Len()); err != nil { + return err + } + + maxQuery, err := sql.BuildUniqueKeyMaxValuesPreparedQuery(mt.SourceDatabaseName, mt.SourceTableName, mt.UniqueKey) + if err != nil { + return err + } + if mt.MigrationRangeMaxValues, err = apl.scanMoveTableRangeBoundary(tx, maxQuery, mt.UniqueKey.Len()); err != nil { + return err + } + + apl.migrationContext.Log.Infof("Move-table %s.%s migration range: [%s]..[%s]", + mt.SourceDatabaseName, mt.SourceTableName, mt.MigrationRangeMinValues, mt.MigrationRangeMaxValues) + return tx.Commit() +} + +// scanMoveTableRangeBoundary runs a single min/max unique-key boundary query and +// returns the scanned values (nil if the table is empty). The result set is +// closed via defer, so each boundary query is fully closed before the next one +// runs on the same transaction. +func (apl *Applier) scanMoveTableRangeBoundary(tx *gosql.Tx, query string, keyLen int) (*sql.ColumnValues, error) { + rows, err := tx.Query(query) + if err != nil { + return nil, err + } + defer rows.Close() + var values *sql.ColumnValues + for rows.Next() { + values = sql.NewColumnValues(keyLen) + if err := rows.Scan(values.ValuesPointers...); err != nil { + return nil, err + } + } + if err := rows.Err(); err != nil { + return nil, err + } + return values, nil +} + +// CalculateMoveTableNextIterationRangeEndValues computes the next chunk's +// range-end for a single migrated table, storing it in the table's container. +// It returns false when the table has no further range to iterate (row copy +// complete for that table). It is the move-tables analogue of +// CalculateNextIterationRangeEndValues. +func (apl *Applier) CalculateMoveTableNextIterationRangeEndValues(db *gosql.DB, mt *base.MoveTable) (hasFurtherRange bool, err error) { + if db == nil { + db = apl.db + } + for i := 0; i < 2; i++ { + buildFunc := sql.BuildUniqueKeyRangeEndPreparedQueryViaOffset + if i == 1 { + buildFunc = sql.BuildUniqueKeyRangeEndPreparedQueryViaTemptable + } + query, explodedArgs, err := buildFunc( + mt.SourceDatabaseName, + mt.SourceTableName, + &mt.UniqueKey.Columns, + mt.MigrationIterationRangeMinValues.AbstractValues(), + mt.MigrationRangeMaxValues.AbstractValues(), + atomic.LoadInt64(&apl.migrationContext.ChunkSize), + mt.GetIteration() == 0, + fmt.Sprintf("iteration:%d", mt.GetIteration()), + ) + if err != nil { + return hasFurtherRange, err + } + + rows, err := db.Query(query, explodedArgs...) + if err != nil { + return hasFurtherRange, err + } + defer rows.Close() + iterationRangeMaxValues := sql.NewColumnValues(mt.UniqueKey.Len()) + for rows.Next() { + if err = rows.Scan(iterationRangeMaxValues.ValuesPointers...); err != nil { + return hasFurtherRange, err + } + hasFurtherRange = true + } + if err = rows.Err(); err != nil { + return hasFurtherRange, err + } + if hasFurtherRange { + mt.MigrationIterationRangeMaxValues = iterationRangeMaxValues + return hasFurtherRange, nil + } + } + apl.migrationContext.Log.Debugf("Move-table %s.%s iteration complete: no further range", mt.SourceDatabaseName, mt.SourceTableName) + return hasFurtherRange, nil +} + func (apl *Applier) ApplyIterationInsertQuery() (chunkSize int64, rowsAffected int64, duration time.Duration, err error) { startTime := time.Now() chunkSize = atomic.LoadInt64(&apl.migrationContext.ChunkSize) query, explodedArgs, err := sql.BuildRangeInsertPreparedQuery( apl.migrationContext.DatabaseName, - apl.migrationContext.OriginalTableName, + apl.originalTableName(), apl.migrationContext.GetGhostTableName(), apl.migrationContext.SharedColumns.Names(), apl.migrationContext.MappedSharedColumns.Names(), @@ -1244,15 +1926,150 @@ func (apl *Applier) ApplyIterationInsertQuery() (chunkSize int64, rowsAffected i return chunkSize, rowsAffected, duration, nil } +// ApplyIterationMoveTableCopyQueries issues a SELECT query on the original table and an INSERT query on the target table, +// copying a chunk of rows. It is used when `--move-tables` is specified, instead of ApplyIterationInsertQuery. +func (apl *Applier) ApplyIterationMoveTableCopyQueries(sourceDB *gosql.DB, mt *base.MoveTable) (chunkSize int64, rowsAffected int64, duration time.Duration, err error) { + startTime := time.Now() + chunkSize = atomic.LoadInt64(&apl.migrationContext.ChunkSize) + if sourceDB == nil { + return chunkSize, rowsAffected, duration, errors.New("source DB is required for move-tables copy") + } + if mt == nil { + return chunkSize, rowsAffected, duration, errors.New("move-table container is required for move-tables copy") + } + builders := apl.moveTablesBuilders[mt.SourceTableName] + if builders == nil { + return chunkSize, rowsAffected, duration, fmt.Errorf("no query builders registered for move-table %s.%s", mt.SourceDatabaseName, mt.SourceTableName) + } + + // First, select data from the source database: + rows, err := func() ([]*sql.ColumnValues, error) { + var qb *sql.MoveTableCopySelectQueryBuilder + if mt.GetIteration() == 0 { + qb = builders.copySelectFirstQueryBuilder + } else { + qb = builders.copySelectNextQueryBuilder + } + query, explodedArgs, err := qb.BuildQuery( + mt.MigrationIterationRangeMinValues.AbstractValues(), + mt.MigrationIterationRangeMaxValues.AbstractValues(), + ) + if err != nil { + return nil, err + } + sqlRows, err := sourceDB.Query(query, explodedArgs...) + if err != nil { + return nil, err + } + defer sqlRows.Close() + chunkRows := make([]*sql.ColumnValues, 0, chunkSize) + for sqlRows.Next() { + row := sql.NewColumnValues(mt.SharedColumns.Len()) + err := sqlRows.Scan(row.ValuesPointers...) + if err != nil { + return nil, err + } + chunkRows = append(chunkRows, row) + } + if rowsErr := sqlRows.Err(); rowsErr != nil { + return nil, rowsErr + } + return chunkRows, nil + }() + if err != nil { + return chunkSize, rowsAffected, duration, err + } + + // no need to INSERT if there are no rows to copy: + if len(rows) == 0 { + duration = time.Since(startTime) + return chunkSize, 0, duration, nil + } + + // Then, insert data into the destination database: + sqlResult, err := func() (gosql.Result, error) { + query, explodedArgs, err := builders.copyInsertQueryBuilder.BuildQuery(rows) + if err != nil { + return nil, err + } + tx, err := apl.moveTablesTargetDB.Begin() + if err != nil { + return nil, err + } + defer tx.Rollback() + + sessionQuery := fmt.Sprintf(`SET SESSION time_zone = '%s', %s`, + apl.migrationContext.ApplierTimeZone, + apl.generateSqlModeQuery()) + if _, err := tx.Exec(sessionQuery); err != nil { + return nil, err + } + + sqlResult, err := tx.Exec(query, explodedArgs...) + if err != nil { + return nil, err + } + + if apl.migrationContext.PanicOnWarnings { + rows, err := tx.Query("SHOW WARNINGS") + if err != nil { + return nil, err + } + defer rows.Close() + if err = rows.Err(); err != nil { + return nil, err + } + migrationKeyRegex, err := compileKeyWarningRegex(mt.TargetTableName, mt.UniqueKey.NameInGhostTable) + if err != nil { + return nil, err + } + var sqlWarnings []string + for rows.Next() { + var level, message string + var code int + if err := rows.Scan(&level, &code, &message); err != nil { + apl.migrationContext.Log.Warningf("Failed to read SHOW WARNINGS row") + continue + } + if strings.Contains(message, "Duplicate entry") && migrationKeyRegex.MatchString(message) { + continue + } + sqlWarnings = append(sqlWarnings, fmt.Sprintf("%s: %s (%d)", level, message, code)) + } + apl.migrationContext.MigrationLastInsertSQLWarnings = sqlWarnings + } + + if err := tx.Commit(); err != nil { + return nil, err + } + return sqlResult, nil + }() + if err != nil { + return chunkSize, rowsAffected, duration, err + } + rowsAffected, _ = sqlResult.RowsAffected() + duration = time.Since(startTime) + apl.migrationContext.Log.Debugf( + "Issued SELECT+INSERT on %s.%s range: [%s]..[%s]; iteration: %d; chunk-size: %d", + mt.SourceDatabaseName, mt.SourceTableName, + mt.MigrationIterationRangeMinValues, + mt.MigrationIterationRangeMaxValues, + mt.GetIteration(), + chunkSize, + ) + + return chunkSize, rowsAffected, duration, nil +} + // LockOriginalTable places a write lock on the original table func (apl *Applier) LockOriginalTable() error { query := fmt.Sprintf(`lock /* gh-ost */ tables %s.%s write`, sql.EscapeName(apl.migrationContext.DatabaseName), - sql.EscapeName(apl.migrationContext.OriginalTableName), + sql.EscapeName(apl.originalTableName()), ) apl.migrationContext.Log.Infof("Locking %s.%s", sql.EscapeName(apl.migrationContext.DatabaseName), - sql.EscapeName(apl.migrationContext.OriginalTableName), + sql.EscapeName(apl.originalTableName()), ) apl.migrationContext.LockTablesStartTime = time.Now() if _, err := sqlutils.ExecNoPrepare(apl.singletonDB, query); err != nil { @@ -1280,7 +2097,7 @@ func (apl *Applier) UnlockTables() error { func (apl *Applier) SwapTablesQuickAndBumpy() error { query := fmt.Sprintf(`alter /* gh-ost */ table %s.%s rename %s`, sql.EscapeName(apl.migrationContext.DatabaseName), - sql.EscapeName(apl.migrationContext.OriginalTableName), + sql.EscapeName(apl.originalTableName()), sql.EscapeName(apl.migrationContext.GetOldTableName()), ) apl.migrationContext.Log.Infof("Renaming original table") @@ -1291,9 +2108,9 @@ func (apl *Applier) SwapTablesQuickAndBumpy() error { query = fmt.Sprintf(`alter /* gh-ost */ table %s.%s rename %s`, sql.EscapeName(apl.migrationContext.DatabaseName), sql.EscapeName(apl.migrationContext.GetGhostTableName()), - sql.EscapeName(apl.migrationContext.OriginalTableName), + sql.EscapeName(apl.originalTableName()), ) - apl.migrationContext.Log.Infof("Renaming ghost table") + apl.migrationContext.Log.Infof("Renaming target table") if _, err := sqlutils.ExecNoPrepare(apl.db, query); err != nil { return err } @@ -1310,13 +2127,13 @@ func (apl *Applier) RenameTablesRollback() (renameError error) { // We prefer the single, atomic operation: query := fmt.Sprintf(`rename /* gh-ost */ table %s.%s to %s.%s, %s.%s to %s.%s`, sql.EscapeName(apl.migrationContext.DatabaseName), - sql.EscapeName(apl.migrationContext.OriginalTableName), + sql.EscapeName(apl.originalTableName()), sql.EscapeName(apl.migrationContext.DatabaseName), sql.EscapeName(apl.migrationContext.GetGhostTableName()), sql.EscapeName(apl.migrationContext.DatabaseName), sql.EscapeName(apl.migrationContext.GetOldTableName()), sql.EscapeName(apl.migrationContext.DatabaseName), - sql.EscapeName(apl.migrationContext.OriginalTableName), + sql.EscapeName(apl.originalTableName()), ) apl.migrationContext.Log.Infof("Renaming back both tables") if _, err := sqlutils.ExecNoPrepare(apl.db, query); err == nil { @@ -1325,7 +2142,7 @@ func (apl *Applier) RenameTablesRollback() (renameError error) { // But, if for some reason the above was impossible to do, we rename one by one. query = fmt.Sprintf(`rename /* gh-ost */ table %s.%s to %s.%s`, sql.EscapeName(apl.migrationContext.DatabaseName), - sql.EscapeName(apl.migrationContext.OriginalTableName), + sql.EscapeName(apl.originalTableName()), sql.EscapeName(apl.migrationContext.DatabaseName), sql.EscapeName(apl.migrationContext.GetGhostTableName()), ) @@ -1337,7 +2154,7 @@ func (apl *Applier) RenameTablesRollback() (renameError error) { sql.EscapeName(apl.migrationContext.DatabaseName), sql.EscapeName(apl.migrationContext.GetOldTableName()), sql.EscapeName(apl.migrationContext.DatabaseName), - sql.EscapeName(apl.migrationContext.OriginalTableName), + sql.EscapeName(apl.originalTableName()), ) apl.migrationContext.Log.Infof("Renaming back to original table") if _, err := sqlutils.ExecNoPrepare(apl.db, query); err != nil { @@ -1586,13 +2403,13 @@ func (apl *Applier) AtomicCutOverMagicLock(sessionIdChan chan int64, tableLocked query = fmt.Sprintf(`lock /* gh-ost */ tables %s.%s write, %s.%s write`, sql.EscapeName(apl.migrationContext.DatabaseName), - sql.EscapeName(apl.migrationContext.OriginalTableName), + sql.EscapeName(apl.originalTableName()), sql.EscapeName(apl.migrationContext.DatabaseName), sql.EscapeName(apl.migrationContext.GetOldTableName()), ) apl.migrationContext.Log.Infof("Locking %s.%s, %s.%s", sql.EscapeName(apl.migrationContext.DatabaseName), - sql.EscapeName(apl.migrationContext.OriginalTableName), + sql.EscapeName(apl.originalTableName()), sql.EscapeName(apl.migrationContext.DatabaseName), sql.EscapeName(apl.migrationContext.GetOldTableName()), ) @@ -1642,7 +2459,7 @@ func (apl *Applier) AtomicCutOverMagicLock(sessionIdChan chan int64, tableLocked // Tables still locked apl.migrationContext.Log.Infof("Releasing lock from %s.%s, %s.%s", sql.EscapeName(apl.migrationContext.DatabaseName), - sql.EscapeName(apl.migrationContext.OriginalTableName), + sql.EscapeName(apl.originalTableName()), sql.EscapeName(apl.migrationContext.DatabaseName), sql.EscapeName(apl.migrationContext.GetOldTableName()), ) @@ -1681,13 +2498,13 @@ func (apl *Applier) AtomicCutoverRename(sessionIdChan chan int64, tablesRenamed query = fmt.Sprintf(`rename /* gh-ost */ table %s.%s to %s.%s, %s.%s to %s.%s`, sql.EscapeName(apl.migrationContext.DatabaseName), - sql.EscapeName(apl.migrationContext.OriginalTableName), + sql.EscapeName(apl.originalTableName()), sql.EscapeName(apl.migrationContext.DatabaseName), sql.EscapeName(apl.migrationContext.GetOldTableName()), sql.EscapeName(apl.migrationContext.DatabaseName), sql.EscapeName(apl.migrationContext.GetGhostTableName()), sql.EscapeName(apl.migrationContext.DatabaseName), - sql.EscapeName(apl.migrationContext.OriginalTableName), + sql.EscapeName(apl.originalTableName()), ) apl.migrationContext.Log.Infof("Issuing and expecting this to block: %s", query) if _, err := tx.Exec(query); err != nil { @@ -1700,8 +2517,12 @@ func (apl *Applier) AtomicCutoverRename(sessionIdChan chan int64, tablesRenamed } func (apl *Applier) ShowStatusVariable(variableName string) (result int64, err error) { + targetDB := apl.db + if apl.migrationContext.IsMoveTablesMode() { + targetDB = apl.moveTablesTargetDB + } query := fmt.Sprintf(`show /* gh-ost */ global status like '%s'`, variableName) - if err := apl.db.QueryRow(query).Scan(&variableName, &result); err != nil { + if err := targetDB.QueryRow(query).Scan(&variableName, &result); err != nil { return 0, err } return result, nil @@ -1710,9 +2531,9 @@ func (apl *Applier) ShowStatusVariable(variableName string) (result int64, err e // updateModifiesUniqueKeyColumns checks whether a UPDATE DML event actually // modifies values of the migration's unique key (the iterated key). This will call // for special handling. -func (apl *Applier) updateModifiesUniqueKeyColumns(dmlEvent *binlog.BinlogDMLEvent) (modifiedColumn string, isModified bool) { - for _, column := range apl.migrationContext.UniqueKey.Columns.Columns() { - tableOrdinal := apl.migrationContext.OriginalTableColumns.Ordinals[column.Name] +func (apl *Applier) updateModifiesUniqueKeyColumns(dmlEvent *binlog.BinlogDMLEvent, uniqueKey *sql.UniqueKey, originalTableColumns *sql.ColumnList) (modifiedColumn string, isModified bool) { + for _, column := range uniqueKey.Columns.Columns() { + tableOrdinal := originalTableColumns.Ordinals[column.Name] whereColumnValue := dmlEvent.WhereColumnValues.AbstractValues()[tableOrdinal] newColumnValue := dmlEvent.NewColumnValues.AbstractValues()[tableOrdinal] @@ -1726,20 +2547,41 @@ func (apl *Applier) updateModifiesUniqueKeyColumns(dmlEvent *binlog.BinlogDMLEve // buildDMLEventQuery creates a query to operate on the ghost table, based on an intercepted binlog // event entry on the original table. func (apl *Applier) buildDMLEventQuery(dmlEvent *binlog.BinlogDMLEvent) []*dmlBuildResult { + // Resolve the query builders + schema for the table this event targets. In + // move-tables mode the set is selected by source table name (one binlog + // stream feeds every table; routing happens here, §2.1). In standard mode + // there is a single set on the applier. + deleteBuilder := apl.dmlDeleteQueryBuilder + insertBuilder := apl.dmlInsertQueryBuilder + updateBuilder := apl.dmlUpdateQueryBuilder + uniqueKey := apl.migrationContext.UniqueKey + originalTableColumns := apl.migrationContext.OriginalTableColumns + if apl.migrationContext.IsMoveTablesMode() { + b := apl.moveTablesBuilders[dmlEvent.TableName] + if b == nil { + return []*dmlBuildResult{newDmlBuildResultError(fmt.Errorf("no query builder registered for move-table %s.%s", dmlEvent.DatabaseName, dmlEvent.TableName))} + } + deleteBuilder = b.dmlDeleteQueryBuilder + insertBuilder = b.dmlInsertQueryBuilder + updateBuilder = b.dmlUpdateQueryBuilder + uniqueKey = b.uniqueKey + originalTableColumns = b.originalTableColumns + } + switch dmlEvent.DML { case binlog.DeleteDML: { - query, uniqueKeyArgs, err := apl.dmlDeleteQueryBuilder.BuildQuery(dmlEvent.WhereColumnValues.AbstractValues()) + query, uniqueKeyArgs, err := deleteBuilder.BuildQuery(dmlEvent.WhereColumnValues.AbstractValues()) return []*dmlBuildResult{newDmlBuildResult(query, uniqueKeyArgs, -1, err)} } case binlog.InsertDML: { - query, sharedArgs, err := apl.dmlInsertQueryBuilder.BuildQuery(dmlEvent.NewColumnValues.AbstractValues()) + query, sharedArgs, err := insertBuilder.BuildQuery(dmlEvent.NewColumnValues.AbstractValues()) return []*dmlBuildResult{newDmlBuildResult(query, sharedArgs, 1, err)} } case binlog.UpdateDML: { - if _, isModified := apl.updateModifiesUniqueKeyColumns(dmlEvent); isModified { + if _, isModified := apl.updateModifiesUniqueKeyColumns(dmlEvent, uniqueKey, originalTableColumns); isModified { results := make([]*dmlBuildResult, 0, 2) dmlEvent.DML = binlog.DeleteDML results = append(results, apl.buildDMLEventQuery(dmlEvent)...) @@ -1747,7 +2589,7 @@ func (apl *Applier) buildDMLEventQuery(dmlEvent *binlog.BinlogDMLEvent) []*dmlBu results = append(results, apl.buildDMLEventQuery(dmlEvent)...) return results } - query, updateArgs, err := apl.dmlUpdateQueryBuilder.BuildQuery(dmlEvent.NewColumnValues.AbstractValues(), dmlEvent.WhereColumnValues.AbstractValues()) + query, updateArgs, err := updateBuilder.BuildQuery(dmlEvent.NewColumnValues.AbstractValues(), dmlEvent.WhereColumnValues.AbstractValues()) args := sqlutils.Args() args = append(args, updateArgs...) return []*dmlBuildResult{newDmlBuildResult(query, args, 0, err)} @@ -1863,7 +2705,11 @@ func (apl *Applier) ApplyDMLEventQueries(dmlEvents [](*binlog.BinlogDMLEvent)) e ctx := context.Background() err := func() error { - conn, err := apl.db.Conn(ctx) + db := apl.db + if apl.migrationContext.IsMoveTablesMode() { + db = apl.moveTablesTargetDB + } + conn, err := db.Conn(ctx) if err != nil { return err } @@ -1972,6 +2818,9 @@ func (apl *Applier) Teardown() { apl.releaseMigrationLock() apl.db.Close() apl.singletonDB.Close() + if apl.moveTablesTargetDB != nil { + apl.moveTablesTargetDB.Close() + } atomic.StoreInt64(&apl.finishedMigrating, 1) } @@ -1988,12 +2837,12 @@ func (apl *Applier) ExpectMetadataLock(sessionId int64) error { err := sqlutils.QueryRowsMap(apl.db, query, func(m sqlutils.RowMap) error { found = true return nil - }, apl.migrationContext.DatabaseName, apl.migrationContext.OriginalTableName, sessionId) + }, apl.migrationContext.DatabaseName, apl.originalTableName(), sessionId) if err != nil { return err } if !found { - err = fmt.Errorf("cannot find PENDING metadata lock on original table: `%s`.`%s`", apl.migrationContext.DatabaseName, apl.migrationContext.OriginalTableName) + err = fmt.Errorf("cannot find PENDING metadata lock on original table: `%s`.`%s`", apl.migrationContext.DatabaseName, apl.originalTableName()) return apl.migrationContext.Log.Errore(err) } return nil diff --git a/go/logic/applier_test.go b/go/logic/applier_test.go index f1fa28bc8..6812f6339 100644 --- a/go/logic/applier_test.go +++ b/go/logic/applier_test.go @@ -9,6 +9,7 @@ import ( "context" gosql "database/sql" "errors" + "net" "strings" "testing" "time" @@ -83,7 +84,7 @@ func TestApplierUpdateModifiesUniqueKeyColumns(t *testing.T) { DML: binlog.UpdateDML, NewColumnValues: columnValues, WhereColumnValues: columnValues, - }) + }, migrationContext.UniqueKey, migrationContext.OriginalTableColumns) require.Equal(t, "", modifiedColumn) require.False(t, isModified) }) @@ -94,7 +95,7 @@ func TestApplierUpdateModifiesUniqueKeyColumns(t *testing.T) { DML: binlog.UpdateDML, NewColumnValues: sql.ToColumnValues([]interface{}{123456, 24}), WhereColumnValues: columnValues, - }) + }, migrationContext.UniqueKey, migrationContext.OriginalTableColumns) require.Equal(t, "item_id", modifiedColumn) require.True(t, isModified) }) @@ -338,6 +339,7 @@ type ApplierTestSuite struct { mysqlContainer testcontainers.Container db *gosql.DB + otherDB *gosql.DB } func (suite *ApplierTestSuite) SetupSuite() { @@ -358,12 +360,30 @@ func (suite *ApplierTestSuite) SetupSuite() { db, err := gosql.Open("mysql", dsn) suite.Require().NoError(err) - suite.db = db + + containerHost, err := mysqlContainer.Host(ctx) + suite.Require().NoError(err) + containerPort, err := mysqlContainer.MappedPort(ctx, "3306/tcp") + suite.Require().NoError(err) + + // Second database & connection for move-tables tests: + _, err = suite.db.ExecContext(ctx, fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s", testMysqlDatabaseOther)) + suite.Require().NoError(err) + otherConf := drivermysql.NewConfig() + otherConf.DBName = testMysqlDatabaseOther + otherConf.User = testMysqlUser + otherConf.Passwd = testMysqlPass + otherConf.Net = "tcp" + otherConf.Addr = net.JoinHostPort(containerHost, containerPort.Port()) + otherDB, err := gosql.Open("mysql", otherConf.FormatDSN()) + suite.Require().NoError(err) + suite.otherDB = otherDB } func (suite *ApplierTestSuite) TearDownSuite() { suite.Assert().NoError(suite.db.Close()) + suite.Assert().NoError(suite.otherDB.Close()) suite.Assert().NoError(testcontainers.TerminateContainer(suite.mysqlContainer)) } @@ -380,6 +400,10 @@ func (suite *ApplierTestSuite) TearDownTest() { suite.Require().NoError(err) _, err = suite.db.ExecContext(ctx, "DROP TABLE IF EXISTS "+getTestGhostTableName()) suite.Require().NoError(err) + _, err = suite.otherDB.ExecContext(ctx, "DROP TABLE IF EXISTS "+getTestOtherTableName()) + suite.Require().NoError(err) + _, err = suite.db.ExecContext(ctx, fmt.Sprintf("DROP TABLE IF EXISTS `%s`.`_%s_ghc`", testMysqlDatabase, testMysqlTableName)) + suite.Require().NoError(err) } func (suite *ApplierTestSuite) TestInitDBConnections() { @@ -411,6 +435,41 @@ func (suite *ApplierTestSuite) TestInitDBConnections() { suite.Require().Equal(sql.NewColumnList([]string{"id", "item_id"}), migrationContext.OriginalTableColumnsOnApplier) } +func (suite *ApplierTestSuite) TestInitiateApplierMoveTablesMode_NoGhostOrChangelogTable() { + ctx := context.Background() + + var err error + _, err = suite.db.ExecContext(ctx, fmt.Sprintf("CREATE TABLE %s (id INT, item_id INT);", getTestTableName())) + suite.Require().NoError(err) + + connectionConfig, err := getTestConnectionConfig(ctx, suite.mysqlContainer) + suite.Require().NoError(err) + + migrationContext := newTestMigrationContext() + migrationContext.MoveTables.TableNames = []string{testMysqlTableName} + migrationContext.MoveTables.TargetDatabase = testMysqlDatabaseOther + migrationContext.ApplierConnectionConfig = connectionConfig + migrationContext.MoveTables.ConnectionConfig = connectionConfig + migrationContext.SetConnectionConfig("innodb") + + applier := NewApplier(migrationContext) + defer applier.Teardown() + + migrationContext.OriginalTableColumns = sql.NewColumnList([]string{"id", "item_id"}) + + err = applier.InitDBConnections() + suite.Require().NoError(err) + + // #8206 [Task] [1.2] Skip ghost/changelog tables, heartbeat in gh-ost move-tables mode + // In move-tables mode, no ghost or changelog table should exist. + // InitDBConnections() should succeed without them. + suite.Require().False(applier.tableExists("_testing_gho"), "ghost table should not exist in move-tables mode") + suite.Require().False(applier.tableExists("_testing_ghc"), "changelog table should not exist in move-tables mode") + + // In move-tables mode, OriginalTableColumnsOnApplier is unused and intentionally never populated + suite.Require().Nil(migrationContext.OriginalTableColumnsOnApplier) +} + func (suite *ApplierTestSuite) TestApplyDMLEventQueries() { ctx := context.Background() @@ -476,6 +535,65 @@ func (suite *ApplierTestSuite) TestApplyDMLEventQueries() { suite.Require().Equal(int64(0), migrationContext.RowsDeltaEstimate) } +// finalCleanup() requires a fully wired migrator to call directly. +// This test verifies the IsMoveTablesMode() predicate that gates the early return. +// Full behavioral coverage relies on the suite: no ghost/changelog tables are +// created (Test #1), and WriteChangelog is a no-op (Test #2). +func (suite *ApplierTestSuite) TestFinalCleanupMoveTablesMode_SkipsDrops() { + migrationContext := newTestMigrationContext() + migrationContext.MoveTables.TableNames = []string{testMysqlTableName} + migrationContext.MoveTables.TargetDatabase = testMysqlDatabaseOther + + suite.Require().True(migrationContext.IsMoveTablesMode()) +} + +// initiateStreaming() requires a binlog-capable MySQL connection to call directly. +// This test verifies IsMoveTablesMode() and that no changelog table is referenced +// in move-tables mode (§1.2): no `_ghc` table exists on the source or target +// database. A new streamer always starts with zero listeners; the real proof that +// no changelog listener is registered comes from the full run not failing on a +// nonexistent _ghc table. +func (suite *ApplierTestSuite) TestInitiateStreamingMoveTablesMode_NoChangelogListener() { + ctx := context.Background() + migrationContext := newTestMigrationContext() + migrationContext.MoveTables.TableNames = []string{testMysqlTableName} + migrationContext.MoveTables.TargetDatabase = testMysqlDatabaseOther + + suite.Require().True(migrationContext.IsMoveTablesMode()) + + // In move-tables mode there is no changelog table. Verify none exists on + // either the source or target database (LIKE '%\_ghc' matches a literal + // trailing "_ghc"). + for _, schema := range []string{testMysqlDatabase, testMysqlDatabaseOther} { + var count int + err := suite.db.QueryRowContext(ctx, + `SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = ? AND table_name LIKE '%\_ghc'`, + schema, + ).Scan(&count) + suite.Require().NoError(err) + suite.Require().Equal(0, count, "no changelog (_ghc) table should exist in move-tables mode in schema %s", schema) + } + + streamer := NewEventsStreamer(migrationContext) + suite.Require().Empty(streamer.listeners, "new streamer should have no listeners") +} + +// initiateApplier() requires a full migrator to call directly. +// This test verifies the IsMoveTablesMode() predicate that gates InitiateHeartbeat(). +// Even if heartbeat ran, TestWriteChangelogNoOpInMoveTablesMode proves WriteChangelog +// is a no-op, so no SQL would execute against a nonexistent changelog table. +// +// A stronger test would instrument InitiateHeartbeat() (e.g., via a callback or +// channel) to assert the goroutine is never started. That requires test-infrastructure +// changes to the Applier and is beyond #8206's scope. +func (suite *ApplierTestSuite) TestNoHeartbeatInMoveTablesMode() { + migrationContext := newTestMigrationContext() + migrationContext.MoveTables.TableNames = []string{testMysqlTableName} + migrationContext.MoveTables.TargetDatabase = testMysqlDatabaseOther + + suite.Require().True(migrationContext.IsMoveTablesMode()) +} + func (suite *ApplierTestSuite) TestValidateOrDropExistingTables() { ctx := context.Background() @@ -574,6 +692,43 @@ func (suite *ApplierTestSuite) TestValidateOrDropExistingTablesWithGhostTableExi suite.Require().Equal(gosql.ErrNoRows, err) } +func (suite *ApplierTestSuite) TestWriteChangelogNoOpInMoveTablesMode() { + ctx := context.Background() + + var err error + _, err = suite.db.ExecContext(ctx, fmt.Sprintf("CREATE TABLE %s (id INT, item_id INT);", getTestTableName())) + suite.Require().NoError(err) + + connectionConfig, err := getTestConnectionConfig(ctx, suite.mysqlContainer) + suite.Require().NoError(err) + + migrationContext := newTestMigrationContext() + migrationContext.MoveTables.TableNames = []string{testMysqlTableName} + migrationContext.MoveTables.TargetDatabase = testMysqlDatabaseOther + migrationContext.ApplierConnectionConfig = connectionConfig + migrationContext.MoveTables.ConnectionConfig = connectionConfig + migrationContext.SetConnectionConfig("innodb") + migrationContext.OriginalTableColumns = sql.NewColumnList([]string{"id", "item_id"}) + + applier := NewApplier(migrationContext) + defer applier.Teardown() + + err = applier.InitDBConnections() + suite.Require().NoError(err) + + // #8206 [Task] [1.2] Skip ghost/changelog tables, heartbeat in gh-ost move-tables mode + // WriteChangelog should be a no-op in move-tables mode. + // No changelog table exists, so if it tried to execute, it would fail. + hint, err := applier.WriteChangelog("heartbeat", "2026-06-05T00:00:00Z") + suite.Require().NoError(err) + suite.Require().Empty(hint) + + // Also verify state writes are no-ops + hint, err = applier.WriteChangelogState("Migrated") + suite.Require().NoError(err) + suite.Require().Equal("", hint) +} + func (suite *ApplierTestSuite) TestAcquireMigrationLockSucceedsWhenFree() { ctx := context.Background() @@ -736,6 +891,114 @@ func (suite *ApplierTestSuite) TestAnalyzeGhostTable() { suite.Require().ErrorContains(applier.AnalyzeGhostTable(), "failed; refusing cut-over") } +// TestCreateTargetTable_HappyPath exercises #8207 AC #1: +// "A move table run targeting a clean target schema creates the migrated table +// with a SHOW CREATE TABLE output equivalent to the source's." +// +// It calls CreateTargetTable (not just IsMoveTablesMode()), asserts the table +// exists on the target database, verifies schema equivalence via SHOW CREATE TABLE, +// and confirms no table was accidentally created on the source. +func (suite *ApplierTestSuite) TestCreateTargetTable_HappyPath() { + ctx := context.Background() + + _, err := suite.db.ExecContext(ctx, fmt.Sprintf("CREATE TABLE %s (id INT PRIMARY KEY, name VARCHAR(64), updated_at DATETIME);", getTestTableName())) + suite.Require().NoError(err) + + connectionConfig, err := getTestConnectionConfig(ctx, suite.mysqlContainer) + suite.Require().NoError(err) + + migrationContext := newTestMigrationContext() + migrationContext.MoveTables.TableNames = []string{testMysqlTableName} + migrationContext.MoveTables.TargetDatabase = testMysqlDatabaseOther + migrationContext.ApplierConnectionConfig = connectionConfig + migrationContext.MoveTables.ConnectionConfig = connectionConfig + migrationContext.SetConnectionConfig("innodb") + migrationContext.OriginalTableColumns = sql.NewColumnList([]string{"id", "name", "updated_at"}) + + applier := NewApplier(migrationContext) + defer applier.Teardown() + + err = applier.InitDBConnections() + suite.Require().NoError(err) + + var dummy, sourceCreateDDL string + err = suite.db.QueryRow(fmt.Sprintf("SHOW CREATE TABLE %s", getTestTableName())).Scan(&dummy, &sourceCreateDDL) + suite.Require().NoError(err) + + var count int + err = suite.otherDB.QueryRow( + "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema=? AND table_name=?", + testMysqlDatabaseOther, testMysqlTableName, + ).Scan(&count) + suite.Require().NoError(err) + suite.Require().Equal(0, count, "precondition: target table must not exist before CreateTargetTable") + + err = applier.CreateTargetTableForName(testMysqlTableName, sourceCreateDDL) + suite.Require().NoError(err) + + var targetTableName, targetCreateDDL string + err = suite.otherDB.QueryRow(fmt.Sprintf("SHOW CREATE TABLE `%s`.`%s`", testMysqlDatabaseOther, testMysqlTableName)).Scan(&targetTableName, &targetCreateDDL) + suite.Require().NoError(err) + suite.Require().Equal(testMysqlTableName, targetTableName) + suite.Require().Equal(sourceCreateDDL, targetCreateDDL, "target table schema must be equivalent to source") + + err = suite.otherDB.QueryRow( + "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema=? AND table_name=?", + testMysqlDatabaseOther, testMysqlTableName, + ).Scan(&count) + suite.Require().NoError(err) + suite.Require().Equal(1, count, "target table must exist exactly once on the target database") +} + +// TestCreateTargetTable_AbortsIfExists exercises #8207 AC #2: +// "A move table run aborts before any data is copied if the target table already exists." +// +// It pre-creates the target table, then calls CreateTargetTable and asserts it +// returns a descriptive error (not just MySQL's raw ERROR 1050). +func (suite *ApplierTestSuite) TestCreateTargetTable_AbortsIfExists() { + ctx := context.Background() + + _, err := suite.db.ExecContext(ctx, fmt.Sprintf("CREATE TABLE %s (id INT PRIMARY KEY);", getTestTableName())) + suite.Require().NoError(err) + + _, err = suite.otherDB.ExecContext(ctx, fmt.Sprintf("CREATE TABLE `%s`.`%s` (id INT PRIMARY KEY);", testMysqlDatabaseOther, testMysqlTableName)) + suite.Require().NoError(err) + + var count int + err = suite.otherDB.QueryRow( + "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema=? AND table_name=?", + testMysqlDatabaseOther, testMysqlTableName, + ).Scan(&count) + suite.Require().NoError(err) + suite.Require().Equal(1, count, "precondition: target table must exist before CreateTargetTable") + + connectionConfig, err := getTestConnectionConfig(ctx, suite.mysqlContainer) + suite.Require().NoError(err) + + migrationContext := newTestMigrationContext() + migrationContext.MoveTables.TableNames = []string{testMysqlTableName} + migrationContext.MoveTables.TargetDatabase = testMysqlDatabaseOther + migrationContext.ApplierConnectionConfig = connectionConfig + migrationContext.MoveTables.ConnectionConfig = connectionConfig + migrationContext.SetConnectionConfig("innodb") + migrationContext.OriginalTableColumns = sql.NewColumnList([]string{"id"}) + + applier := NewApplier(migrationContext) + defer applier.Teardown() + + err = applier.InitDBConnections() + suite.Require().NoError(err) + + var dummy, sourceCreateDDL string + err = suite.db.QueryRow(fmt.Sprintf("SHOW CREATE TABLE %s", getTestTableName())).Scan(&dummy, &sourceCreateDDL) + suite.Require().NoError(err) + + err = applier.CreateTargetTableForName(testMysqlTableName, sourceCreateDDL) + suite.Require().Error(err, "CreateTargetTable must return an error when target table already exists") + suite.Require().Contains(err.Error(), "already exists", "error message must mention 'already exists'") + suite.Require().Contains(err.Error(), testMysqlTableName, "error message must name the table") +} + func (suite *ApplierTestSuite) TestPanicOnWarningsInApplyIterationInsertQuerySucceedsWithUniqueKeyWarningInsertedByDMLEvent() { ctx := context.Background() @@ -788,11 +1051,11 @@ func (suite *ApplierTestSuite) TestPanicOnWarningsInApplyIterationInsertQuerySuc err = applier.CreateChangelogTable() suite.Require().NoError(err) - err = applier.ReadMigrationRangeValues() + err = applier.ReadMigrationRangeValues(nil) suite.Require().NoError(err) migrationContext.SetNextIterationRangeMinValues() - hasFurtherRange, err := applier.CalculateNextIterationRangeEndValues() + hasFurtherRange, err := applier.CalculateNextIterationRangeEndValues(nil) suite.Require().NoError(err) suite.Require().True(hasFurtherRange) @@ -865,14 +1128,14 @@ func (suite *ApplierTestSuite) TestPanicOnWarningsInApplyIterationInsertQueryFai err = applier.CreateChangelogTable() suite.Require().NoError(err) - err = applier.ReadMigrationRangeValues() + err = applier.ReadMigrationRangeValues(nil) suite.Require().NoError(err) err = applier.AlterGhost() suite.Require().NoError(err) migrationContext.SetNextIterationRangeMinValues() - hasFurtherRange, err := applier.CalculateNextIterationRangeEndValues() + hasFurtherRange, err := applier.CalculateNextIterationRangeEndValues(nil) suite.Require().NoError(err) suite.Require().True(hasFurtherRange) @@ -938,7 +1201,7 @@ func (suite *ApplierTestSuite) TestWriteCheckpoint() { err = applier.prepareQueries() suite.Require().NoError(err) - err = applier.ReadMigrationRangeValues() + err = applier.ReadMigrationRangeValues(nil) suite.Require().NoError(err) // checkpoint table is empty @@ -971,6 +1234,175 @@ func (suite *ApplierTestSuite) TestWriteCheckpoint() { suite.Require().Equal(chk.RowsCopied, gotChk.RowsCopied) suite.Require().Equal(chk.DMLApplied, gotChk.DMLApplied) suite.Require().Equal(chk.IsCutover, gotChk.IsCutover) + suite.Require().False(gotChk.MoveTablesCutOverStarted) + suite.Require().Nil(gotChk.MoveTablesCutOverDrainGTID) +} + +func (suite *ApplierTestSuite) TestWriteCheckpointMoveTables() { + ctx := context.Background() + + var err error + + _, err = suite.db.ExecContext(ctx, fmt.Sprintf("CREATE TABLE %s (id int not null, id2 char(4) CHARACTER SET utf8mb4, primary key(id, id2))", getTestTableName())) + suite.Require().NoError(err) + + _, err = suite.db.ExecContext(ctx, fmt.Sprintf("INSERT INTO %s (id, id2) VALUES (?,?), (?,?), (?,?)", getTestTableName()), 411, "君子懷德", 411, "小人懷土", 212, "君子不器") + suite.Require().NoError(err) + + connectionConfig, err := getTestConnectionConfig(ctx, suite.mysqlContainer) + suite.Require().NoError(err) + + migrationContext := newTestMigrationContext() + migrationContext.ApplierConnectionConfig = connectionConfig + migrationContext.InspectorConnectionConfig = connectionConfig + migrationContext.SetConnectionConfig("innodb") + migrationContext.UseGTIDs = true + + migrationContext.OriginalTableColumns = sql.NewColumnList([]string{"id", "id2"}) + migrationContext.SharedColumns = sql.NewColumnList([]string{"id", "id2"}) + migrationContext.MappedSharedColumns = sql.NewColumnList([]string{"id", "id2"}) + migrationContext.Checkpoint = true + migrationContext.MoveTables.TableNames = []string{testMysqlTableName} + migrationContext.MoveTables.TargetDatabase = testMysqlDatabase + migrationContext.MoveTables.ConnectionConfig = connectionConfig + migrationContext.UniqueKey = &sql.UniqueKey{ + Name: "PRIMARY", + NameInGhostTable: "PRIMARY", + Columns: *sql.NewColumnList([]string{"id", "id2"}), + } + + // Populate the per-table container the move-tables checkpoint path operates on. + migrationContext.InitMoveTableContainers() + mt := migrationContext.GetMoveTable(testMysqlTableName) + suite.Require().NotNil(mt) + mt.OriginalTableColumns = migrationContext.OriginalTableColumns + mt.SharedColumns = migrationContext.SharedColumns + mt.MappedSharedColumns = migrationContext.MappedSharedColumns + mt.UniqueKey = migrationContext.UniqueKey + + inspector := NewInspector(migrationContext) + suite.Require().NoError(inspector.InitDBConnections()) + + applier := NewApplier(migrationContext) + + err = applier.InitDBConnections() + suite.Require().NoError(err) + + err = applier.CreateCheckpointTable() + suite.Require().NoError(err) + + err = applier.prepareQueries() + suite.Require().NoError(err) + + err = applier.ReadMoveTableMigrationRangeValues(inspector.db, mt) + suite.Require().NoError(err) + + coords, err := mysql.NewGTIDBinlogCoordinates(mysql.MySQLFlavor, "00000000-0000-0000-0000-000000000001:1-10") + suite.Require().NoError(err) + drainGTID, err := mysql.NewGTIDBinlogCoordinates(mysql.MySQLFlavor, "00000000-0000-0000-0000-000000000001:1-20") + suite.Require().NoError(err) + + chk := &Checkpoint{ + TableName: testMysqlTableName, + LastTrxCoords: coords, + IterationRangeMin: mt.MigrationRangeMinValues, + IterationRangeMax: mt.MigrationRangeMaxValues, + Iteration: 3, + RowsCopied: 1000, + DMLApplied: 2000, + IsCutover: false, + MoveTablesCutOverStarted: true, + MoveTablesCutOverDrainGTID: drainGTID, + } + err = applier.WriteMoveTableCheckpoints([]*Checkpoint{chk}) + suite.Require().NoError(err) + + gotCheckpoints, err := applier.ReadMoveTableCheckpoints() + suite.Require().NoError(err) + gotChk := gotCheckpoints[testMysqlTableName] + suite.Require().NotNil(gotChk) + + suite.Require().Equal(chk.Iteration, gotChk.Iteration) + suite.Require().Equal(chk.LastTrxCoords.String(), gotChk.LastTrxCoords.String()) + // The fresh read yields typed values (e.g. int -> "212") while the checkpoint + // round-trips them as []byte (-> hex "323132"). Both serialize identically and + // are used identically as prepared-statement args on resume, so compare the + // serialized (resumable) form rather than the typed String() rendering. + suite.Require().Equal(serializeRangeValues(chk.IterationRangeMin), serializeRangeValues(gotChk.IterationRangeMin)) + suite.Require().Equal(serializeRangeValues(chk.IterationRangeMax), serializeRangeValues(gotChk.IterationRangeMax)) + suite.Require().Equal(chk.RowsCopied, gotChk.RowsCopied) + suite.Require().Equal(chk.DMLApplied, gotChk.DMLApplied) + suite.Require().Equal(chk.IsCutover, gotChk.IsCutover) + suite.Require().True(gotChk.MoveTablesCutOverStarted) + suite.Require().NotNil(gotChk.MoveTablesCutOverDrainGTID) + suite.Require().Equal(drainGTID.String(), gotChk.MoveTablesCutOverDrainGTID.String()) +} + +func (suite *ApplierTestSuite) TestReadMoveTablesCutOverCheckpointIgnoresRowCopyCheckpoints() { + ctx := context.Background() + + var err error + + _, err = suite.db.ExecContext(ctx, fmt.Sprintf("CREATE TABLE %s (id int not null primary key)", getTestTableName())) + suite.Require().NoError(err) + + _, err = suite.db.ExecContext(ctx, fmt.Sprintf("INSERT INTO %s (id) VALUES (1), (2), (3)", getTestTableName())) + suite.Require().NoError(err) + + connectionConfig, err := getTestConnectionConfig(ctx, suite.mysqlContainer) + suite.Require().NoError(err) + + migrationContext := newTestMigrationContext() + migrationContext.ApplierConnectionConfig = connectionConfig + migrationContext.InspectorConnectionConfig = connectionConfig + migrationContext.SetConnectionConfig("innodb") + migrationContext.Checkpoint = true + migrationContext.MoveTables.TableNames = []string{testMysqlTableName} + migrationContext.MoveTables.TargetDatabase = testMysqlDatabase + migrationContext.MoveTables.ConnectionConfig = connectionConfig + migrationContext.OriginalTableColumns = sql.NewColumnList([]string{"id"}) + migrationContext.SharedColumns = sql.NewColumnList([]string{"id"}) + migrationContext.MappedSharedColumns = sql.NewColumnList([]string{"id"}) + migrationContext.UniqueKey = &sql.UniqueKey{ + Name: "PRIMARY", + NameInGhostTable: "PRIMARY", + Columns: *sql.NewColumnList([]string{"id"}), + } + + migrationContext.InitMoveTableContainers() + mt := migrationContext.GetMoveTable(testMysqlTableName) + suite.Require().NotNil(mt) + mt.OriginalTableColumns = migrationContext.OriginalTableColumns + mt.SharedColumns = migrationContext.SharedColumns + mt.MappedSharedColumns = migrationContext.MappedSharedColumns + mt.UniqueKey = migrationContext.UniqueKey + + inspector := NewInspector(migrationContext) + suite.Require().NoError(inspector.InitDBConnections()) + + applier := NewApplier(migrationContext) + suite.Require().NoError(applier.InitDBConnections()) + suite.Require().NoError(applier.CreateCheckpointTable()) + suite.Require().NoError(applier.prepareQueries()) + suite.Require().NoError(applier.ReadMoveTableMigrationRangeValues(inspector.db, mt)) + + coords := mysql.NewFileBinlogCoordinates("mysql-bin.000003", int64(1234)) + // A row-copy checkpoint: cutover has not started, so the cutover-resume read + // must ignore it. + chk := &Checkpoint{ + TableName: testMysqlTableName, + LastTrxCoords: coords, + IterationRangeMin: mt.MigrationRangeMinValues, + IterationRangeMax: mt.MigrationRangeMaxValues, + Iteration: 1, + RowsCopied: 3, + DMLApplied: 0, + } + err = applier.WriteMoveTableCheckpoints([]*Checkpoint{chk}) + suite.Require().NoError(err) + + _, err = applier.ReadMoveTablesCutOverCheckpoint() + suite.Require().ErrorIs(err, ErrNoCheckpointFound) } func (suite *ApplierTestSuite) TestPanicOnWarningsWithDuplicateKeyOnNonMigrationIndex() { @@ -1728,6 +2160,413 @@ func (suite *ApplierTestSuite) TestMultipleDMLEventsInBatch() { // Critically: id=2 (bob@example.com) is NOT present, proving event #3 was rolled back } +func (suite *ApplierTestSuite) TestApplyDMLEventQueriesMoveTablesMode() { + ctx := context.Background() + var err error + + _, err = suite.db.ExecContext(ctx, fmt.Sprintf("CREATE TABLE %s (id INT, item_id INT);", getTestTableName())) + suite.Require().NoError(err) + _, err = suite.otherDB.ExecContext(ctx, fmt.Sprintf("CREATE TABLE %s (id INT, item_id INT);", getTestOtherTableName())) + suite.Require().NoError(err) + + connectionConfig, err := getTestConnectionConfig(ctx, suite.mysqlContainer) + suite.Require().NoError(err) + + migrationContext := newTestMigrationContext() + migrationContext.ApplierConnectionConfig = connectionConfig + migrationContext.MoveTables.ConnectionConfig = connectionConfig + migrationContext.SetConnectionConfig("innodb") + + migrationContext.OriginalTableColumns = sql.NewColumnList([]string{"id", "item_id"}) + migrationContext.SharedColumns = sql.NewColumnList([]string{"id", "item_id"}) + migrationContext.MappedSharedColumns = sql.NewColumnList([]string{"id", "item_id"}) + migrationContext.UniqueKey = &sql.UniqueKey{ + Name: "primary_key", + Columns: *sql.NewColumnList([]string{"id"}), + } + migrationContext.MoveTables.TableNames = []string{testMysqlTableName} + migrationContext.MoveTables.TargetDatabase = testMysqlDatabaseOther + + // Populate the per-table container that prepareQueries/ApplyDMLEventQueries + // route DML through (there is no representative table in move-tables mode). + migrationContext.InitMoveTableContainers() + mt := migrationContext.GetMoveTable(testMysqlTableName) + suite.Require().NotNil(mt) + mt.OriginalTableColumns = migrationContext.OriginalTableColumns + mt.SharedColumns = migrationContext.SharedColumns + mt.MappedSharedColumns = migrationContext.MappedSharedColumns + mt.UniqueKey = migrationContext.UniqueKey + + applier := NewApplier(migrationContext) + suite.Require().NoError(applier.prepareQueries()) + defer applier.Teardown() + + err = applier.InitDBConnections() + suite.Require().NoError(err) + + dmlEvents := []*binlog.BinlogDMLEvent{ + { + DatabaseName: testMysqlDatabase, + TableName: testMysqlTableName, + DML: binlog.InsertDML, + NewColumnValues: sql.ToColumnValues([]interface{}{123456, 42}), + }, + } + err = applier.ApplyDMLEventQueries(dmlEvents) + suite.Require().NoError(err) + + // Check that the row was inserted into the ghost table via moveTablesTargetDB + rows, err := suite.otherDB.Query("SELECT * FROM " + getTestOtherTableName()) + suite.Require().NoError(err) + defer rows.Close() + + var count, id, item_id int + for rows.Next() { + err = rows.Scan(&id, &item_id) + suite.Require().NoError(err) + count += 1 + } + suite.Require().NoError(rows.Err()) + + suite.Require().Equal(1, count) + suite.Require().Equal(123456, id) + suite.Require().Equal(42, item_id) + + suite.Require().Equal(int64(1), migrationContext.TotalDMLEventsApplied) + suite.Require().Equal(int64(0), migrationContext.RowsDeltaEstimate) +} + +func (suite *ApplierTestSuite) TestApplyDMLEventQueriesMoveTablesGeneratedColumns() { + ctx := context.Background() + createTable := "CREATE TABLE %s (id INT NOT NULL, a INT NOT NULL, virtual_sum INT AS (a + 10) VIRTUAL, b INT NOT NULL, stored_sum INT AS (a + b) STORED, PRIMARY KEY(id));" + _, err := suite.db.ExecContext(ctx, fmt.Sprintf(createTable, getTestTableName())) + suite.Require().NoError(err) + _, err = suite.otherDB.ExecContext(ctx, fmt.Sprintf(createTable, getTestOtherTableName())) + suite.Require().NoError(err) + + connectionConfig, err := getTestConnectionConfig(ctx, suite.mysqlContainer) + suite.Require().NoError(err) + + migrationContext := newTestMigrationContext() + migrationContext.ApplierConnectionConfig = connectionConfig + migrationContext.MoveTables.ConnectionConfig = connectionConfig + migrationContext.SetConnectionConfig("innodb") + migrationContext.MoveTables.TableNames = []string{testMysqlTableName} + migrationContext.MoveTables.TargetDatabase = testMysqlDatabaseOther + migrationContext.InitMoveTableContainers() + mt := migrationContext.GetMoveTable(testMysqlTableName) + suite.Require().NotNil(mt) + mt.OriginalTableColumns = sql.NewColumnList([]string{"id", "a", "virtual_sum", "b", "stored_sum"}) + mt.SharedColumns = sql.NewColumnList([]string{"id", "a", "b"}) + mt.MappedSharedColumns = sql.NewColumnList([]string{"id", "a", "b"}) + mt.UniqueKey = &sql.UniqueKey{Name: "PRIMARY", Columns: *sql.NewColumnList([]string{"id"})} + + applier := NewApplier(migrationContext) + suite.Require().NoError(applier.prepareQueries()) + defer applier.Teardown() + suite.Require().NoError(applier.InitDBConnections()) + + err = applier.ApplyDMLEventQueries([]*binlog.BinlogDMLEvent{ + { + DatabaseName: testMysqlDatabase, + TableName: testMysqlTableName, + DML: binlog.InsertDML, + NewColumnValues: sql.ToColumnValues([]interface{}{1, 2, 12, 3, 5}), + }, + { + DatabaseName: testMysqlDatabase, + TableName: testMysqlTableName, + DML: binlog.UpdateDML, + WhereColumnValues: sql.ToColumnValues([]interface{}{1, 2, 12, 3, 5}), + NewColumnValues: sql.ToColumnValues([]interface{}{1, 7, 17, 11, 18}), + }, + }) + suite.Require().NoError(err) + + var id, a, virtualSum, b, storedSum int + err = suite.otherDB.QueryRowContext(ctx, "SELECT id, a, virtual_sum, b, stored_sum FROM "+getTestOtherTableName()).Scan(&id, &a, &virtualSum, &b, &storedSum) + suite.Require().NoError(err) + suite.Require().Equal([]int{1, 7, 17, 11, 18}, []int{id, a, virtualSum, b, storedSum}) + + err = applier.ApplyDMLEventQueries([]*binlog.BinlogDMLEvent{ + { + DatabaseName: testMysqlDatabase, + TableName: testMysqlTableName, + DML: binlog.DeleteDML, + WhereColumnValues: sql.ToColumnValues([]interface{}{1, 7, 17, 11, 18}), + }, + }) + suite.Require().NoError(err) + + var count int + err = suite.otherDB.QueryRowContext(ctx, "SELECT COUNT(*) FROM "+getTestOtherTableName()).Scan(&count) + suite.Require().NoError(err) + suite.Require().Zero(count) +} + +func (suite *ApplierTestSuite) TestApplyIterationMoveTableCopyQueries() { + ctx := context.Background() + var err error + + _, err = suite.db.ExecContext(ctx, fmt.Sprintf("CREATE TABLE %s (id INT NOT NULL, name VARCHAR(50), created_at DATETIME NOT NULL, PRIMARY KEY(id));", getTestTableName())) + suite.Require().NoError(err) + _, err = suite.otherDB.ExecContext(ctx, fmt.Sprintf("CREATE TABLE %s (id INT NOT NULL, name VARCHAR(50), created_at DATETIME NOT NULL, PRIMARY KEY(id));", getTestOtherTableName())) + suite.Require().NoError(err) + _, err = suite.db.ExecContext(ctx, fmt.Sprintf("INSERT INTO %s (id, name, created_at) VALUES (1, 'alice', '2024-01-15 10:30:00'), (2, 'bob', '2024-06-20 14:45:00'), (3, 'carol', '2025-12-31 23:59:59');", getTestTableName())) + suite.Require().NoError(err) + + connectionConfig, err := getTestConnectionConfig(ctx, suite.mysqlContainer) + suite.Require().NoError(err) + + migrationContext := newTestMigrationContext() + migrationContext.ApplierConnectionConfig = connectionConfig + migrationContext.MoveTables.ConnectionConfig = connectionConfig + migrationContext.SetConnectionConfig("innodb") + migrationContext.OriginalTableColumns = sql.NewColumnList([]string{"id", "name", "created_at"}) + migrationContext.SharedColumns = sql.NewColumnList([]string{"id", "name", "created_at"}) + migrationContext.MappedSharedColumns = sql.NewColumnList([]string{"id", "name", "created_at"}) + migrationContext.UniqueKey = &sql.UniqueKey{ + Name: "PRIMARY", + Columns: *sql.NewColumnList([]string{"id"}), + } + migrationContext.MoveTables.TableNames = []string{testMysqlTableName} + migrationContext.MoveTables.TargetDatabase = testMysqlDatabaseOther + + // Populate the per-table container the move-tables copy path operates on. + migrationContext.InitMoveTableContainers() + mt := migrationContext.GetMoveTable(testMysqlTableName) + suite.Require().NotNil(mt) + mt.OriginalTableColumns = migrationContext.OriginalTableColumns + mt.SharedColumns = migrationContext.SharedColumns + mt.MappedSharedColumns = migrationContext.MappedSharedColumns + mt.UniqueKey = migrationContext.UniqueKey + + applier := NewApplier(migrationContext) + applier.prepareQueries() + defer applier.Teardown() + + err = applier.InitDBConnections() + suite.Require().NoError(err) + + err = applier.ReadMoveTableMigrationRangeValues(nil, mt) + suite.Require().NoError(err) + + mt.SetNextIterationRangeMinValues() + hasFurtherRange, err := applier.CalculateMoveTableNextIterationRangeEndValues(applier.db, mt) + suite.Require().NoError(err) + suite.Require().True(hasFurtherRange) + + chunkSize, rowsAffected, duration, err := applier.ApplyIterationMoveTableCopyQueries(applier.db, mt) + suite.Require().NoError(err) + suite.Require().Equal(int64(3), rowsAffected) + suite.Require().Equal(int64(1000), chunkSize) + suite.Require().Greater(duration, time.Duration(0)) + + // Verify rows were copied to the other table + rows, err := suite.otherDB.QueryContext(ctx, "SELECT id, name, created_at FROM "+getTestOtherTableName()+" ORDER BY id") + suite.Require().NoError(err) + defer rows.Close() + + type row struct { + id int + name string + createdAt string + } + var results []row + for rows.Next() { + var r row + err = rows.Scan(&r.id, &r.name, &r.createdAt) + suite.Require().NoError(err) + results = append(results, r) + } + suite.Require().NoError(rows.Err()) + + suite.Require().Len(results, 3) + suite.Require().Equal(1, results[0].id) + suite.Require().Equal("alice", results[0].name) + suite.Require().Equal("2024-01-15 10:30:00", results[0].createdAt) + suite.Require().Equal(2, results[1].id) + suite.Require().Equal("bob", results[1].name) + suite.Require().Equal("2024-06-20 14:45:00", results[1].createdAt) + suite.Require().Equal(3, results[2].id) + suite.Require().Equal("carol", results[2].name) + suite.Require().Equal("2025-12-31 23:59:59", results[2].createdAt) +} + +func (suite *ApplierTestSuite) TestApplyIterationMoveTableCopyQueriesGeneratedColumns() { + ctx := context.Background() + createTable := "CREATE TABLE %s (id INT NOT NULL, a INT NOT NULL, virtual_sum INT AS (a + 10) VIRTUAL, b INT NOT NULL, stored_sum INT AS (a + b) STORED NOT NULL, UNIQUE KEY stored_sum_uidx (stored_sum));" + _, err := suite.db.ExecContext(ctx, fmt.Sprintf(createTable, getTestTableName())) + suite.Require().NoError(err) + _, err = suite.otherDB.ExecContext(ctx, fmt.Sprintf(createTable, getTestOtherTableName())) + suite.Require().NoError(err) + _, err = suite.db.ExecContext(ctx, "INSERT INTO "+getTestTableName()+" (id, a, b) VALUES (1, 2, 3), (2, 5, 8)") + suite.Require().NoError(err) + + connectionConfig, err := getTestConnectionConfig(ctx, suite.mysqlContainer) + suite.Require().NoError(err) + + migrationContext := newTestMigrationContext() + migrationContext.ApplierConnectionConfig = connectionConfig + migrationContext.MoveTables.ConnectionConfig = connectionConfig + migrationContext.SetConnectionConfig("innodb") + migrationContext.MoveTables.TableNames = []string{testMysqlTableName} + migrationContext.MoveTables.TargetDatabase = testMysqlDatabaseOther + migrationContext.InitMoveTableContainers() + mt := migrationContext.GetMoveTable(testMysqlTableName) + suite.Require().NotNil(mt) + mt.OriginalTableColumns = sql.NewColumnList([]string{"id", "a", "virtual_sum", "b", "stored_sum"}) + mt.SharedColumns = sql.NewColumnList([]string{"id", "a", "b"}) + mt.MappedSharedColumns = sql.NewColumnList([]string{"id", "a", "b"}) + uniqueKeyColumns := sql.NewColumnList([]string{"stored_sum"}) + uniqueKeyColumns.GetColumn("stored_sum").IsVirtual = true + mt.UniqueKey = &sql.UniqueKey{Name: "stored_sum_uidx", Columns: *uniqueKeyColumns} + + applier := NewApplier(migrationContext) + suite.Require().NoError(applier.prepareQueries()) + defer applier.Teardown() + suite.Require().NoError(applier.InitDBConnections()) + suite.Require().NoError(applier.ReadMoveTableMigrationRangeValues(nil, mt)) + + mt.SetNextIterationRangeMinValues() + hasFurtherRange, err := applier.CalculateMoveTableNextIterationRangeEndValues(applier.db, mt) + suite.Require().NoError(err) + suite.Require().True(hasFurtherRange) + _, rowsAffected, _, err := applier.ApplyIterationMoveTableCopyQueries(applier.db, mt) + suite.Require().NoError(err) + suite.Require().Equal(int64(2), rowsAffected) + + rows, err := suite.otherDB.QueryContext(ctx, "SELECT id, a, virtual_sum, b, stored_sum FROM "+getTestOtherTableName()+" ORDER BY id") + suite.Require().NoError(err) + defer rows.Close() + var results [][]int + for rows.Next() { + var id, a, virtualSum, b, storedSum int + suite.Require().NoError(rows.Scan(&id, &a, &virtualSum, &b, &storedSum)) + results = append(results, []int{id, a, virtualSum, b, storedSum}) + } + suite.Require().NoError(rows.Err()) + suite.Require().Equal([][]int{{1, 2, 12, 3, 5}, {2, 5, 15, 8, 13}}, results) +} + +func (suite *ApplierTestSuite) TestApplyIterationMoveTableCopyQueriesNoRows() { + ctx := context.Background() + var err error + + _, err = suite.db.ExecContext(ctx, fmt.Sprintf("CREATE TABLE %s (id INT NOT NULL, name VARCHAR(50), created_at DATETIME NOT NULL, PRIMARY KEY(id));", getTestTableName())) + suite.Require().NoError(err) + _, err = suite.otherDB.ExecContext(ctx, fmt.Sprintf("CREATE TABLE %s (id INT NOT NULL, name VARCHAR(50), created_at DATETIME NOT NULL, PRIMARY KEY(id));", getTestOtherTableName())) + suite.Require().NoError(err) + _, err = suite.db.ExecContext(ctx, fmt.Sprintf("INSERT INTO %s (id, name, created_at) VALUES (1, 'alice', '2024-01-15 10:30:00'), (2, 'bob', '2024-06-20 14:45:00'), (3, 'carol', '2025-12-31 23:59:59');", getTestTableName())) + suite.Require().NoError(err) + + connectionConfig, err := getTestConnectionConfig(ctx, suite.mysqlContainer) + suite.Require().NoError(err) + + migrationContext := newTestMigrationContext() + migrationContext.ApplierConnectionConfig = connectionConfig + migrationContext.MoveTables.ConnectionConfig = connectionConfig + migrationContext.SetConnectionConfig("innodb") + migrationContext.OriginalTableColumns = sql.NewColumnList([]string{"id", "name", "created_at"}) + migrationContext.SharedColumns = sql.NewColumnList([]string{"id", "name", "created_at"}) + migrationContext.MappedSharedColumns = sql.NewColumnList([]string{"id", "name", "created_at"}) + migrationContext.UniqueKey = &sql.UniqueKey{ + Name: "PRIMARY", + Columns: *sql.NewColumnList([]string{"id"}), + } + migrationContext.MoveTables.TableNames = []string{testMysqlTableName} + migrationContext.MoveTables.TargetDatabase = testMysqlDatabaseOther + + // Populate the per-table container the move-tables copy path operates on. + migrationContext.InitMoveTableContainers() + mt := migrationContext.GetMoveTable(testMysqlTableName) + suite.Require().NotNil(mt) + mt.OriginalTableColumns = migrationContext.OriginalTableColumns + mt.SharedColumns = migrationContext.SharedColumns + mt.MappedSharedColumns = migrationContext.MappedSharedColumns + mt.UniqueKey = migrationContext.UniqueKey + + applier := NewApplier(migrationContext) + applier.prepareQueries() + defer applier.Teardown() + + err = applier.InitDBConnections() + suite.Require().NoError(err) + + // Point the iteration range at a key range that contains no rows so the + // SELECT returns an empty result set and the INSERT is skipped. + mt.MigrationIterationRangeMinValues = sql.ToColumnValues([]interface{}{100}) + mt.MigrationIterationRangeMaxValues = sql.ToColumnValues([]interface{}{200}) + + chunkSize, rowsAffected, duration, err := applier.ApplyIterationMoveTableCopyQueries(applier.db, mt) + suite.Require().NoError(err) + suite.Require().Equal(int64(0), rowsAffected) + suite.Require().Equal(int64(1000), chunkSize) + suite.Require().Greater(duration, time.Duration(0)) + + // Verify no rows were copied to the target table. + var count int + err = suite.otherDB.QueryRowContext(ctx, "SELECT COUNT(*) FROM "+getTestOtherTableName()).Scan(&count) + suite.Require().NoError(err) + suite.Require().Equal(0, count) +} + +func (suite *ApplierTestSuite) TestShowStatusVariable() { + ctx := context.Background() + + _, err := suite.db.ExecContext(ctx, fmt.Sprintf("CREATE TABLE %s (id INT, item_id INT);", getTestTableName())) + suite.Require().NoError(err) + + connectionConfig, err := getTestConnectionConfig(ctx, suite.mysqlContainer) + suite.Require().NoError(err) + + migrationContext := newTestMigrationContext() + migrationContext.ApplierConnectionConfig = connectionConfig + migrationContext.SetConnectionConfig("innodb") + + applier := NewApplier(migrationContext) + defer applier.Teardown() + + suite.Require().NoError(applier.InitDBConnections()) + + // Uptime is always present in `SHOW GLOBAL STATUS` and is non-negative. + result, err := applier.ShowStatusVariable("Uptime") + suite.Require().NoError(err) + suite.Require().GreaterOrEqual(result, int64(0)) +} + +func (suite *ApplierTestSuite) TestShowStatusVariableMoveTablesMode() { + ctx := context.Background() + + _, err := suite.db.ExecContext(ctx, fmt.Sprintf("CREATE TABLE %s (id INT, item_id INT);", getTestTableName())) + suite.Require().NoError(err) + + connectionConfig, err := getTestConnectionConfig(ctx, suite.mysqlContainer) + suite.Require().NoError(err) + + migrationContext := newTestMigrationContext() + migrationContext.ApplierConnectionConfig = connectionConfig + migrationContext.MoveTables.ConnectionConfig = connectionConfig + migrationContext.SetConnectionConfig("innodb") + migrationContext.OriginalTableColumns = sql.NewColumnList([]string{"id", "item_id"}) + migrationContext.MoveTables.TableNames = []string{testMysqlTableName} + migrationContext.MoveTables.TargetDatabase = testMysqlDatabaseOther + + applier := NewApplier(migrationContext) + defer applier.Teardown() + + suite.Require().NoError(applier.InitDBConnections()) + + // In move-tables mode the status variable must be read from the + // move-tables target DB connection rather than the applier DB. + suite.Require().True(migrationContext.IsMoveTablesMode()) + suite.Require().NotNil(applier.moveTablesTargetDB) + + result, err := applier.ShowStatusVariable("Uptime") + suite.Require().NoError(err) + suite.Require().GreaterOrEqual(result, int64(0)) +} + func TestApplier(t *testing.T) { if testing.Short() { t.Skip("skipping applier test suite in short mode") diff --git a/go/logic/checkpoint.go b/go/logic/checkpoint.go index cffe08c4b..079cd69fa 100644 --- a/go/logic/checkpoint.go +++ b/go/logic/checkpoint.go @@ -6,6 +6,9 @@ package logic import ( + "encoding/hex" + "fmt" + "strings" "time" "github.com/github/gh-ost/go/mysql" @@ -16,6 +19,10 @@ import ( type Checkpoint struct { Id int64 Timestamp time.Time + // TableName is the migrated table this checkpoint row belongs to. Empty in + // standard (single-table) mode; set per table in move-tables mode, where the + // checkpoint table holds one row per migrated table. + TableName string // LastTrxCoords are coordinates of a transaction // that has been applied on ghost table. LastTrxCoords mysql.BinlogCoordinates @@ -24,9 +31,84 @@ type Checkpoint struct { IterationRangeMin *sql.ColumnValues // IterationRangeMax is the max shared key value // for the chunk copier range. - IterationRangeMax *sql.ColumnValues - Iteration int64 - RowsCopied int64 - DMLApplied int64 - IsCutover bool + IterationRangeMax *sql.ColumnValues + Iteration int64 + RowsCopied int64 + DMLApplied int64 + IsCutover bool + MoveTablesCutOverStarted bool + MoveTablesCutOverDrainGTID mysql.BinlogCoordinates +} + +// moveTableCheckpointNullToken marks a NULL value in a serialized range. Hex +// encoding never produces "~", so it is unambiguous. +const moveTableCheckpointNullToken = "~" + +// serializeRangeValues encodes a unique-key range (one or more column values) +// into a portable, table-agnostic text form: each value hex-encoded, comma- +// joined. This lets the single move-tables checkpoint table store ranges for +// tables with heterogeneous unique keys without per-key typed columns. +func serializeRangeValues(cv *sql.ColumnValues) string { + if cv == nil { + return "" + } + vals := cv.AbstractValues() + parts := make([]string, len(vals)) + for i, v := range vals { + if v == nil { + parts[i] = moveTableCheckpointNullToken + continue + } + var b []byte + switch t := v.(type) { + case []byte: + b = t + case string: + b = []byte(t) + default: + b = []byte(fmt.Sprintf("%v", t)) + } + parts[i] = hex.EncodeToString(b) + } + return strings.Join(parts, ",") +} + +// deserializeRangeValues reverses serializeRangeValues for a key of arity n. The +// values come back as []byte (or nil), which are accepted as prepared-statement +// args and coerced by MySQL to the target column type for comparison. +func deserializeRangeValues(s string, n int) *sql.ColumnValues { + abstract := make([]interface{}, n) + if s != "" { + parts := strings.Split(s, ",") + for i := 0; i < n && i < len(parts); i++ { + p := parts[i] + if p == "" || p == moveTableCheckpointNullToken { + continue // leave nil + } + if b, err := hex.DecodeString(p); err == nil { + abstract[i] = b + } + } + } + return sql.ToColumnValues(abstract) +} + +// isEmptyRange reports whether a deserialized range carries no usable boundary +// (zero columns, or every column value nil). Such a range means the table had no +// completed chunk when the checkpoint was written, so on resume it must start +// from the table minimum rather than from this empty boundary. +func isEmptyRange(cv *sql.ColumnValues) bool { + if cv == nil { + return true + } + vals := cv.AbstractValues() + if len(vals) == 0 { + return true + } + for _, v := range vals { + if v != nil { + return false + } + } + return true } diff --git a/go/logic/checkpoint_test.go b/go/logic/checkpoint_test.go new file mode 100644 index 000000000..ac59a32e9 --- /dev/null +++ b/go/logic/checkpoint_test.go @@ -0,0 +1,89 @@ +/* + Copyright 2025 GitHub Inc. + See https://github.com/github/gh-ost/blob/master/LICENSE +*/ + +package logic + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/github/gh-ost/go/base" + "github.com/github/gh-ost/go/mysql" + "github.com/github/gh-ost/go/sql" +) + +// TestSerializeRangeValues covers the table-agnostic, hex-per-value encoding used +// to store a move-table's unique-key range in the single checkpoint table. +func TestSerializeRangeValues(t *testing.T) { + // nil ColumnValues serializes to the empty string. + require.Equal(t, "", serializeRangeValues(nil)) + + // A single integer key: hex of its decimal text ("172" -> 31 37 32). + require.Equal(t, "313732", serializeRangeValues(sql.ToColumnValues([]interface{}{172}))) + + // A varchar key: hex of the UTF-8 bytes ("code_8" -> 63 6f 64 65 5f 38). + require.Equal(t, "636f64655f38", serializeRangeValues(sql.ToColumnValues([]interface{}{"code_8"}))) + + // A compound key of heterogeneous types is comma-joined. + require.Equal(t, "3235,636f64655f38", + serializeRangeValues(sql.ToColumnValues([]interface{}{25, "code_8"}))) + + // Raw bytes are hex-encoded as-is. + require.Equal(t, "e590", + serializeRangeValues(sql.ToColumnValues([]interface{}{[]byte{0xe5, 0x90}}))) + + // A nil column value is encoded with the unambiguous NULL token. + require.Equal(t, moveTableCheckpointNullToken, + serializeRangeValues(sql.ToColumnValues([]interface{}{nil}))) +} + +// TestDeserializeRangeValuesRoundTrip verifies the encode->store->decode cycle. +// Values come back as []byte (accepted directly as prepared-statement args), so +// the round trip is checked on the serialized (canonical) form, which is what a +// resumed run actually compares. +func TestDeserializeRangeValuesRoundTrip(t *testing.T) { + orig := sql.ToColumnValues([]interface{}{172, "code_8"}) + s := serializeRangeValues(orig) + + got := deserializeRangeValues(s, 2) + require.Equal(t, s, serializeRangeValues(got), "re-serializing the decoded range must reproduce the stored text") + + vals := got.AbstractValues() + require.Len(t, vals, 2) + require.Equal(t, []byte("172"), vals[0]) + require.Equal(t, []byte("code_8"), vals[1]) +} + +// TestDeserializeRangeValuesNullToken verifies the NULL marker decodes back to a +// nil column value while other columns decode normally. +func TestDeserializeRangeValuesNullToken(t *testing.T) { + got := deserializeRangeValues(moveTableCheckpointNullToken+",3235", 2) + vals := got.AbstractValues() + require.Len(t, vals, 2) + require.Nil(t, vals[0]) + require.Equal(t, []byte("25"), vals[1]) +} + +// TestIsEmptyRange verifies the predicate that tells a resumed run a table had no +// completed chunk yet (so it must restart from the table minimum). +func TestIsEmptyRange(t *testing.T) { + require.True(t, isEmptyRange(nil), "nil range is empty") + require.True(t, isEmptyRange(sql.NewColumnValues(0)), "zero-column range is empty") + require.True(t, isEmptyRange(deserializeRangeValues(moveTableCheckpointNullToken, 1)), "all-nil range is empty") + require.False(t, isEmptyRange(sql.ToColumnValues([]interface{}{1})), "a range with a value is not empty") +} + +func TestParseCheckpointCoordinatesUsesMariaDBFlavor(t *testing.T) { + migrationContext := base.NewMigrationContext() + migrationContext.UseGTIDs = true + migrationContext.ApplierMySQLVersion = "10.6.18-MariaDB-log" + + coordinates, err := NewApplier(migrationContext).parseCheckpointCoordinates("0-1-100") + require.NoError(t, err) + require.Equal(t, "0-1-100", coordinates.String()) + _, ok := coordinates.(*mysql.GTIDBinlogCoordinates) + require.True(t, ok) +} diff --git a/go/logic/hooks.go b/go/logic/hooks.go index 1b36ede63..28cde0ca6 100644 --- a/go/logic/hooks.go +++ b/go/logic/hooks.go @@ -11,6 +11,7 @@ import ( "os" "os/exec" "path/filepath" + "strings" "sync/atomic" "github.com/github/gh-ost/go/base" @@ -220,9 +221,32 @@ func NewHooksExecutor(migrationContext *base.MigrationContext) *HooksExecutor { func (he *HooksExecutor) applyEnvironmentVariables(extraVariables ...string) []string { env := os.Environ() env = append(env, fmt.Sprintf("GH_OST_DATABASE_NAME=%s", he.migrationContext.DatabaseName)) - env = append(env, fmt.Sprintf("GH_OST_TABLE_NAME=%s", he.migrationContext.OriginalTableName)) - env = append(env, fmt.Sprintf("GH_OST_GHOST_TABLE_NAME=%s", he.migrationContext.GetGhostTableName())) - env = append(env, fmt.Sprintf("GH_OST_OLD_TABLE_NAME=%s", he.migrationContext.GetOldTableName())) + + var tableNameEnv string + if he.migrationContext.IsMoveTablesMode() { + tableNameEnv = strings.Join(he.migrationContext.MoveTables.TableNames, ",") + } else { + tableNameEnv = he.migrationContext.OriginalTableName + } + env = append(env, fmt.Sprintf("GH_OST_TABLE_NAME=%s", tableNameEnv)) + var ghostTableNameEnv string + var oldTableNameEnv string + if he.migrationContext.IsMoveTablesMode() { + // No ghost or old tables in move-tables mode: the destination keeps each + // source table's name, and the rollback handles are the per-table + // `_
_del` tables produced by the atomic cutover RENAME. + ghostTableNameEnv = strings.Join(he.migrationContext.MoveTables.TableNames, ",") + delNames := make([]string, 0, len(he.migrationContext.MoveTables.TableNames)) + for _, tableName := range he.migrationContext.MoveTables.TableNames { + delNames = append(delNames, he.migrationContext.MoveTableDelName(tableName)) + } + oldTableNameEnv = strings.Join(delNames, ",") + } else { + ghostTableNameEnv = he.migrationContext.GetGhostTableName() + oldTableNameEnv = he.migrationContext.GetOldTableName() + } + env = append(env, fmt.Sprintf("GH_OST_GHOST_TABLE_NAME=%s", ghostTableNameEnv)) + env = append(env, fmt.Sprintf("GH_OST_OLD_TABLE_NAME=%s", oldTableNameEnv)) env = append(env, fmt.Sprintf("GH_OST_DDL=%s", he.migrationContext.AlterStatement)) env = append(env, fmt.Sprintf("GH_OST_ELAPSED_SECONDS=%f", he.migrationContext.ElapsedTime().Seconds())) env = append(env, fmt.Sprintf("GH_OST_ELAPSED_COPY_SECONDS=%f", he.migrationContext.ElapsedRowCopyTime().Seconds())) @@ -233,8 +257,19 @@ func (he *HooksExecutor) applyEnvironmentVariables(extraVariables ...string) []s env = append(env, fmt.Sprintf("GH_OST_MIGRATED_HOST=%s", he.migrationContext.GetApplierHostname())) env = append(env, fmt.Sprintf("GH_OST_INSPECTED_HOST=%s", he.migrationContext.GetInspectorHostname())) env = append(env, fmt.Sprintf("GH_OST_EXECUTING_HOST=%s", he.migrationContext.Hostname)) + env = append(env, fmt.Sprintf("GH_OST_TARGET_HOST=%s", he.migrationContext.GetTargetHostname())) env = append(env, fmt.Sprintf("GH_OST_INSPECTED_LAG=%f", he.migrationContext.GetCurrentLagDuration().Seconds())) - env = append(env, fmt.Sprintf("GH_OST_HEARTBEAT_LAG=%f", he.migrationContext.TimeSinceLastHeartbeatOnChangelog().Seconds())) + // In move-tables mode there is no changelog heartbeat; writer lag (now - last + // applied binlog event timestamp) replaces the heartbeat-derived lag. Re-point + // GH_OST_HEARTBEAT_LAG at it so existing hooks keep seeing a meaningful value, + // and also expose it explicitly as GH_OST_BINLOG_WRITER_LAG_SECONDS. + heartbeatLagSeconds := he.migrationContext.TimeSinceLastHeartbeatOnChangelog().Seconds() + binlogWriterLagSeconds := he.migrationContext.GetBinlogWriterLag().Seconds() + if he.migrationContext.IsMoveTablesMode() { + heartbeatLagSeconds = binlogWriterLagSeconds + } + env = append(env, fmt.Sprintf("GH_OST_HEARTBEAT_LAG=%f", heartbeatLagSeconds)) + env = append(env, fmt.Sprintf("GH_OST_BINLOG_WRITER_LAG_SECONDS=%f", binlogWriterLagSeconds)) env = append(env, fmt.Sprintf("GH_OST_PROGRESS=%f", he.migrationContext.GetProgressPct())) env = append(env, fmt.Sprintf("GH_OST_ETA_SECONDS=%d", he.migrationContext.GetETASeconds())) env = append(env, fmt.Sprintf("GH_OST_HOOKS_HINT=%s", he.migrationContext.HooksHintMessage)) @@ -242,7 +277,20 @@ func (he *HooksExecutor) applyEnvironmentVariables(extraVariables ...string) []s env = append(env, fmt.Sprintf("GH_OST_HOOKS_HINT_TOKEN=%s", he.migrationContext.HooksHintToken)) env = append(env, fmt.Sprintf("GH_OST_DRY_RUN=%t", he.migrationContext.Noop)) env = append(env, fmt.Sprintf("GH_OST_REVERT=%t", he.migrationContext.Revert)) + env = append(env, fmt.Sprintf("GH_OST_MOVE_TABLES=%t", he.migrationContext.IsMoveTablesMode())) + if he.migrationContext.IsMoveTablesMode() { + // Comma-joined list of all migrated tables (§2.4). + env = append(env, fmt.Sprintf("GH_OST_TABLES=%s", strings.Join(he.migrationContext.MoveTables.TableNames, ","))) + } + env = append(env, fmt.Sprintf("GH_OST_TARGET_DATABASE_NAME=%s", he.migrationContext.GetTargetDatabaseName())) + var targetTableNameEnv string + if he.migrationContext.IsMoveTablesMode() { + targetTableNameEnv = strings.Join(he.migrationContext.MoveTables.TableNames, ",") + } else { + targetTableNameEnv = he.migrationContext.GetGhostTableName() + } + env = append(env, fmt.Sprintf("GH_OST_TARGET_TABLE_NAME=%s", targetTableNameEnv)) env = append(env, extraVariables...) return env } @@ -320,8 +368,11 @@ func (he *HooksExecutor) OnInteractiveCommand(command string) error { } func (he *HooksExecutor) OnSuccess(instantDDL bool) error { - v := fmt.Sprintf("GH_OST_INSTANT_DDL=%t", instantDDL) - return he.executeHooks(onSuccess, v) + v := []string{fmt.Sprintf("GH_OST_INSTANT_DDL=%t", instantDDL)} + if he.migrationContext.IsMoveTablesMode() && he.migrationContext.MoveTables.DrainGTID != nil { + v = append(v, fmt.Sprintf("GH_OST_DRAIN_GTID=%s", he.migrationContext.MoveTables.DrainGTID.String())) + } + return he.executeHooks(onSuccess, v...) } func (he *HooksExecutor) OnFailure() error { diff --git a/go/logic/hooks_test.go b/go/logic/hooks_test.go index 0729df31f..2865ebac6 100644 --- a/go/logic/hooks_test.go +++ b/go/logic/hooks_test.go @@ -20,6 +20,7 @@ import ( "github.com/stretchr/testify/require" "github.com/github/gh-ost/go/base" + "github.com/github/gh-ost/go/mysql" ) type recordingHooks struct { @@ -242,9 +243,130 @@ func TestHooksExecutorExecuteHooks(t *testing.T) { require.Equal(t, migrationContext.OriginalTableName, split[1]) case "GH_OST_INSTANT_DDL": require.Equal(t, "false", split[1]) + case "GH_OST_MOVE_TABLES": + require.Equal(t, "false", split[1]) + case "GH_OST_TARGET_DATABASE_NAME": + require.Equal(t, migrationContext.DatabaseName, split[1]) + case "GH_OST_TARGET_TABLE_NAME": + require.Equal(t, fmt.Sprintf("_%s_gho", migrationContext.OriginalTableName), split[1]) + case "GH_OST_TARGET_HOST": + require.Equal(t, migrationContext.GetApplierHostname(), split[1]) case "TEST": require.Equal(t, t.Name(), split[1]) } } }) } + +func TestHooksExecutorMoveTablesEnvironmentVariables(t *testing.T) { + migrationContext := base.NewMigrationContext() + migrationContext.DatabaseName = "source_db" + migrationContext.OriginalTableName = "tablename" + migrationContext.MoveTables.TableNames = []string{"tablename"} + migrationContext.MoveTables.TargetDatabase = "target_db" + migrationContext.MoveTables.ConnectionConfig = mysql.NewConnectionConfig() + migrationContext.MoveTables.ConnectionConfig.Key.Hostname = "target.example.com" + migrationContext.MoveTables.ConnectionConfig.ImpliedKey = &migrationContext.MoveTables.ConnectionConfig.Key + + hooksExecutor := NewHooksExecutor(migrationContext) + + hooksPath, err := os.MkdirTemp("", "TestHooksExecutorMoveTablesEnvironmentVariables") + require.NoError(t, err) + defer os.RemoveAll(hooksPath) + migrationContext.HooksPath = hooksPath + + require.NoError(t, os.WriteFile( + filepath.Join(hooksPath, "success-hook"), + []byte("#!/bin/sh\nenv"), + 0o777, + )) + + var buf bytes.Buffer + hooksExecutor.writer = &buf + require.Nil(t, hooksExecutor.executeHooks("success-hook")) + + scanner := bufio.NewScanner(&buf) + for scanner.Scan() { + split := strings.SplitN(scanner.Text(), "=", 2) + switch split[0] { + case "GH_OST_MOVE_TABLES": + require.Equal(t, "true", split[1]) + case "GH_OST_TARGET_DATABASE_NAME": + require.Equal(t, "target_db", split[1]) + case "GH_OST_TARGET_TABLE_NAME": + require.Equal(t, "tablename", split[1]) + case "GH_OST_TARGET_HOST": + require.Equal(t, "target.example.com", split[1]) + } + } +} + +func TestHooksExecutorOnSuccessEnvironmentVariables(t *testing.T) { + writeOnSuccessHook := func(t *testing.T, hooksPath string) { + t.Helper() + require.NoError(t, os.WriteFile( + filepath.Join(hooksPath, onSuccess), + []byte("#!/bin/sh\nenv"), + 0o777, + )) + } + + envFromBuffer := func(buf *bytes.Buffer) map[string]string { + envMap := map[string]string{} + scanner := bufio.NewScanner(buf) + for scanner.Scan() { + split := strings.SplitN(scanner.Text(), "=", 2) + if len(split) == 2 { + envMap[split[0]] = split[1] + } + } + return envMap + } + + t.Run("move-tables-includes-drain-gtid", func(t *testing.T) { + migrationContext := base.NewMigrationContext() + migrationContext.DatabaseName = "source_db" + migrationContext.OriginalTableName = "tablename" + migrationContext.MoveTables.TableNames = []string{"tablename"} + migrationContext.MoveTables.DrainGTID = &mysql.FileBinlogCoordinates{LogFile: "mysql-bin.000001", LogPos: 12345} + + hooksExecutor := NewHooksExecutor(migrationContext) + + hooksPath, err := os.MkdirTemp("", "TestHooksExecutorOnSuccessEnvironmentVariables-move-tables") + require.NoError(t, err) + defer os.RemoveAll(hooksPath) + migrationContext.HooksPath = hooksPath + writeOnSuccessHook(t, hooksPath) + + var buf bytes.Buffer + hooksExecutor.writer = &buf + require.NoError(t, hooksExecutor.OnSuccess(true)) + + envMap := envFromBuffer(&buf) + require.Equal(t, "true", envMap["GH_OST_INSTANT_DDL"]) + require.Equal(t, migrationContext.MoveTables.DrainGTID.String(), envMap["GH_OST_DRAIN_GTID"]) + }) + + t.Run("non-move-tables-omits-drain-gtid", func(t *testing.T) { + migrationContext := base.NewMigrationContext() + migrationContext.DatabaseName = "source_db" + migrationContext.OriginalTableName = "tablename" + + hooksExecutor := NewHooksExecutor(migrationContext) + + hooksPath, err := os.MkdirTemp("", "TestHooksExecutorOnSuccessEnvironmentVariables-standard") + require.NoError(t, err) + defer os.RemoveAll(hooksPath) + migrationContext.HooksPath = hooksPath + writeOnSuccessHook(t, hooksPath) + + var buf bytes.Buffer + hooksExecutor.writer = &buf + require.NoError(t, hooksExecutor.OnSuccess(false)) + + envMap := envFromBuffer(&buf) + require.Equal(t, "false", envMap["GH_OST_INSTANT_DDL"]) + _, exists := envMap["GH_OST_DRAIN_GTID"] + require.False(t, exists) + }) +} diff --git a/go/logic/inspect.go b/go/logic/inspect.go index 05d6b67b5..35c6d8ab4 100644 --- a/go/logic/inspect.go +++ b/go/logic/inspect.go @@ -87,17 +87,20 @@ func (isp *Inspector) InitDBConnections() (err error) { } func (isp *Inspector) ValidateOriginalTable() (err error) { + if isp.migrationContext.IsMoveTablesMode() { + return errors.New("ValidateOriginalTable is not available in move-tables mode; each migrated table is validated individually via validateTableExistsAndNotView / validateTableForeignKeysFor / validateTableTriggersFor") + } if err := isp.validateTable(); err != nil { return err } if err := isp.validateTableForeignKeys(isp.migrationContext.DiscardForeignKeys); err != nil { - return err + return fmt.Errorf("failed to validate table foreign keys: %w", err) } if err := isp.validateTableTriggers(); err != nil { - return err + return fmt.Errorf("failed to validate table triggers: %w", err) } if err := isp.estimateTableRowsViaExplain(); err != nil { - return err + return fmt.Errorf("failed to estimate table rows: %w", err) } return nil } @@ -119,17 +122,27 @@ func (isp *Inspector) InspectTableColumnsAndUniqueKeys(tableName string) (column } func (isp *Inspector) InspectOriginalTable() (err error) { - isp.migrationContext.OriginalTableColumns, isp.migrationContext.OriginalTableVirtualColumns, isp.migrationContext.OriginalTableUniqueKeys, err = isp.InspectTableColumnsAndUniqueKeys(isp.migrationContext.OriginalTableName) + if isp.migrationContext.IsMoveTablesMode() { + return errors.New("InspectOriginalTable is not available in move-tables mode; use InspectMoveTable per table") + } + isp.migrationContext.OriginalTableColumns, isp.migrationContext.OriginalTableVirtualColumns, isp.migrationContext.OriginalTableUniqueKeys, err = isp.InspectTableColumnsAndUniqueKeys(isp.originalTableName()) if err != nil { return err } - isp.migrationContext.OriginalTableAutoIncrement, err = isp.getAutoIncrementValue(isp.migrationContext.OriginalTableName) + isp.migrationContext.OriginalTableAutoIncrement, err = isp.getAutoIncrementValue(isp.originalTableName()) if err != nil { return err } return nil } +func (isp *Inspector) originalTableName() string { + if isp.migrationContext.IsMoveTablesMode() { + panic("inspector.originalTableName() must not be called in move-tables mode; inspect each table via its name (e.g. validateTableFor/InspectMoveTable)") + } + return isp.migrationContext.OriginalTableName +} + // inspectOriginalAndGhostTables compares original and ghost tables to see whether the migration // makes sense and is valid. It extracts the list of shared columns and the chosen migration unique key func (isp *Inspector) inspectOriginalAndGhostTables() (err error) { @@ -144,30 +157,7 @@ func (isp *Inspector) inspectOriginalAndGhostTables() (err error) { return err } sharedUniqueKeys := isp.getSharedUniqueKeys(isp.migrationContext.OriginalTableUniqueKeys, isp.migrationContext.GhostTableUniqueKeys) - for i, sharedUniqueKey := range sharedUniqueKeys { - isp.applyColumnTypes(isp.migrationContext.DatabaseName, isp.migrationContext.OriginalTableName, &sharedUniqueKey.Columns) - uniqueKeyIsValid := true - for _, column := range sharedUniqueKey.Columns.Columns() { - switch column.Type { - case sql.FloatColumnType: - { - isp.migrationContext.Log.Warningf("Will not use %+v as shared key due to FLOAT data type", sharedUniqueKey.Name) - uniqueKeyIsValid = false - } - case sql.JSONColumnType: - { - // Noteworthy that at this time MySQL does not allow JSON indexing anyhow, but this code - // will remain in place to potentially handle the future case where JSON is supported in indexes. - isp.migrationContext.Log.Warningf("Will not use %+v as shared key due to JSON data type", sharedUniqueKey.Name) - uniqueKeyIsValid = false - } - } - } - if uniqueKeyIsValid { - isp.migrationContext.UniqueKey = sharedUniqueKeys[i] - break - } - } + isp.migrationContext.UniqueKey = isp.selectUniqueKey(isp.originalTableName(), sharedUniqueKeys) if isp.migrationContext.UniqueKey == nil { return fmt.Errorf("no shared unique key can be found after ALTER! Bailing out") } @@ -187,7 +177,7 @@ func (isp *Inspector) inspectOriginalAndGhostTables() (err error) { // This additional step looks at which columns are unsigned. We could have merged this within // the `getTableColumns()` function, but it's a later patch and introduces some complexity; I feel // comfortable in doing this as a separate step. - isp.applyColumnTypes(isp.migrationContext.DatabaseName, isp.migrationContext.OriginalTableName, isp.migrationContext.OriginalTableColumns, isp.migrationContext.SharedColumns, &isp.migrationContext.UniqueKey.Columns) + isp.applyColumnTypes(isp.migrationContext.DatabaseName, isp.originalTableName(), isp.migrationContext.OriginalTableColumns, isp.migrationContext.SharedColumns, &isp.migrationContext.UniqueKey.Columns) isp.applyColumnTypes(isp.migrationContext.DatabaseName, isp.migrationContext.GetGhostTableName(), isp.migrationContext.GhostTableColumns, isp.migrationContext.MappedSharedColumns) for i := range isp.migrationContext.SharedColumns.Columns() { @@ -218,6 +208,33 @@ func (isp *Inspector) inspectOriginalAndGhostTables() (err error) { return nil } +func (isp *Inspector) selectUniqueKey(tableName string, candidateKeys []*sql.UniqueKey) *sql.UniqueKey { + for i, candidateKey := range candidateKeys { + isp.applyColumnTypes(isp.migrationContext.DatabaseName, tableName, &candidateKey.Columns) + uniqueKeyIsValid := true + for _, column := range candidateKey.Columns.Columns() { + switch column.Type { + case sql.FloatColumnType: + { + isp.migrationContext.Log.Warningf("Will not use %+v as unique key due to FLOAT data type", candidateKey.Name) + uniqueKeyIsValid = false + } + case sql.JSONColumnType: + { + // Noteworthy that at this time MySQL does not allow JSON indexing anyhow, but this code + // will remain in place to potentially handle the future case where JSON is supported in indexes. + isp.migrationContext.Log.Warningf("Will not use %+v as unique key due to JSON data type", candidateKey.Name) + uniqueKeyIsValid = false + } + } + } + if uniqueKeyIsValid { + return candidateKeys[i] + } + } + return nil +} + // validateConnection issues a simple can-connect to MySQL func (isp *Inspector) validateConnection() error { version, err := base.ValidateConnection(isp.db, isp.connectionConfig, isp.migrationContext, isp.name) @@ -480,33 +497,52 @@ func (isp *Inspector) validateLogSlaveUpdates() error { // validateTable makes sure the table we need to operate on actually exists func (isp *Inspector) validateTable() error { - query := fmt.Sprintf(`show /* gh-ost */ table status from %s like '%s'`, sql.EscapeName(isp.migrationContext.DatabaseName), isp.migrationContext.OriginalTableName) - - tableFound := false + if err := isp.validateTableExistsAndNotView(isp.originalTableName()); err != nil { + return err + } + query := fmt.Sprintf(`show /* gh-ost */ table status from %s like '%s'`, sql.EscapeName(isp.migrationContext.DatabaseName), isp.originalTableName()) err := sqlutils.QueryRowsMap(isp.db, query, func(rowMap sqlutils.RowMap) error { isp.migrationContext.TableEngine = rowMap.GetString("Engine") isp.migrationContext.RowsEstimate = rowMap.GetInt64("Rows") isp.migrationContext.UsedRowsEstimateMethod = base.TableStatusRowsEstimate + return nil + }) + if err != nil { + return err + } + isp.migrationContext.Log.Infof("Table found. Engine=%s", isp.migrationContext.TableEngine) + isp.migrationContext.Log.Debugf("Estimated number of rows via STATUS: %d", isp.migrationContext.RowsEstimate) + return nil +} + +// validateTableExistsAndNotView verifies the named table exists and is a real +// table (not a view). Unlike validateTable it does not mutate shared migration +// state, so it is safe to call per table in move-tables mode. +func (isp *Inspector) validateTableExistsAndNotView(tableName string) error { + query := fmt.Sprintf(`show /* gh-ost */ table status from %s like '%s'`, sql.EscapeName(isp.migrationContext.DatabaseName), tableName) + tableFound := false + err := sqlutils.QueryRowsMap(isp.db, query, func(rowMap sqlutils.RowMap) error { if rowMap.GetString("Comment") == "VIEW" { - return fmt.Errorf("%s.%s is a VIEW, not a real table. Bailing out", sql.EscapeName(isp.migrationContext.DatabaseName), sql.EscapeName(isp.migrationContext.OriginalTableName)) + return fmt.Errorf("%s.%s is a VIEW, not a real table. Bailing out", sql.EscapeName(isp.migrationContext.DatabaseName), sql.EscapeName(tableName)) } tableFound = true - return nil }) if err != nil { return err } if !tableFound { - return isp.migrationContext.Log.Errorf("cannot find table %s.%s!", sql.EscapeName(isp.migrationContext.DatabaseName), sql.EscapeName(isp.migrationContext.OriginalTableName)) + return isp.migrationContext.Log.Errorf("cannot find table %s.%s!", sql.EscapeName(isp.migrationContext.DatabaseName), sql.EscapeName(tableName)) } - isp.migrationContext.Log.Infof("Table found. Engine=%s", isp.migrationContext.TableEngine) - isp.migrationContext.Log.Debugf("Estimated number of rows via STATUS: %d", isp.migrationContext.RowsEstimate) return nil } // validateTableForeignKeys makes sure no foreign keys exist on the migrated table func (isp *Inspector) validateTableForeignKeys(allowChildForeignKeys bool) error { + return isp.validateTableForeignKeysFor(isp.originalTableName(), allowChildForeignKeys) +} + +func (isp *Inspector) validateTableForeignKeysFor(tableName string, allowChildForeignKeys bool) error { if isp.migrationContext.SkipForeignKeyChecks { isp.migrationContext.Log.Warning("--skip-foreign-key-checks provided: will not check for foreign keys") return nil @@ -532,26 +568,26 @@ func (isp *Inspector) validateTableForeignKeys(allowChildForeignKeys bool) error return nil }, isp.migrationContext.DatabaseName, - isp.migrationContext.OriginalTableName, + tableName, isp.migrationContext.DatabaseName, - isp.migrationContext.OriginalTableName, + tableName, isp.migrationContext.DatabaseName, - isp.migrationContext.OriginalTableName, + tableName, isp.migrationContext.DatabaseName, - isp.migrationContext.OriginalTableName, + tableName, ) if err != nil { return err } if numParentForeignKeys > 0 { - return isp.migrationContext.Log.Errorf("found %d parent-side foreign keys on %s.%s. Parent-side foreign keys are not supported. Bailing out", numParentForeignKeys, sql.EscapeName(isp.migrationContext.DatabaseName), sql.EscapeName(isp.migrationContext.OriginalTableName)) + return isp.migrationContext.Log.Errorf("found %d parent-side foreign keys on %s.%s. Parent-side foreign keys are not supported. Bailing out", numParentForeignKeys, sql.EscapeName(isp.migrationContext.DatabaseName), sql.EscapeName(tableName)) } if numChildForeignKeys > 0 { if allowChildForeignKeys { isp.migrationContext.Log.Debugf("Foreign keys found and will be dropped, as per given --discard-foreign-keys flag") return nil } - return isp.migrationContext.Log.Errorf("found %d child-side foreign keys on %s.%s. Child-side foreign keys are not supported. Bailing out", numChildForeignKeys, sql.EscapeName(isp.migrationContext.DatabaseName), sql.EscapeName(isp.migrationContext.OriginalTableName)) + return isp.migrationContext.Log.Errorf("found %d child-side foreign keys on %s.%s. Child-side foreign keys are not supported. Bailing out", numChildForeignKeys, sql.EscapeName(isp.migrationContext.DatabaseName), sql.EscapeName(tableName)) } isp.migrationContext.Log.Debugf("Validated no foreign keys exist on table") return nil @@ -559,6 +595,12 @@ func (isp *Inspector) validateTableForeignKeys(allowChildForeignKeys bool) error // validateTableTriggers makes sure no triggers exist on the migrated table. if --include_triggers is used then it fetches the triggers func (isp *Inspector) validateTableTriggers() error { + return isp.validateTableTriggersFor(isp.originalTableName()) +} + +// validateTableTriggersFor performs the trigger validation for a specific table, +// so it can be applied per table in move-tables mode. +func (isp *Inspector) validateTableTriggersFor(tableName string) error { query := ` SELECT /* gh-ost */ COUNT(*) AS num_triggers FROM @@ -573,15 +615,15 @@ func (isp *Inspector) validateTableTriggers() error { return nil }, isp.migrationContext.DatabaseName, - isp.migrationContext.OriginalTableName, + tableName, ) if err != nil { return err } if numTriggers > 0 { if isp.migrationContext.IncludeTriggers { - isp.migrationContext.Log.Infof("Found %d triggers on %s.%s.", numTriggers, sql.EscapeName(isp.migrationContext.DatabaseName), sql.EscapeName(isp.migrationContext.OriginalTableName)) - isp.migrationContext.Triggers, err = mysql.GetTriggers(isp.db, isp.migrationContext.DatabaseName, isp.migrationContext.OriginalTableName) + isp.migrationContext.Log.Infof("Found %d triggers on %s.%s.", numTriggers, sql.EscapeName(isp.migrationContext.DatabaseName), sql.EscapeName(tableName)) + isp.migrationContext.Triggers, err = mysql.GetTriggers(isp.db, isp.migrationContext.DatabaseName, tableName) if err != nil { return err } @@ -593,7 +635,7 @@ func (isp *Inspector) validateTableTriggers() error { } return nil } - return isp.migrationContext.Log.Errorf("found triggers on %s.%s. Tables with triggers are supported only when using \"include-triggers\" flag. Bailing out", sql.EscapeName(isp.migrationContext.DatabaseName), sql.EscapeName(isp.migrationContext.OriginalTableName)) + return isp.migrationContext.Log.Errorf("found triggers on %s.%s. Tables with triggers are supported only when using \"include-triggers\" flag. Bailing out", sql.EscapeName(isp.migrationContext.DatabaseName), sql.EscapeName(tableName)) } isp.migrationContext.Log.Debugf("Validated no triggers exist on table") return nil @@ -646,7 +688,7 @@ func (isp *Inspector) validateGhostTriggersLength() error { // estimateTableRowsViaExplain estimates number of rows on original table func (isp *Inspector) estimateTableRowsViaExplain() error { - query := fmt.Sprintf(`explain select /* gh-ost */ * from %s.%s where 1=1`, sql.EscapeName(isp.migrationContext.DatabaseName), sql.EscapeName(isp.migrationContext.OriginalTableName)) + query := fmt.Sprintf(`explain select /* gh-ost */ * from %s.%s where 1=1`, sql.EscapeName(isp.migrationContext.DatabaseName), sql.EscapeName(isp.originalTableName())) outputFound := false err := sqlutils.QueryRowsMap(isp.db, query, func(rowMap sqlutils.RowMap) error { @@ -660,53 +702,140 @@ func (isp *Inspector) estimateTableRowsViaExplain() error { return err } if !outputFound { - return isp.migrationContext.Log.Errorf("cannot run EXPLAIN on %s.%s!", sql.EscapeName(isp.migrationContext.DatabaseName), sql.EscapeName(isp.migrationContext.OriginalTableName)) + return isp.migrationContext.Log.Errorf("cannot run EXPLAIN on %s.%s!", sql.EscapeName(isp.migrationContext.DatabaseName), sql.EscapeName(isp.originalTableName())) } isp.migrationContext.Log.Infof("Estimated number of rows via EXPLAIN: %d", isp.migrationContext.RowsEstimate) return nil } +// estimateTableRows estimates the number of rows in the given source table via +// EXPLAIN, returning the estimate rather than mutating shared context state. It +// is used to estimate each migrated table independently in move-tables mode. +func (isp *Inspector) estimateTableRows(tableName string) (int64, error) { + query := fmt.Sprintf(`explain select /* gh-ost */ * from %s.%s where 1=1`, + sql.EscapeName(isp.migrationContext.DatabaseName), sql.EscapeName(tableName)) + var rowsEstimate int64 + outputFound := false + err := sqlutils.QueryRowsMap(isp.db, query, func(rowMap sqlutils.RowMap) error { + rowsEstimate = rowMap.GetInt64("rows") + outputFound = true + return nil + }) + if err != nil { + return 0, err + } + if !outputFound { + return 0, isp.migrationContext.Log.Errorf("cannot run EXPLAIN on %s.%s!", + sql.EscapeName(isp.migrationContext.DatabaseName), sql.EscapeName(tableName)) + } + return rowsEstimate, nil +} + +// InspectMoveTable inspects a single source table for move-tables mode and +// returns its columns, virtual columns, chosen unique key, and row estimate. +// Unlike InspectOriginalTable it does not mutate shared migration context +// fields, so each migrated table can be inspected independently into its own +// per-table container. +func (isp *Inspector) InspectMoveTable(tableName string) (columns *sql.ColumnList, virtualColumns *sql.ColumnList, uniqueKeys [](*sql.UniqueKey), uniqueKey *sql.UniqueKey, rowsEstimate int64, err error) { + columns, virtualColumns, uniqueKeys, err = isp.InspectTableColumnsAndUniqueKeys(tableName) + if err != nil { + return nil, nil, nil, nil, 0, err + } + uniqueKey = isp.selectUniqueKey(tableName, uniqueKeys) + if uniqueKey == nil { + return nil, nil, nil, nil, 0, fmt.Errorf("no valid PRIMARY nor UNIQUE key found for table %s.%s; Bailing out", + sql.EscapeName(isp.migrationContext.DatabaseName), sql.EscapeName(tableName)) + } + rowsEstimate, err = isp.estimateTableRows(tableName) + if err != nil { + return nil, nil, nil, nil, 0, fmt.Errorf("failed to estimate rows for table %s.%s: %w", + sql.EscapeName(isp.migrationContext.DatabaseName), sql.EscapeName(tableName), err) + } + return columns, virtualColumns, uniqueKeys, uniqueKey, rowsEstimate, nil +} + // CountTableRows counts exact number of rows on the original table func (isp *Inspector) CountTableRows(ctx context.Context) error { + if isp.migrationContext.IsMoveTablesMode() { + return errors.New("CountTableRows is not available in move-tables mode; use CountMoveTablesRows") + } atomic.StoreInt64(&isp.migrationContext.CountingRowsFlag, 1) defer atomic.StoreInt64(&isp.migrationContext.CountingRowsFlag, 0) isp.migrationContext.Log.Infof("As instructed, I'm issuing a SELECT COUNT(*) on the table. This may take a while") - conn, err := isp.db.Conn(ctx) + rowsEstimate, err := isp.countTableRowsFor(ctx, isp.originalTableName()) if err != nil { return err } + + // row count query finished. nil out the cancel func, so the main migration thread + // doesn't bother calling it after row copy is done. + isp.migrationContext.SetCountTableRowsCancelFunc(nil) + + atomic.StoreInt64(&isp.migrationContext.RowsEstimate, rowsEstimate) + isp.migrationContext.UsedRowsEstimateMethod = base.CountRowsEstimate + + isp.migrationContext.Log.Infof("Exact number of rows via COUNT: %d", rowsEstimate) + + return nil +} + +// CountMoveTablesRows counts exact rows across every migrated table, recording +// each table's count in its container and the sum as the run-wide estimate. It +// is the move-tables equivalent of CountTableRows, with no representative table. +func (isp *Inspector) CountMoveTablesRows(ctx context.Context) error { + if !isp.migrationContext.IsMoveTablesMode() { + return errors.New("CountMoveTablesRows is only available in move-tables mode; use CountTableRows") + } + atomic.StoreInt64(&isp.migrationContext.CountingRowsFlag, 1) + defer atomic.StoreInt64(&isp.migrationContext.CountingRowsFlag, 0) + + isp.migrationContext.Log.Infof("As instructed, counting exact rows across all migrated tables. This may take a while") + var total int64 + for _, mt := range isp.migrationContext.OrderedMoveTables() { + count, err := isp.countTableRowsFor(ctx, mt.SourceTableName) + if err != nil { + return err + } + atomic.StoreInt64(&mt.RowsEstimate, count) + total += count + } + + isp.migrationContext.SetCountTableRowsCancelFunc(nil) + atomic.StoreInt64(&isp.migrationContext.RowsEstimate, total) + isp.migrationContext.UsedRowsEstimateMethod = base.CountRowsEstimate + isp.migrationContext.Log.Infof("Exact number of rows via COUNT across %d table(s): %d", len(isp.migrationContext.MoveTables.TableNames), total) + return nil +} + +// countTableRowsFor issues a blocking SELECT COUNT(*) for a single table and +// returns the exact count. A cancelled context kills the running query. +func (isp *Inspector) countTableRowsFor(ctx context.Context, tableName string) (int64, error) { + conn, err := isp.db.Conn(ctx) + if err != nil { + return 0, err + } defer conn.Close() var connectionID string if err := conn.QueryRowContext(ctx, `SELECT /* gh-ost */ CONNECTION_ID()`).Scan(&connectionID); err != nil { - return err + return 0, err } - query := fmt.Sprintf(`select /* gh-ost */ count(*) as count_rows from %s.%s`, sql.EscapeName(isp.migrationContext.DatabaseName), sql.EscapeName(isp.migrationContext.OriginalTableName)) + query := fmt.Sprintf(`select /* gh-ost */ count(*) as count_rows from %s.%s`, sql.EscapeName(isp.migrationContext.DatabaseName), sql.EscapeName(tableName)) var rowsEstimate int64 queryStartTime := time.Now() if err := conn.QueryRowContext(ctx, query).Scan(&rowsEstimate); err != nil { metrics.RecordQueryDuration(isp.migrationContext.Metrics, "source", "row_count", time.Since(queryStartTime), err) if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { isp.migrationContext.Log.Infof("exact row count cancelled (%s), likely because I'm about to cut over. I'm going to kill that query.", ctx.Err()) - return mysql.Kill(isp.db, connectionID) + return 0, mysql.Kill(isp.db, connectionID) } - return err + return 0, err } metrics.RecordQueryDuration(isp.migrationContext.Metrics, "source", "row_count", time.Since(queryStartTime), nil) - - // row count query finished. nil out the cancel func, so the main migration thread - // doesn't bother calling it after row copy is done. - isp.migrationContext.SetCountTableRowsCancelFunc(nil) - - atomic.StoreInt64(&isp.migrationContext.RowsEstimate, rowsEstimate) - isp.migrationContext.UsedRowsEstimateMethod = base.CountRowsEstimate - - isp.migrationContext.Log.Infof("Exact number of rows via COUNT: %d", rowsEstimate) - - return nil + return rowsEstimate, nil } // applyColumnTypes diff --git a/go/logic/migrator.go b/go/logic/migrator.go index f2f6b3f20..d5d5a6a6a 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -7,6 +7,7 @@ package logic import ( "context" + gosql "database/sql" "errors" "fmt" "io" @@ -28,6 +29,10 @@ var ( ErrMigrationNotAllowedOnMaster = errors.New("it seems like this migration attempt to run directly on master. Preferably it would be executed on a replica (this reduces load from the master). To proceed please provide --allow-on-master") RetrySleepFn = time.Sleep checkpointTimeout = 2 * time.Second + + // moveTablesCutOverDrainPollInterval is the per-iteration sleep in T3's + // drain poll. 100ms per move_table_mode.md §1.5. + moveTablesCutOverDrainPollInterval = 100 * time.Millisecond ) type ChangelogState string @@ -51,9 +56,10 @@ type lockProcessedStruct struct { } type applyEventStruct struct { - writeFunc *tableWriteFunc - dmlEvent *binlog.BinlogDMLEvent - coords mysql.BinlogCoordinates + writeFunc *tableWriteFunc + dmlEvent *binlog.BinlogDMLEvent + coords mysql.BinlogCoordinates + eventTimestamp time.Time } func newApplyEventStructByFunc(writeFunc *tableWriteFunc) *applyEventStruct { @@ -62,7 +68,7 @@ func newApplyEventStructByFunc(writeFunc *tableWriteFunc) *applyEventStruct { } func newApplyEventStructByDML(dmlEntry *binlog.BinlogEntry) *applyEventStruct { - result := &applyEventStruct{dmlEvent: dmlEntry.DmlEvent, coords: dmlEntry.Coordinates} + result := &applyEventStruct{dmlEvent: dmlEntry.DmlEvent, coords: dmlEntry.Coordinates, eventTimestamp: dmlEntry.Timestamp} return result } @@ -89,6 +95,12 @@ type Migrator struct { migrationContext *base.MigrationContext statusWriter io.Writer + // sourcePrimaryDB is the writable source-cluster primary handle used only for + // move-tables cutover writes (the RENAME + drain-GTID capture) and the source + // `__del` DROP. Source reads use the inspector/streamer connections, which may + // point at a read replica. nil outside move-tables mode. + sourcePrimaryDB *gosql.DB + firstThrottlingCollected chan bool ghostTableMigrated chan bool rowCopyComplete chan error @@ -98,8 +110,9 @@ type Migrator struct { rowCopyCompleteFlag int64 // copyRowsQueue should not be buffered; if buffered some non-damaging but // excessive work happens at the end of the iteration as new copy-jobs arrive before realizing the copy is complete - copyRowsQueue chan tableWriteFunc - applyEventsQueue chan *applyEventStruct + copyRowsQueue chan tableWriteFunc + applyEventsQueue chan *applyEventStruct + applyEventsInFlight int64 finishedMigrating int64 } @@ -416,7 +429,11 @@ func (mgtr *Migrator) countTableRows() (err error) { } countRowsFunc := func(ctx context.Context) error { - if err := mgtr.inspector.CountTableRows(ctx); err != nil { + if mgtr.migrationContext.IsMoveTablesMode() { + if err := mgtr.inspector.CountMoveTablesRows(ctx); err != nil { + return err + } + } else if err := mgtr.inspector.CountTableRows(ctx); err != nil { return err } if err := mgtr.hooksExecutor.OnRowCountComplete(); err != nil { @@ -503,7 +520,7 @@ func (mgtr *Migrator) Migrate() (err error) { defer mgtr.teardown() if err := mgtr.initiateInspector(); err != nil { - return err + return fmt.Errorf("failed to initiate inspector: %w", err) } if err := mgtr.checkAbort(); err != nil { return err @@ -612,7 +629,7 @@ func (mgtr *Migrator) Migrate() (err error) { if err := mgtr.addDMLEventsListener(); err != nil { return err } - if err := mgtr.applier.ReadMigrationRangeValues(); err != nil { + if err := mgtr.applier.ReadMigrationRangeValues(nil); err != nil { return err } @@ -798,6 +815,738 @@ func (mgtr *Migrator) Revert() error { return nil } +func moveTablesWritableColumns(columns, virtualColumns *sql.ColumnList) *sql.ColumnList { + generatedColumnNames := make(map[string]bool, virtualColumns.Len()) + for _, columnName := range virtualColumns.Names() { + generatedColumnNames[strings.ToLower(columnName)] = true + } + + writableColumnNames := make([]string, 0, columns.Len()) + for _, columnName := range columns.Names() { + if !generatedColumnNames[strings.ToLower(columnName)] { + writableColumnNames = append(writableColumnNames, columnName) + } + } + return sql.NewColumnList(writableColumnNames) +} + +func prepareMoveTableColumnMetadata(inspector *Inspector, databaseName, tableName string, mt *base.MoveTable) error { + // Generated columns are present in row events but are not writable on the target. + // Keep separate source and target lists because query builders may mutate column metadata. + mt.SharedColumns = moveTablesWritableColumns(mt.OriginalTableColumns, mt.OriginalTableVirtualColumns) + if mt.SharedColumns.Len() == 0 { + return fmt.Errorf("move-table %s.%s has no writable columns after excluding generated columns", + sql.EscapeName(databaseName), sql.EscapeName(tableName)) + } + mt.MappedSharedColumns = moveTablesWritableColumns(mt.OriginalTableColumns, mt.OriginalTableVirtualColumns) + + // Move-tables does not perform schema conversions, but query builders still + // need type metadata to encode values such as JSON, unsigned, and binary correctly. + if err := inspector.applyColumnTypes( + databaseName, + tableName, + mt.OriginalTableColumns, + mt.SharedColumns, + mt.MappedSharedColumns, + &mt.UniqueKey.Columns, + ); err != nil { + return fmt.Errorf("failed to inspect column types for move-table %s.%s: %w", + sql.EscapeName(databaseName), sql.EscapeName(tableName), err) + } + return nil +} + +// prepareMoveTablesCopyState initializes per-table runtime state for row copy in +// move-tables mode (§2.1). Each migrated table is inspected and validated +// independently into its own container (schema, unique key, row estimate, CREATE +// statement). There is no representative table: a single-entry --move-tables is +// simply an array of one, handled by the same per-table loop. +func (mgtr *Migrator) prepareMoveTablesCopyState() error { + mgtr.migrationContext.InitMoveTableContainers() + + var totalRowsEstimate int64 + for _, mt := range mgtr.migrationContext.OrderedMoveTables() { + // Validate each entry like a standard single-table run: it must exist, be a + // real table (not a view), have no unsupported foreign keys, and no triggers + // (unless --include-triggers). + if err := mgtr.inspector.validateTableExistsAndNotView(mt.SourceTableName); err != nil { + return fmt.Errorf("failed to validate move-table %s.%s: %w", + sql.EscapeName(mt.SourceDatabaseName), sql.EscapeName(mt.SourceTableName), err) + } + columns, virtualColumns, uniqueKeys, uniqueKey, rowsEstimate, err := mgtr.inspector.InspectMoveTable(mt.SourceTableName) + if err != nil { + return fmt.Errorf("failed to inspect move-table %s.%s: %w", + sql.EscapeName(mt.SourceDatabaseName), sql.EscapeName(mt.SourceTableName), err) + } + if err := mgtr.inspector.validateTableForeignKeysFor(mt.SourceTableName, mgtr.migrationContext.DiscardForeignKeys); err != nil { + return fmt.Errorf("failed to validate foreign keys on move-table %s.%s: %w", + sql.EscapeName(mt.SourceDatabaseName), sql.EscapeName(mt.SourceTableName), err) + } + if err := mgtr.inspector.validateTableTriggersFor(mt.SourceTableName); err != nil { + return fmt.Errorf("failed to validate triggers on move-table %s.%s: %w", + sql.EscapeName(mt.SourceDatabaseName), sql.EscapeName(mt.SourceTableName), err) + } + createStatement, err := mgtr.inspector.showCreateTable(mt.SourceTableName) + if err != nil { + return fmt.Errorf("failed to fetch create table statement for %s.%s: %w", + sql.EscapeName(mt.SourceDatabaseName), sql.EscapeName(mt.SourceTableName), err) + } + + mt.OriginalTableColumns = columns + mt.OriginalTableVirtualColumns = virtualColumns + mt.OriginalTableUniqueKeys = uniqueKeys + mt.UniqueKey = uniqueKey + if err := prepareMoveTableColumnMetadata(mgtr.inspector, mt.SourceDatabaseName, mt.SourceTableName, mt); err != nil { + return err + } + mt.RowsEstimate = rowsEstimate + mt.CreateTableStatement = createStatement + totalRowsEstimate += rowsEstimate + } + + // Aggregate the row estimate across all tables for overall progress reporting. + atomic.StoreInt64(&mgtr.migrationContext.RowsEstimate, totalRowsEstimate) + return nil +} + +func (mgtr *Migrator) hydrateMoveTablesStateFromTarget() error { + probeContext := base.NewMigrationContext() + probeContext.DatabaseName = mgtr.migrationContext.GetTargetDatabaseName() + targetInspector := &Inspector{db: mgtr.applier.moveTablesTargetDB, migrationContext: probeContext} + + mgtr.migrationContext.InitMoveTableContainers() + for _, mt := range mgtr.migrationContext.OrderedMoveTables() { + columns, virtualColumns, uniqueKeys, err := targetInspector.InspectTableColumnsAndUniqueKeys(mt.TargetTableName) + if err != nil { + return err + } + uniqueKey := targetInspector.selectUniqueKey(mt.TargetTableName, uniqueKeys) + if uniqueKey == nil { + return fmt.Errorf("no valid unique key found on target table %s.%s while resuming", + sql.EscapeName(mt.TargetDatabaseName), sql.EscapeName(mt.TargetTableName)) + } + mt.OriginalTableColumns = columns + mt.OriginalTableVirtualColumns = virtualColumns + mt.OriginalTableUniqueKeys = uniqueKeys + mt.UniqueKey = uniqueKey + if err := prepareMoveTableColumnMetadata(targetInspector, mt.TargetDatabaseName, mt.TargetTableName, mt); err != nil { + return fmt.Errorf("failed to hydrate move-table state while resuming: %w", err) + } + } + return nil +} + +func (mgtr *Migrator) persistMoveTablesCutOverCheckpoint(drainGTID mysql.BinlogCoordinates, isCutover bool) error { + mgtr.applier.CurrentCoordinatesMutex.Lock() + safeCoords := mgtr.applier.CurrentCoordinates + mgtr.applier.CurrentCoordinatesMutex.Unlock() + + if safeCoords == nil || safeCoords.IsEmpty() { + // In move-tables mode CurrentCoordinates may never advance on a quiet source + // (no _ghc heartbeats, no DML). If there is no backlog, the streamer's + // frontier is a safe fallback for checkpointing. + if mgtr.eventsStreamer != nil && len(mgtr.applyEventsQueue) == 0 && len(mgtr.eventsStreamer.eventsChannel) == 0 { + safeCoords = mgtr.eventsStreamer.GetCurrentBinlogCoordinates() + } + if safeCoords == nil || safeCoords.IsEmpty() { + return errors.New("current coordinates are empty, cannot checkpoint move-tables cutover") + } + } + safeCoords = safeCoords.Clone() + + rows := mgtr.buildMoveTableCheckpointRows(safeCoords, isCutover, true, drainGTID) + return mgtr.applier.WriteMoveTableCheckpoints(rows) +} + +// moveTablesDrainCoordinateReached returns true when current is at-or-ahead of +// drain within the same coordinate family. For GTID drains, current must also +// be GTID-backed; mixed GTID/file-pos comparisons are treated as not reached. +func moveTablesDrainCoordinateReached(current mysql.BinlogCoordinates, drain mysql.BinlogCoordinates) bool { + if current == nil || current.IsEmpty() || drain == nil || drain.IsEmpty() { + return false + } + switch drain.(type) { + case *mysql.GTIDBinlogCoordinates: + if _, ok := current.(*mysql.GTIDBinlogCoordinates); !ok { + return false + } + } + return !current.SmallerThan(drain) +} + +// moveTablesDrainProvenByStreamerProgress is the non-DML-tail fallback used +// in T3: if both queues are empty and the streamer has advanced to drain, +// drain is considered complete even when applier coords did not move. +func moveTablesDrainProvenByStreamerProgress(drain mysql.BinlogCoordinates, streamer mysql.BinlogCoordinates, applyBacklog int, streamerBacklog int) bool { + if applyBacklog != 0 || streamerBacklog != 0 { + return false + } + return moveTablesDrainCoordinateReached(streamer, drain) +} + +func (mgtr *Migrator) drainMoveTablesCutOver(drainGTID mysql.BinlogCoordinates) error { + drainTimeout := time.Duration(mgtr.migrationContext.CutOverLockTimeoutSeconds) * time.Second + mgtr.migrationContext.Log.Infof("T3: draining applier to drain GTID (timeout %s, poll %s)", + drainTimeout, moveTablesCutOverDrainPollInterval) + drainCtx, cancel := context.WithTimeout(context.Background(), drainTimeout) + defer cancel() + ticker := time.NewTicker(moveTablesCutOverDrainPollInterval) + defer ticker.Stop() + // fallbackStreak counts consecutive polls where the streamer-frontier fallback + // held. Requiring two consecutive observations closes the sub-µs window between + // `eventStruct := <-applyEventsQueue` in executeWriteFuncs and the in-flight + // increment inside onApplyEventStruct: across a full poll interval a popped-but- + // unapplied event is either applied (advancing applierCoords, so the normal path + // handles it) or still in flight (applyInFlight>0 resets the streak). + fallbackStreak := 0 + const fallbackStreakRequired = 2 + for { + if err := mgtr.checkAbort(); err != nil { + return err + } + // Primary signal: applier's coordinate (advances when relevant apply work runs). + mgtr.applier.CurrentCoordinatesMutex.Lock() + applierCoords := mgtr.applier.CurrentCoordinates + mgtr.applier.CurrentCoordinatesMutex.Unlock() + drainReached := moveTablesDrainCoordinateReached(applierCoords, drainGTID) + // Backlogs gate completion: if either queue is non-empty, drain is not done. + applyBacklog := len(mgtr.applyEventsQueue) + streamerBacklog := 0 + var streamerCoords mysql.BinlogCoordinates + applierDisplay := "" + if applierCoords != nil { + applierDisplay = applierCoords.DisplayString() + } + if mgtr.eventsStreamer != nil { + streamerBacklog = len(mgtr.eventsStreamer.eventsChannel) + if mgtr.eventsStreamer.binlogReader != nil { + // Secondary signal: streamer's latest source position. + streamerCoords = mgtr.eventsStreamer.GetCurrentBinlogCoordinates() + } + } + applyInFlight := atomic.LoadInt64(&mgtr.applyEventsInFlight) + // Normal completion path: applier reached drain, both queues are empty, + // and no apply handler is still running. + if drainReached && applyBacklog == 0 && streamerBacklog == 0 && applyInFlight == 0 { + mgtr.migrationContext.Log.Infof("T3: drain complete; applier caught up to drain GTID") + return nil + } + // Fallback for non-DML tail: GTID can advance due to unrelated/non-row events, + // so applier may stop moving while streamer has already crossed drain. + // Debounced across consecutive polls so the receive-vs-in-flight window + // cannot trigger a premature completion (see fallbackStreak above). + if applyInFlight == 0 && moveTablesDrainProvenByStreamerProgress(drainGTID, streamerCoords, applyBacklog, streamerBacklog) { + fallbackStreak++ + if fallbackStreak >= fallbackStreakRequired { + mgtr.migrationContext.Log.Infof("T3: drain complete via streamer frontier (non-DML tail after T2)") + return nil + } + mgtr.migrationContext.Log.Debugf("T3: streamer frontier reached, debouncing (%d/%d)", fallbackStreak, fallbackStreakRequired) + } else { + fallbackStreak = 0 + } + if drainReached { + mgtr.migrationContext.Log.Debugf("T3: drain GTID reached but backlog remains (apply=%d, streamer=%d, in_flight=%d)", applyBacklog, streamerBacklog, applyInFlight) + } else { + mgtr.migrationContext.Log.Debugf("T3: applier still behind drain GTID, polling (applier=%s drain=%s in_flight=%d)", applierDisplay, drainGTID.DisplayString(), applyInFlight) + } + select { + case <-drainCtx.Done(): + streamerDisplay := "" + if streamerCoords != nil { + streamerDisplay = streamerCoords.DisplayString() + } + return fmt.Errorf("drain poll timed out after %s: applier did not catch up to drain GTID (applier=%s drain=%s streamer=%s apply_backlog=%d streamer_backlog=%d in_flight=%d)", + drainTimeout, applierDisplay, drainGTID.DisplayString(), streamerDisplay, applyBacklog, streamerBacklog, applyInFlight) + case <-ticker.C: + } + } +} + +func (mgtr *Migrator) resumeMoveTablesCutOverFromCheckpoint(chk *Checkpoint) error { + if chk == nil || !chk.MoveTablesCutOverStarted || chk.MoveTablesCutOverDrainGTID == nil || chk.MoveTablesCutOverDrainGTID.IsEmpty() { + return errors.New("checkpoint does not contain move-tables cutover resume state") + } + // The checkpoint proves the source RENAME already happened in a prior run, so + // `__del` exists on the source. Mark it so a failed resume emits the rollback + // hint. + atomic.StoreInt64(&mgtr.migrationContext.MoveTablesSourceRenamedFlag, 1) + if chk.LastTrxCoords != nil && !chk.LastTrxCoords.IsEmpty() { + mgtr.applier.CurrentCoordinatesMutex.Lock() + mgtr.applier.CurrentCoordinates = chk.LastTrxCoords.Clone() + mgtr.applier.CurrentCoordinatesMutex.Unlock() + } + mgtr.migrationContext.Log.Infof("Resuming move-tables cutover from checkpoint at coords=%+v drain_gtid=%s", + chk.LastTrxCoords, chk.MoveTablesCutOverDrainGTID.DisplayString()) + if err := mgtr.drainMoveTablesCutOver(chk.MoveTablesCutOverDrainGTID); err != nil { + return err + } + if mgtr.migrationContext.Checkpoint { + if err := mgtr.persistMoveTablesCutOverCheckpoint(chk.MoveTablesCutOverDrainGTID, true); err != nil { + mgtr.migrationContext.Log.Warningf("failed to checkpoint drained move-tables cutover: %+v", err) + } + } + atomic.StoreInt64(&mgtr.migrationContext.CutOverCompleteFlag, 1) + mgtr.migrationContext.Log.Debugf("T4: CutOverCompleteFlag set") + mgtr.migrationContext.MoveTables.DrainGTID = chk.MoveTablesCutOverDrainGTID + if err := mgtr.hooksExecutor.OnSuccess(false); err != nil { + return fmt.Errorf("on-success hook failed: %w", err) + } + return nil +} + +func (mgtr *Migrator) MoveTables() (err error) { + mgtr.migrationContext.Log.Infof("Moving tables %v (run %s) from %s to %s (%s)", + mgtr.migrationContext.MoveTables.TableNames, + mgtr.migrationContext.MoveTablesRunToken(), + sql.EscapeName(mgtr.migrationContext.DatabaseName), + sql.EscapeName(mgtr.migrationContext.GetTargetDatabaseName()), mgtr.migrationContext.MoveTables.TargetHost) + mgtr.migrationContext.StartTime = time.Now() + + // Ensure context is cancelled on exit (cleanup) + defer mgtr.migrationContext.CancelContext() + + if mgtr.migrationContext.Hostname, err = os.Hostname(); err != nil { + return err + } + + go mgtr.listenOnPanicAbort() + + // Run on-startup hook: + if err := mgtr.hooksExecutor.OnStartup(); err != nil { + return err + } + + // After this point, we'll need to teardown anything that's been started + // so we don't leave things hanging around + defer mgtr.teardown() + + // If the run fails after the source RENAME, the source `__del` table is the + // rollback handle. Emit a clear rollback hint on any error return once the + // rename has happened. finalCleanup errors are + // swallowed below (they return nil), so this never fires on a successful + // cutover whose only failure was post-success cleanup. + defer func() { + if err != nil && atomic.LoadInt64(&mgtr.migrationContext.MoveTablesSourceRenamedFlag) > 0 { + mgtr.logMoveTablesRollbackHint() + } + }() + + if mgtr.migrationContext.Checkpoint && mgtr.migrationContext.Resume { + mgtr.migrationContext.ApplierConnectionConfig = mgtr.migrationContext.MoveTables.ConnectionConfig + mgtr.applier = NewApplier(mgtr.migrationContext) + if err := mgtr.applier.InitDBConnections(); err != nil { + return err + } + cutoverResumeCheckpoint, err := mgtr.applier.ReadMoveTablesCutOverCheckpoint() + if err != nil && !errors.Is(err, ErrNoCheckpointFound) { + return err + } + if cutoverResumeCheckpoint != nil && cutoverResumeCheckpoint.MoveTablesCutOverStarted && cutoverResumeCheckpoint.MoveTablesCutOverDrainGTID != nil && !cutoverResumeCheckpoint.MoveTablesCutOverDrainGTID.IsEmpty() { + mgtr.migrationContext.InitialStreamerCoords = cutoverResumeCheckpoint.LastTrxCoords + mgtr.migrationContext.Iteration = cutoverResumeCheckpoint.Iteration + atomic.StoreInt64(&mgtr.migrationContext.TotalRowsCopied, cutoverResumeCheckpoint.RowsCopied) + atomic.StoreInt64(&mgtr.migrationContext.TotalDMLEventsApplied, cutoverResumeCheckpoint.DMLApplied) + if err := mgtr.hydrateMoveTablesStateFromTarget(); err != nil { + return fmt.Errorf("failed to hydrate move-tables resume state from target: %w", err) + } + if err := mgtr.createFlagFiles(); err != nil { + return err + } + if err := mgtr.initiateStreaming(); err != nil { + return err + } + // The cutover-resume path skips initiateInspector, so set up the + // source-primary connection here. It is needed by finalCleanup to drop + // the source `__del` rollback handle on a writable primary; the streamer + // connection may be a read replica. Uses the streamer's source version + // for replica-status terminology. + if err := mgtr.setupMoveTablesSourcePrimary(mgtr.eventsStreamer.dbVersion); err != nil { + return err + } + if err := mgtr.applier.prepareQueries(); err != nil { + return err + } + if err := mgtr.hooksExecutor.OnValidated(); err != nil { + return err + } + if err := mgtr.initiateServer(); err != nil { + return err + } + defer mgtr.server.RemoveSocketFile() + if err := mgtr.addDMLEventsListener(); err != nil { + return err + } + mgtr.initiateThrottler() + go func() { + if err := mgtr.executeWriteFuncs(); err != nil { + _ = base.SendWithContext(mgtr.migrationContext.GetContext(), mgtr.migrationContext.PanicAbort, err) + } + }() + // Do not initiate status ticker in cutover resume path: inspector is not initialized, + // and we're only doing drain polling + hooks before exit (no row copy to monitor). + if err := mgtr.resumeMoveTablesCutOverFromCheckpoint(cutoverResumeCheckpoint); err != nil { + return err + } + if err := mgtr.finalCleanup(); err != nil { + return nil + } + mgtr.migrationContext.Log.Infof("Done moving tables %v from %s to %s (%s)", + mgtr.migrationContext.MoveTables.TableNames, sql.EscapeName(mgtr.migrationContext.DatabaseName), + sql.EscapeName(mgtr.migrationContext.GetTargetDatabaseName()), mgtr.migrationContext.MoveTables.TargetHost) + if err := mgtr.checkAbort(); err != nil { + return err + } + return nil + } + // Do not teardown this preflight applier on the miss path. Its DB handles + // come from the shared connection cache keyed by migration UUID, and + // closing them here would poison the later inspector/applier init path + // with "sql: database is closed". + mgtr.applier = nil + } + + if err := mgtr.initiateInspector(); err != nil { + return err + } + if err := mgtr.checkAbort(); err != nil { + return err + } + if err := mgtr.prepareMoveTablesCopyState(); err != nil { + return err + } + if err := mgtr.initiateApplier(); err != nil { + return err + } + if err := mgtr.checkAbort(); err != nil { + return err + } + if mgtr.migrationContext.Checkpoint && mgtr.migrationContext.Resume { + checkpoints, err := mgtr.applier.ReadMoveTableCheckpoints() + if err != nil { + return mgtr.migrationContext.Log.Errorf("no checkpoint found, unable to resume: %+v", err) + } + var resumeCoords mysql.BinlogCoordinates + var totalRowsCopied, totalDMLApplied int64 + for _, mt := range mgtr.migrationContext.OrderedMoveTables() { + chk, ok := checkpoints[mt.SourceTableName] + if !ok { + // No checkpoint row for this table yet; it resumes from scratch. + continue + } + // Run-wide state is replicated on every row; capture it regardless of + // whether this table had completed a chunk. + totalRowsCopied += chk.RowsCopied + if chk.DMLApplied > totalDMLApplied { + totalDMLApplied = chk.DMLApplied + } + // Resume the single applied stream from the earliest per-table frontier + // so no table misses events; re-applied row-copy/DML is idempotent. + if chk.LastTrxCoords != nil && !chk.LastTrxCoords.IsEmpty() { + if resumeCoords == nil || chk.LastTrxCoords.SmallerThan(resumeCoords) { + resumeCoords = chk.LastTrxCoords + } + } + // Only restore the per-table iteration window if a chunk actually + // completed; an empty range means this table must start from its minimum. + if isEmptyRange(chk.IterationRangeMin) || isEmptyRange(chk.IterationRangeMax) { + continue + } + mt.RestoreFromCheckpoint(chk.IterationRangeMin, chk.IterationRangeMax, chk.Iteration, chk.RowsCopied) + mgtr.migrationContext.Log.Infof("Resuming move-table %s from checkpoint range_min=%+v range_max=%+v iteration=%d", + mt.SourceTableName, chk.IterationRangeMin.String(), chk.IterationRangeMax.String(), chk.Iteration) + } + atomic.StoreInt64(&mgtr.migrationContext.TotalRowsCopied, totalRowsCopied) + atomic.StoreInt64(&mgtr.migrationContext.TotalDMLEventsApplied, totalDMLApplied) + if resumeCoords != nil { + mgtr.migrationContext.InitialStreamerCoords = resumeCoords + } + mgtr.migrationContext.Log.Infof("Resuming move-tables from checkpoint coords=%+v", resumeCoords) + } + if err := mgtr.createFlagFiles(); err != nil { + return err + } + if err := mgtr.checkAbort(); err != nil { + return err + } + if err := mgtr.initiateStreaming(); err != nil { + return err + } + if err := mgtr.checkAbort(); err != nil { + return err + } + + // this function assumes that the unique key constraint has been set. + if err := mgtr.applier.prepareQueries(); err != nil { + return err + } + if mgtr.migrationContext.Checkpoint && !mgtr.migrationContext.Resume { + if err := mgtr.applier.CreateCheckpointTable(); err != nil { + mgtr.migrationContext.Log.Errorf("unable to create checkpoint table, see further error details") + } + } + + // Validation complete! Run on-validated hook. + if err := mgtr.hooksExecutor.OnValidated(); err != nil { + return err + } + + if err := mgtr.initiateServer(); err != nil { + return err + } + defer mgtr.server.RemoveSocketFile() + + if err := mgtr.countTableRows(); err != nil { + return err + } + if err := mgtr.addDMLEventsListener(); err != nil { + return err + } + // Read each migrated table's full row-copy range into its per-table container + // (§2.3). Ranges are read from the source via the inspector connection. + for _, mt := range mgtr.migrationContext.OrderedMoveTables() { + if err := mgtr.applier.ReadMoveTableMigrationRangeValues(mgtr.inspector.db, mt); err != nil { + return fmt.Errorf("failed to read migration range for %s.%s: %w", + sql.EscapeName(mt.SourceDatabaseName), sql.EscapeName(mt.SourceTableName), err) + } + } + + mgtr.initiateThrottler() + + // Run on-before-row-copy hook + if err := mgtr.hooksExecutor.OnBeforeRowCopy(); err != nil { + return err + } + go func() { + if err := mgtr.executeWriteFuncs(); err != nil { + // Send error to PanicAbort to trigger abort + _ = base.SendWithContext(mgtr.migrationContext.GetContext(), mgtr.migrationContext.PanicAbort, err) + } + }() + go mgtr.iterateChunksMoveTables() + mgtr.migrationContext.MarkRowCopyStartTime() + go mgtr.initiateStatus() + if mgtr.migrationContext.Checkpoint { + go mgtr.checkpointLoop() + } + + mgtr.migrationContext.Log.Debugf("Operating until row copy is complete") + mgtr.consumeRowCopyComplete() + mgtr.migrationContext.Log.Infof("Row copy complete") + // Check if row copy was aborted due to error + if err := mgtr.checkAbort(); err != nil { + return err + } + if err := mgtr.hooksExecutor.OnRowCopyComplete(); err != nil { + return err + } + + if err := mgtr.moveTablesCutOver(); err != nil { + return err + } + + if err := mgtr.finalCleanup(); err != nil { + return nil + } + mgtr.migrationContext.Log.Infof("Done moving tables %v from %s to %s (%s)", + mgtr.migrationContext.MoveTables.TableNames, sql.EscapeName(mgtr.migrationContext.DatabaseName), + sql.EscapeName(mgtr.migrationContext.GetTargetDatabaseName()), mgtr.migrationContext.MoveTables.TargetHost) + // Final check for abort before declaring success + if err := mgtr.checkAbort(); err != nil { + return err + } + return nil +} + +// moveTablesCutOver orchestrates the cooperative cutover protocol for move-tables +// mode. It implements the T0-T6 transitions described in +// docs/learning/design-refs/coop_cutover.md §1.3. +// +// NOT the standard cutOver() path: every internal call from cutOver() (throttle, +// atomicCutOver, waitForEventsUpToLock, heartbeat-lag) was built on a +// single-server assumption that no longer holds when the applier writes target +// and the streamer reads source. Each is replaced or dropped here. +func (mgtr *Migrator) moveTablesCutOver() (err error) { + if mgtr.migrationContext.Noop { + mgtr.migrationContext.Log.Debugf("Noop operation; not really moving tables") + return nil + } + defer atomic.StoreInt64(&mgtr.migrationContext.InCutOverCriticalSectionFlag, 0) + + // ----- Postpone gate (precedes T0) ----- + // Mirrors standard cutOver()'s sleepWhileTrue postpone structure but DROPS the + // heartbeat-lag branch: move-tables mode disables _ghc heartbeat writes (#8206), + // so TimeSinceLastHeartbeatOnChangelog() returns ~58 years (time.Since(zero)) + // and would deadlock the gate forever. KEEPS the postpone-flag-file + + // unpostpone-socket gate because per coop_cutover.md §1.1 P4, operator-removes- + // postpone is the trigger for the entire cutover phase. + mgtr.migrationContext.Log.Debugf("checking for cut-over postpone") + if err := mgtr.sleepWhileTrue("cut_over_postpone", func() (bool, error) { + if mgtr.migrationContext.PostponeCutOverFlagFile == "" { + return false, nil + } + if atomic.LoadInt64(&mgtr.migrationContext.UserCommandedUnpostponeFlag) > 0 { + atomic.StoreInt64(&mgtr.migrationContext.UserCommandedUnpostponeFlag, 0) + return false, nil + } + if base.FileExists(mgtr.migrationContext.PostponeCutOverFlagFile) { + if atomic.LoadInt64(&mgtr.migrationContext.IsPostponingCutOver) == 0 { + if err := mgtr.hooksExecutor.OnBeginPostponed(); err != nil { + return true, err + } + } + atomic.StoreInt64(&mgtr.migrationContext.IsPostponingCutOver, 1) + return true, nil + } + return false, nil + }); err != nil { + return err + } + atomic.StoreInt64(&mgtr.migrationContext.IsPostponingCutOver, 0) + mgtr.migrationContext.Log.Debugf("checking for cut-over postpone: complete") + + // Disables throttling and background checkpoint loop + atomic.StoreInt64(&mgtr.migrationContext.InCutOverCriticalSectionFlag, 1) + + // ----- T0: on-before-cut-over hook ----- + // Non-zero hook exit aborts cutover BEFORE any source DDL fires. + if err := mgtr.hooksExecutor.OnBeforeCutOver(); err != nil { + return fmt.Errorf("on-before-cut-over hook failed: %w", err) + } + + // ----- T1 + T2: RENAME then capture @@gtid_executed in ONE round trip ----- + // A single multi-statement query (RENAME ...; SELECT @@global.gtid_executed) + // collapses the two operations into one server round trip, so there is no + // client-side or pool-scheduling gap between them: database/sql runs both + // statements on the same pooled connection, and the server executes them in + // order. The captured GTID is therefore the source executed set immediately + // after — and causally after — the rename commit, and a client-side crash can + // no longer land between the rename and the capture. The source-primary DSN + // sets multiStatements=true (see setupMoveTablesSourcePrimary). + // + // We use the dedicated source-primary handle (NOT mgtr.inspector.db): the + // inspector may be a read replica, while the RENAME must run on a writable + // primary. + // + // No retry on the RENAME: it is not idempotent — a partial success leaves the + // table already renamed and a retry would fail. The operator re-runs the whole + // hook chain on failure. + cutOverCtx := mgtr.migrationContext.GetContext() + if mgtr.sourcePrimaryDB == nil { + return errors.New("source primary connection not initialized; cannot perform move-tables cutover") + } + + sourceDB := mgtr.migrationContext.DatabaseName + // Build a single atomic multi-table RENAME covering every table in + // --move-tables order (§2.4): `RENAME TABLE db.t1 TO db._t1_del, db.t2 TO + // db._t2_del, ...`. MySQL executes this as one event group with one GTID, so + // the existing single-drain-GTID mechanism covers the whole move set. + renameClauses := make([]string, 0, len(mgtr.migrationContext.MoveTables.TableNames)) + for _, tableName := range mgtr.migrationContext.MoveTables.TableNames { + delTable := mgtr.migrationContext.MoveTableDelName(tableName) + renameClauses = append(renameClauses, fmt.Sprintf("%s.%s to %s.%s", + sql.EscapeName(sourceDB), sql.EscapeName(tableName), + sql.EscapeName(sourceDB), sql.EscapeName(delTable))) + } + renameAndCaptureQuery := fmt.Sprintf("rename /* gh-ost */ table %s;\nselect @@global.gtid_executed", + strings.Join(renameClauses, ", ")) + mgtr.migrationContext.Log.Infof("T1+T2: renaming %d source table(s) and capturing drain GTID: %s", + len(renameClauses), renameAndCaptureQuery) + + // @@GLOBAL scope is explicit so the intent is unambiguous in the SQL itself. + // Design: https://github.com/github/gh-ost-tablemove-poc/blob/9dc6df75c4c88ff473906a497836c7518f5614ec/design/coop_cutover.md#32-correctness-verification-for-p4 + drainGTIDStr, err := func() (string, error) { + rows, err := mgtr.sourcePrimaryDB.QueryContext(cutOverCtx, renameAndCaptureQuery) + if err != nil { + // The error surfaces from the first statement (RENAME); the table was + // NOT renamed, so do not set the rollback flag. + return "", err + } + defer rows.Close() + // QueryContext returned without error, meaning the RENAME committed on the + // source primary. The source `__del` table now exists and is the rollback + // handle; mark the rename as done so any later failure emits the rollback + // hint (and never drops `__del`). + atomic.StoreInt64(&mgtr.migrationContext.MoveTablesSourceRenamedFlag, 1) + + // The RENAME produces no row-bearing result set. Advance to the SELECT's + // result set if the driver left us on the column-less RENAME result. + cols, err := rows.Columns() + if err != nil { + return "", err + } + if len(cols) == 0 { + if !rows.NextResultSet() { + if err := rows.Err(); err != nil { + return "", err + } + return "", errors.New("expected result set for @@global.gtid_executed after RENAME") + } + } + if !rows.Next() { + if err := rows.Err(); err != nil { + return "", err + } + return "", errors.New("no row returned for @@global.gtid_executed") + } + var gtid string + if err := rows.Scan(>id); err != nil { + return "", err + } + return gtid, nil + }() + if err != nil { + return fmt.Errorf("source RENAME + drain GTID capture failed: %w", err) + } + drainGTID, err := mysql.NewGTIDBinlogCoordinates(mysql.FlavorFor(mgtr.migrationContext.InspectorMySQLVersion), drainGTIDStr) + if err != nil { + return fmt.Errorf("drain GTID parse failed: %w", err) + } + mgtr.migrationContext.Log.Infof("T2: captured drain GTID: %s", drainGTID.DisplayString()) + if mgtr.migrationContext.Checkpoint { + if err := mgtr.persistMoveTablesCutOverCheckpoint(drainGTID, false); err != nil { + return fmt.Errorf("failed to persist move-tables cutover checkpoint: %w", err) + } + } + + mgtr.migrationContext.NewFailPoint("move-tables-panic-before-drain-completion", base.WithFailPointWait(2*time.Second)) + + // ------ T3: draining applier to drain GTID ----------- + if err := mgtr.drainMoveTablesCutOver(drainGTID); err != nil { + return err + } + if mgtr.migrationContext.Checkpoint { + if err := mgtr.persistMoveTablesCutOverCheckpoint(drainGTID, true); err != nil { + mgtr.migrationContext.Log.Warningf("failed to checkpoint drained move-tables cutover: %+v", err) + } + } + + // ----- T4: set CutOverCompleteFlag ----- + // MUST be set before T5 so the streamer's canStopStreaming loop (migrator.go:256) + // can wind down in parallel with the (potentially slow) on-success hook. + // Forgetting this has no visible failure at the call site — the run silently + // hangs after cutover because eventsStreamer.StreamEvents() never returns. + atomic.StoreInt64(&mgtr.migrationContext.CutOverCompleteFlag, 1) + mgtr.migrationContext.Log.Debugf("T4: CutOverCompleteFlag set") + + mgtr.migrationContext.NewFailPoint("move-tables-panic-before-on-success-hook", base.WithFailPointWait(2*time.Second)) + + // ----- T5: on-success hook ----- + // Hook unlocks user_rw@target via db-user-management and flips the + // write_cutover? feature flag. Standard env vars only — GH_OST_DRAIN_GTID + + // GH_OST_TARGET_* are #8211 (1.7), not this PR. The pre-protocol placeholder + // OnSuccess call that used to live in MoveTables() (after finalCleanup) has + // been removed so the hook fires in the order coop_cutover.md §3.2 step 6 + // requires (T5 between T4 and T6, BEFORE finalCleanup). + mgtr.migrationContext.MoveTables.DrainGTID = drainGTID + if err := mgtr.hooksExecutor.OnSuccess(false); err != nil { + return fmt.Errorf("on-success hook failed: %w", err) + } + + // ----- T6: return nil ----- + return nil +} + // ExecOnFailureHook executes the onFailure hook, and this method is provided as the only external // hook access point func (mgtr *Migrator) ExecOnFailureHook() (err error) { @@ -1153,29 +1902,156 @@ func (mgtr *Migrator) atomicCutOver() (err error) { metrics.RecordCutOverPhase(mgtr.migrationContext.Metrics, metrics.CutOverPhaseMagicRename, time.Since(phaseStartTime), err) return mgtr.migrationContext.Log.Errore(err) } - mgtr.migrationContext.RenameTablesEndTime = time.Now() - metrics.RecordCutOverPhase(mgtr.migrationContext.Metrics, metrics.CutOverPhaseMagicRename, mgtr.migrationContext.RenameTablesEndTime.Sub(phaseStartTime), nil) + mgtr.migrationContext.RenameTablesEndTime = time.Now() + metrics.RecordCutOverPhase(mgtr.migrationContext.Metrics, metrics.CutOverPhaseMagicRename, mgtr.migrationContext.RenameTablesEndTime.Sub(phaseStartTime), nil) + + // ooh nice! We're actually truly and thankfully done + lockAndRenameDuration := mgtr.migrationContext.RenameTablesEndTime.Sub(mgtr.migrationContext.LockTablesStartTime) + mgtr.migrationContext.Log.Infof("Lock & rename duration: %s. During mgtr time, queries on %s were blocked", lockAndRenameDuration, sql.EscapeName(mgtr.migrationContext.OriginalTableName)) + return nil +} + +// initiateServer begins listening on unix socket/tcp for incoming interactive commands +func (mgtr *Migrator) initiateServer() (err error) { + var f printStatusFunc = func(rule PrintStatusRule, writer io.Writer) { + mgtr.reportStatus(rule, writer) + } + mgtr.server = NewServer(mgtr.migrationContext, mgtr.hooksExecutor, f) + if err := mgtr.server.BindSocketFile(); err != nil { + return err + } + if err := mgtr.server.BindTCPPort(); err != nil { + return err + } + + go mgtr.server.Serve() + return nil +} + +// assertConnectionWritable fails fast if the given connection points at a +// read_only server. Move-tables writes (target table create + INSERTs, +// checkpoint management, the source RENAME + `__del` DROP) all require writable +// primaries; catching this at startup turns a confusing mid-run failure into a +// clear message. super_read_only implies read_only, so a single check covers both. +func assertConnectionWritable(db *gosql.DB, key mysql.InstanceKey, role string) error { + var readOnly bool + if err := db.QueryRow(`select /* gh-ost */ @@global.read_only`).Scan(&readOnly); err != nil { + return fmt.Errorf("failed to check read_only on move-tables %s %+v: %w", role, key, err) + } + if readOnly { + return fmt.Errorf("move-tables %s %+v is read_only; it must be a writable primary", role, key) + } + return nil +} + +// resolveSourcePrimaryConnectionConfig determines the writable source-cluster +// primary for move-tables cutover writes. It reuses the standard gh-ost master +// detection (walking SHOW SLAVE STATUS from the inspector connection), so when +// the source --host is a replica we find its primary, and when --host is itself +// the primary detection returns the inspector config unchanged (graceful +// fallback, no replica required). --assume-master-host forces the primary, +// mirroring the standard non-move-tables path. +func (mgtr *Migrator) resolveSourcePrimaryConnectionConfig(dbVersion string) (*mysql.ConnectionConfig, error) { + if mgtr.migrationContext.AssumeMasterHostname != "" { + key, err := mysql.ParseInstanceKey(mgtr.migrationContext.AssumeMasterHostname) + if err != nil { + return nil, err + } + cfg := mgtr.migrationContext.InspectorConnectionConfig.DuplicateCredentials(*key) + if mgtr.migrationContext.CliMasterUser != "" { + cfg.User = mgtr.migrationContext.CliMasterUser + } + if mgtr.migrationContext.CliMasterPassword != "" { + cfg.Password = mgtr.migrationContext.CliMasterPassword + } + if err := cfg.RegisterTLSConfig(); err != nil { + return nil, err + } + return cfg, nil + } + visitedKeys := mysql.NewInstanceKeyMap() + return mysql.GetMasterConnectionConfigSafe(dbVersion, mgtr.migrationContext.InspectorConnectionConfig, visitedKeys, mgtr.migrationContext.AllowedMasterMaster) +} + +// setupMoveTablesSourcePrimary resolves the source primary connection config, +// opens its DB handle, and asserts it is writable. It is called from +// initiateInspector (normal path) and from the cutover-resume path (which skips +// the inspector); both supply the source MySQL version used for replica-status +// terminology. Safe to call once per run. +func (mgtr *Migrator) setupMoveTablesSourcePrimary(dbVersion string) error { + cfg, err := mgtr.resolveSourcePrimaryConnectionConfig(dbVersion) + if err != nil { + return fmt.Errorf("failed to resolve move-tables source primary: %w", err) + } + mgtr.migrationContext.MoveTables.SourcePrimaryConnectionConfig = cfg - // ooh nice! We're actually truly and thankfully done - lockAndRenameDuration := mgtr.migrationContext.RenameTablesEndTime.Sub(mgtr.migrationContext.LockTablesStartTime) - mgtr.migrationContext.Log.Infof("Lock & rename duration: %s. During mgtr time, queries on %s were blocked", lockAndRenameDuration, sql.EscapeName(mgtr.migrationContext.OriginalTableName)) + uri := cfg.GetDBUri(mgtr.migrationContext.DatabaseName) + "&multiStatements=true" + db, _, err := mysql.GetDB(mgtr.migrationContext.Uuid, uri) + if err != nil { + return err + } + // Assign before the writability check so teardown() reclaims the pool even if + // the gate rejects a read_only host. + mgtr.sourcePrimaryDB = db + if err := assertConnectionWritable(db, cfg.Key, "source primary"); err != nil { + return err + } + mgtr.migrationContext.Log.Infof("Move-tables source primary is %+v; source reads use %+v", + cfg.Key, mgtr.migrationContext.InspectorConnectionConfig.Key) return nil } -// initiateServer begins listening on unix socket/tcp for incoming interactive commands -func (mgtr *Migrator) initiateServer() (err error) { - var f printStatusFunc = func(rule PrintStatusRule, writer io.Writer) { - mgtr.reportStatus(rule, writer) +// validateMoveTablesSourceReadHost stops a move-tables run early when the source +// --host is the cluster primary. The read path (schema inspection, the full row +// copy, and binlog streaming) all run on --host; pointing it at the primary puts +// the copy load on the primary, which is exactly what move-tables aims to avoid. +// We detect this by comparing the resolved source primary against the inspector +// key — when --host is the primary, master detection returns the inspector +// config unchanged, so the keys match. The operator can repoint --host at a +// replica or explicitly opt in with --allow-on-source-primary. +func (mgtr *Migrator) validateMoveTablesSourceReadHost() error { + if mgtr.migrationContext.MoveTables.AllowOnSourcePrimary { + return nil } - mgtr.server = NewServer(mgtr.migrationContext, mgtr.hooksExecutor, f) - if err := mgtr.server.BindSocketFile(); err != nil { - return err + spc := mgtr.migrationContext.MoveTables.SourcePrimaryConnectionConfig + if spc == nil { + return nil } - if err := mgtr.server.BindTCPPort(); err != nil { - return err + if !spc.Key.Equals(&mgtr.migrationContext.InspectorConnectionConfig.Key) { + return nil } + return fmt.Errorf("move-tables source --host %+v is the cluster primary; reading the full table copy from the primary is the load move-tables is meant to avoid. Point --host at a replica so reads come off the primary, or pass --allow-on-source-primary to proceed against the primary anyway", spc.Key) +} - go mgtr.server.Serve() +// dropMoveTablesSourceOldTables drops every source `_
_del` rollback +// handle on the source primary. Move-tables only: each migrated table leaves a +// `_del` handle behind after the atomic cutover RENAME, and there may be several. +// The inspector/streamer source connections may be a read replica, so the drop +// cannot go through them; it must use the writable source-primary handle. +func (mgtr *Migrator) dropMoveTablesSourceOldTables() error { + if !mgtr.migrationContext.IsMoveTablesMode() { + return errors.New("dropMoveTablesSourceOldTables is only available in move-tables mode") + } + if mgtr.sourcePrimaryDB == nil { + return errors.New("source primary connection not initialized; cannot drop source __del table") + } + databaseName := mgtr.migrationContext.DatabaseName + for _, tableName := range mgtr.migrationContext.MoveTables.TableNames { + delTable := mgtr.migrationContext.MoveTableDelName(tableName) + query := fmt.Sprintf(`drop /* gh-ost */ table if exists %s.%s`, + sql.EscapeName(databaseName), + sql.EscapeName(delTable), + ) + mgtr.migrationContext.Log.Infof("Dropping source table %s.%s on primary %+v", + sql.EscapeName(databaseName), + sql.EscapeName(delTable), + mgtr.migrationContext.MoveTables.SourcePrimaryConnectionConfig.Key, + ) + if _, err := mgtr.sourcePrimaryDB.Exec(query); err != nil { + return err + } + } + mgtr.migrationContext.Log.Infof("Source table(s) dropped") return nil } @@ -1191,15 +2067,33 @@ func (mgtr *Migrator) initiateInspector() (err error) { if err := mgtr.inspector.InitDBConnections(); err != nil { return err } - if err := mgtr.inspector.ValidateOriginalTable(); err != nil { - return err - } - if err := mgtr.inspector.InspectOriginalTable(); err != nil { - return err + // Move-tables mode validates and inspects each table independently in + // prepareMoveTablesCopyState; there is no representative single table to run + // the standard single-table validation/inspection pass against. + if !mgtr.migrationContext.IsMoveTablesMode() { + if err := mgtr.inspector.ValidateOriginalTable(); err != nil { + return fmt.Errorf("failed to validate original table: %w", err) + } + if err := mgtr.inspector.InspectOriginalTable(); err != nil { + return fmt.Errorf("failed to inspect original table: %w", err) + } } // So far so good, table is accessible and valid. // Let's get master connection config - if mgtr.migrationContext.AssumeMasterHostname == "" { + if mgtr.migrationContext.IsMoveTablesMode() { + mgtr.migrationContext.ApplierConnectionConfig = mgtr.migrationContext.MoveTables.ConnectionConfig + // The source --host (inspector) is used for all reads and may be a read + // replica. Detect and connect the source-cluster primary, which the cutover + // RENAME, drain-GTID capture, and source `__del` DROP run against. + if err := mgtr.setupMoveTablesSourcePrimary(mgtr.inspector.dbVersion); err != nil { + return err + } + // Guard the read path: if --host turned out to be the primary itself, stop + // early rather than silently copying the whole table off the primary. + if err := mgtr.validateMoveTablesSourceReadHost(); err != nil { + return err + } + } else if mgtr.migrationContext.AssumeMasterHostname == "" { // No forced master host; detect master if mgtr.migrationContext.ApplierConnectionConfig, err = mgtr.inspector.getMasterConnectionConfig(); err != nil { return err @@ -1231,7 +2125,9 @@ func (mgtr *Migrator) initiateInspector() (err error) { mgtr.migrationContext.Log.Infof("--test-on-replica or --migrate-on-replica given. Will not execute on master %+v but rather on replica %+v itself", *mgtr.migrationContext.ApplierConnectionConfig.ImpliedKey, *mgtr.migrationContext.InspectorConnectionConfig.ImpliedKey, ) - mgtr.migrationContext.ApplierConnectionConfig = mgtr.migrationContext.InspectorConnectionConfig.Duplicate() + if !mgtr.migrationContext.IsMoveTablesMode() { + mgtr.migrationContext.ApplierConnectionConfig = mgtr.migrationContext.InspectorConnectionConfig.Duplicate() + } if mgtr.migrationContext.GetThrottleControlReplicaKeys().Len() == 0 { mgtr.migrationContext.AddThrottleControlReplicaKey(mgtr.migrationContext.InspectorConnectionConfig.Key) } @@ -1300,20 +2196,52 @@ func (mgtr *Migrator) initiateStatus() { // migration, and as response to the "status" interactive command. func (mgtr *Migrator) printMigrationStatusHint(writers ...io.Writer) { w := io.MultiWriter(writers...) - fmt.Fprintf(w, "# Migrating %s.%s; Ghost table is %s.%s\n", - sql.EscapeName(mgtr.migrationContext.DatabaseName), - sql.EscapeName(mgtr.migrationContext.OriginalTableName), - sql.EscapeName(mgtr.migrationContext.DatabaseName), - sql.EscapeName(mgtr.migrationContext.GetGhostTableName()), - ) - fmt.Fprintf(w, "# Migrating %+v; inspecting %+v; executing on %+v\n", - *mgtr.applier.connectionConfig.ImpliedKey, - *mgtr.inspector.connectionConfig.ImpliedKey, - mgtr.migrationContext.Hostname, - ) - fmt.Fprintf(w, "# Migration started at %+v\n", - mgtr.migrationContext.StartTime.Format(time.RubyDate), - ) + if mgtr.migrationContext.IsMoveTablesMode() { + // In move-tables mode there may be several migrated tables; list each + // source -> target mapping rather than a single primary table (§2.3). + // Table names match on source and target; only the database may differ. + sourceDatabaseName := mgtr.migrationContext.DatabaseName + targetDatabaseName := mgtr.migrationContext.GetTargetDatabaseName() + fmt.Fprintf(w, "# Moving %d table(s) from %s to %s:\n", + len(mgtr.migrationContext.MoveTables.TableNames), + sql.EscapeName(sourceDatabaseName), + sql.EscapeName(targetDatabaseName), + ) + for _, tableName := range mgtr.migrationContext.MoveTables.TableNames { + fmt.Fprintf(w, "# - %s.%s -> %s.%s\n", + sql.EscapeName(sourceDatabaseName), sql.EscapeName(tableName), + sql.EscapeName(targetDatabaseName), sql.EscapeName(tableName), + ) + } + + // In move-tables mode the applier writes the target cluster and the + // inspector reads the source cluster, so label them as such rather than + // reusing the single-server "migrating/inspecting" phrasing. + fmt.Fprintf(w, "# Applying on target %+v; reading source %+v; executing on %+v\n", + *mgtr.applier.connectionConfig.ImpliedKey, + *mgtr.inspector.connectionConfig.ImpliedKey, + mgtr.migrationContext.Hostname, + ) + fmt.Fprintf(w, "# Move started at %+v\n", + mgtr.migrationContext.StartTime.Format(time.RubyDate), + ) + } else { + fmt.Fprintf(w, "# Migrating %s.%s; Ghost table is %s.%s\n", + sql.EscapeName(mgtr.migrationContext.DatabaseName), + sql.EscapeName(mgtr.migrationContext.OriginalTableName), + sql.EscapeName(mgtr.migrationContext.DatabaseName), + sql.EscapeName(mgtr.migrationContext.GetGhostTableName()), + ) + fmt.Fprintf(w, "# Migrating %+v; inspecting %+v; executing on %+v\n", + *mgtr.applier.connectionConfig.ImpliedKey, + *mgtr.inspector.connectionConfig.ImpliedKey, + mgtr.migrationContext.Hostname, + ) + fmt.Fprintf(w, "# Migration started at %+v\n", + mgtr.migrationContext.StartTime.Format(time.RubyDate), + ) + } + maxLoad := mgtr.migrationContext.GetMaxLoad() criticalLoad := mgtr.migrationContext.GetCriticalLoad() fmt.Fprintf(w, "# chunk-size: %+v; max-lag-millis: %+vms; dml-batch-size: %+v; max-load: %s; critical-load: %s; nice-ratio: %f\n", @@ -1493,15 +2421,28 @@ func (mgtr *Migrator) printStatus(rule PrintStatusRule, snap migrationProgressSn if !mgtr.shouldPrintStatus(rule, snap.elapsedSeconds, snap.etaDuration) { return } + // Lag reporting differs by mode. Standard mode reports the inspected replica + // "Lag" (changelog heartbeat) plus "HeartbeatLag". Move-tables mode has no + // changelog heartbeat, and the source-side replication "Lag" is meaningless + // (writes go to the target, so migration-induced replica lag appears on target + // replicas and is handled separately by throttling). The "Lag" field is + // therefore dropped and writer lag (now - last applied binlog event timestamp) + // is reported as "WriterLag" instead. + lagStatus := fmt.Sprintf("Lag: %.2fs, HeartbeatLag: %.2fs", + snap.replicationLagSeconds, + snap.heartbeatLagSeconds, + ) + if mgtr.migrationContext.IsMoveTablesMode() { + lagStatus = fmt.Sprintf("WriterLag: %.2fs", snap.writerLagSeconds) + } - status := fmt.Sprintf("Copy: %d/%d %.1f%%; Applied: %d; Backlog: %d/%d; Time: %+v(total), %+v(copy); streamer: %+v; Lag: %.2fs, HeartbeatLag: %.2fs, State: %s; ETA: %s", + status := fmt.Sprintf("Copy: %d/%d %.1f%%; Applied: %d; Backlog: %d/%d; Time: %+v(total), %+v(copy); streamer: %+v; %s, State: %s; ETA: %s", snap.totalRowsCopied, snap.rowsEstimate, snap.progressPct, snap.dmlApplied, snap.applyEventsBacklog, snap.applyEventsCapacity, base.PrettifyDurationOutput(snap.elapsedTime), base.PrettifyDurationOutput(snap.elapsedRowCopyTime), snap.streamerBinlogPosition, - snap.replicationLagSeconds, - snap.heartbeatLagSeconds, + lagStatus, snap.state, snap.eta, ) @@ -1512,6 +2453,26 @@ func (mgtr *Migrator) printStatus(rule PrintStatusRule, snap migrationProgressSn w := io.MultiWriter(writers...) fmt.Fprintln(w, status) + // In move-tables mode, surface per-table row-copy progress so all migrated + // tables are visibly advancing concurrently (§2.3). + if mgtr.migrationContext.IsMoveTablesMode() { + for _, mt := range mgtr.migrationContext.OrderedMoveTables() { + copied := atomic.LoadInt64(&mt.RowsCopied) + estimate := atomic.LoadInt64(&mt.RowsEstimate) + pct := 100.0 + if estimate > 0 { + pct = 100.0 * float64(copied) / float64(estimate) + } + tableState := "copying" + if mt.IsRowCopyComplete() { + tableState = "complete" + } + fmt.Fprintf(w, " - %s.%s: Copy %d/%d %.1f%%; iteration %d; %s\n", + sql.EscapeName(mt.SourceDatabaseName), sql.EscapeName(mt.SourceTableName), + copied, estimate, pct, mt.GetIteration(), tableState) + } + } + // This "hack" is required here because the underlying logging library // github.com/outbrain/golib/log provides two functions Info and Infof; but the arguments of // both these functions are eventually redirected to the same function, which internally calls @@ -1532,14 +2493,19 @@ func (mgtr *Migrator) initiateStreaming() error { if err := mgtr.eventsStreamer.InitDBConnections(); err != nil { return err } - mgtr.eventsStreamer.AddListener( - false, - mgtr.migrationContext.DatabaseName, - mgtr.migrationContext.GetChangelogTableName(), - func(dmlEntry *binlog.BinlogEntry) error { - return mgtr.onChangelogEvent(dmlEntry) - }, - ) + + if mgtr.migrationContext.IsMoveTablesMode() { + mgtr.migrationContext.Log.Info("Skipping stream of the changelog table") + } else { + mgtr.eventsStreamer.AddListener( + false, + mgtr.migrationContext.DatabaseName, + mgtr.migrationContext.GetChangelogTableName(), + func(dmlEntry *binlog.BinlogEntry) error { + return mgtr.onChangelogEvent(dmlEntry) + }, + ) + } go func() { mgtr.migrationContext.Log.Debugf("Beginning streaming") @@ -1561,23 +2527,65 @@ func (mgtr *Migrator) initiateStreaming() error { mgtr.migrationContext.SetRecentBinlogCoordinates(mgtr.eventsStreamer.GetCurrentBinlogCoordinates()) } }() + + // In move-tables mode there is no changelog heartbeat. Writer lag is derived + // from binlog-header timestamps of applied events; when the streamer has been + // silent for the heartbeat interval, treat that silence as "caught up" and bump + // the last-applied timestamp to now so lag does not climb forever while idle. + if mgtr.migrationContext.IsMoveTablesMode() { + go func() { + interval := time.Duration(mgtr.migrationContext.HeartbeatIntervalMilliseconds) * time.Millisecond + ticker := time.NewTicker(interval) + defer ticker.Stop() + for range ticker.C { + if atomic.LoadInt64(&mgtr.finishedMigrating) > 0 { + return + } + mgtr.migrationContext.BumpBinlogWriterLagIfIdle(interval) + } + }() + } return nil } -// addDMLEventsListener begins listening for binlog events on the original table, -// and creates & enqueues a write task per such event. +// addDMLEventsListener begins listening for binlog events on the migrated +// table(s), and creates & enqueues a write task per such event. In move-tables +// mode it registers one listener per migrated table on the shared events +// streamer (§2.2); all listeners feed the same apply queue, parameterized only +// by table name. The streamer already dispatches per (database, table), and the +// applier routes DML to the right per-table query builders by table name. func (mgtr *Migrator) addDMLEventsListener() error { - err := mgtr.eventsStreamer.AddListener( + enqueue := func(dmlEntry *binlog.BinlogEntry) error { + // Record that the streamer just delivered an event for a moved table, so + // the idle-bump rule can tell "falling behind" from "source is quiet". + if mgtr.migrationContext.IsMoveTablesMode() { + mgtr.migrationContext.MarkBinlogEventStreamed() + } + // Use helper to prevent deadlock if buffer fills and executeWriteFuncs exits. + // This is critical because this callback blocks the event streamer. + return base.SendWithContext(mgtr.migrationContext.GetContext(), mgtr.applyEventsQueue, newApplyEventStructByDML(dmlEntry)) + } + + if mgtr.migrationContext.IsMoveTablesMode() { + for _, tableName := range mgtr.migrationContext.MoveTables.TableNames { + if err := mgtr.eventsStreamer.AddListener( + false, + mgtr.migrationContext.DatabaseName, + tableName, + enqueue, + ); err != nil { + return err + } + } + return nil + } + + return mgtr.eventsStreamer.AddListener( false, mgtr.migrationContext.DatabaseName, mgtr.migrationContext.OriginalTableName, - func(dmlEntry *binlog.BinlogEntry) error { - // Use helper to prevent deadlock if buffer fills and executeWriteFuncs exits - // This is critical because this callback blocks the event streamer - return base.SendWithContext(mgtr.migrationContext.GetContext(), mgtr.applyEventsQueue, newApplyEventStructByDML(dmlEntry)) - }, + enqueue, ) - return err } // initiateThrottler kicks in the throttling collection and the throttling checks. @@ -1586,7 +2594,9 @@ func (mgtr *Migrator) initiateThrottler() { go mgtr.throttler.initiateThrottlerCollection(mgtr.firstThrottlingCollected) mgtr.migrationContext.Log.Infof("Waiting for first throttle metrics to be collected") - <-mgtr.firstThrottlingCollected // replication lag + if !mgtr.migrationContext.IsMoveTablesMode() { + <-mgtr.firstThrottlingCollected // replication lag + } <-mgtr.firstThrottlingCollected // HTTP status <-mgtr.firstThrottlingCollected // other, general metrics mgtr.migrationContext.Log.Infof("First throttle metrics collected") @@ -1601,38 +2611,72 @@ func (mgtr *Migrator) initiateApplier() error { if err := mgtr.applier.AcquireMigrationLock(mgtr.migrationContext.GetContext()); err != nil { return err } - if mgtr.migrationContext.Revert { - if err := mgtr.applier.CreateChangelogTable(); err != nil { - mgtr.migrationContext.Log.Errorf("unable to create changelog table, see further error details. Perhaps a previous migration failed without dropping the table? OR is there a running migration? Bailing out") - return err - } - } else if !mgtr.migrationContext.Resume { - if err := mgtr.applier.ValidateOrDropExistingTables(); err != nil { - return err - } - if err := mgtr.applier.CreateChangelogTable(); err != nil { - mgtr.migrationContext.Log.Errorf("unable to create changelog table, see further error details. Perhaps a previous migration failed without dropping the table? OR is there a running migration? Bailing out") - return err - } - if err := mgtr.applier.CreateGhostTable(); err != nil { - mgtr.migrationContext.Log.Errorf("unable to create ghost table, see further error details. Perhaps a previous migration failed without dropping the table? Bailing out") - return err - } - if err := mgtr.applier.AlterGhost(); err != nil { - mgtr.migrationContext.Log.Errorf("unable to ALTER ghost table, see further error details. Bailing out") - return err - } - if mgtr.migrationContext.OriginalTableAutoIncrement > 0 && !mgtr.parser.IsAutoIncrementDefined() { - // Original table has AUTO_INCREMENT value and the -alter statement does not indicate any override, - // so we should copy AUTO_INCREMENT value onto our ghost table. - if err := mgtr.applier.AlterGhostAutoIncrement(); err != nil { - mgtr.migrationContext.Log.Errorf("unable to ALTER ghost table AUTO_INCREMENT value, see further error details. Bailing out") + if mgtr.migrationContext.IsMoveTablesMode() { + if !mgtr.migrationContext.Resume { + // Fail early and cleanly: if any target table already exists, abort + // before creating any of them so we never leave a partially-created set + // on the target. + if err := mgtr.applier.ValidateMoveTablesTargetsAbsent(); err != nil { return err } + // Create every migrated table on the target from its captured CREATE + // statement (§2.1). Containers were populated by prepareMoveTablesCopyState. + for _, mt := range mgtr.migrationContext.OrderedMoveTables() { + createTableStatement := mt.CreateTableStatement + if createTableStatement == "" { + var err error + if createTableStatement, err = mgtr.inspector.showCreateTable(mt.SourceTableName); err != nil { + return fmt.Errorf("failed to fetch create table statement for %s.%s: %w", + sql.EscapeName(mt.SourceDatabaseName), sql.EscapeName(mt.SourceTableName), err) + } + } + if err := mgtr.applier.CreateTargetTableForName(mt.TargetTableName, createTableStatement); err != nil { + mgtr.migrationContext.Log.Errorf("unable to create target table %s.%s, see further error details. Perhaps a previous migration failed without dropping the table? Bailing out", + sql.EscapeName(mt.TargetDatabaseName), sql.EscapeName(mt.TargetTableName)) + return err + } + } + } else { + mgtr.migrationContext.Log.Infof("Resuming move-tables; reusing existing target tables %v in %s", + mgtr.migrationContext.MoveTables.TableNames, + sql.EscapeName(mgtr.migrationContext.GetTargetDatabaseName()), + ) } - if _, err := mgtr.applier.WriteChangelogState(string(GhostTableMigrated)); err != nil { - return err + } else { + if mgtr.migrationContext.Revert { + if err := mgtr.applier.CreateChangelogTable(); err != nil { + mgtr.migrationContext.Log.Errorf("unable to create changelog table, see further error details. Perhaps a previous migration failed without dropping the table? OR is there a running migration? Bailing out") + return err + } + } else if !mgtr.migrationContext.Resume { + if err := mgtr.applier.ValidateOrDropExistingTables(); err != nil { + return err + } + if err := mgtr.applier.CreateChangelogTable(); err != nil { + mgtr.migrationContext.Log.Errorf("unable to create changelog table, see further error details. Perhaps a previous migration failed without dropping the table? OR is there a running migration? Bailing out") + return err + } + if err := mgtr.applier.CreateGhostTable(); err != nil { + mgtr.migrationContext.Log.Errorf("unable to create ghost table, see further error details. Perhaps a previous migration failed without dropping the table? Bailing out") + return err + } + if err := mgtr.applier.AlterGhost(); err != nil { + mgtr.migrationContext.Log.Errorf("unable to ALTER ghost table, see further error details. Bailing out") + return err + } + + if mgtr.migrationContext.OriginalTableAutoIncrement > 0 && !mgtr.parser.IsAutoIncrementDefined() { + // Original table has AUTO_INCREMENT value and the -alter statement does not indicate any override, + // so we should copy AUTO_INCREMENT value onto our ghost table. + if err := mgtr.applier.AlterGhostAutoIncrement(); err != nil { + mgtr.migrationContext.Log.Errorf("unable to ALTER ghost table AUTO_INCREMENT value, see further error details. Bailing out") + return err + } + } + if _, err := mgtr.applier.WriteChangelogState(string(GhostTableMigrated)); err != nil { + return err + } } } @@ -1647,7 +2691,9 @@ func (mgtr *Migrator) initiateApplier() error { mgtr.migrationContext.Log.Warning("proceeding without metadata lock check. There is a small chance of data loss if another session accesses the ghost table during cut-over. See https://github.com/github/gh-ost/pull/1536 for details") } - go mgtr.applier.InitiateHeartbeat() + if !mgtr.migrationContext.IsMoveTablesMode() { + go mgtr.applier.InitiateHeartbeat() + } return nil } @@ -1689,7 +2735,7 @@ func (mgtr *Migrator) iterateChunks() error { } // When hasFurtherRange is false, original table might be write locked and CalculateNextIterationRangeEndValues would hangs forever - hasFurtherRange, err := mgtr.applier.CalculateNextIterationRangeEndValues() + hasFurtherRange, err := mgtr.applier.CalculateNextIterationRangeEndValues(nil) if err != nil { return err // wrapping call will retry } @@ -1712,6 +2758,7 @@ func (mgtr *Migrator) iterateChunks() error { if err != nil { return err // wrapping call will retry } + mgtr.migrationContext.Log.Debugf("ApplyIterationInsertQuery affected %d rows", rowsAffected) if mgtr.migrationContext.PanicOnWarnings { if len(mgtr.migrationContext.MigrationLastInsertSQLWarnings) > 0 { @@ -1753,7 +2800,126 @@ func (mgtr *Migrator) iterateChunks() error { } } +// iterateChunksMoveTables drives the interleaved, multi-table row copy (§2.3). +// It round-robins over the migrated tables in --move-tables order, enqueuing one +// chunk-copy task per not-yet-complete table per cycle so all tables make +// progress concurrently through the single shared apply pipeline. Each task +// operates on its table's own per-table container; the single executeWriteFuncs +// consumer runs the tasks one at a time, so per-table range/iteration state is +// never accessed concurrently. Row copy is complete only once EVERY table +// reports complete, at which point the shared rowCopyComplete signal fires once +// (so the on-row-copy-complete hook and cutover fire exactly once, after the +// slowest table). +func (mgtr *Migrator) iterateChunksMoveTables() error { + terminateRowIteration := func(err error) error { + _ = base.SendWithContext(mgtr.migrationContext.GetContext(), mgtr.rowCopyComplete, err) + if err != nil { + return mgtr.migrationContext.Log.Errore(err) + } + return nil + } + if mgtr.migrationContext.Noop { + mgtr.migrationContext.Log.Debugf("Noop operation; not really copying data") + return terminateRowIteration(nil) + } + + tables := mgtr.migrationContext.OrderedMoveTables() + // A table with no rows is immediately complete. + for _, mt := range tables { + if mt.MigrationRangeMinValues == nil { + mgtr.migrationContext.Log.Debugf("No rows found in %s.%s; row copy is implicitly empty", + sql.EscapeName(mt.SourceDatabaseName), sql.EscapeName(mt.SourceTableName)) + mt.SetRowCopyComplete() + } + } + + // enqueueChunk builds and enqueues a single chunk-copy task bound to mt. + enqueueChunk := func(mt *base.MoveTable) error { + copyRowsFunc := func() error { + if mt.IsRowCopyComplete() || atomic.LoadInt64(&mgtr.rowCopyCompleteFlag) == 1 { + return nil + } + mt.SetNextIterationRangeMinValues() + applyCopyRowsFunc := func() error { + if mt.IsRowCopyComplete() || atomic.LoadInt64(&mgtr.rowCopyCompleteFlag) == 1 { + return nil + } + hasFurtherRange, err := mgtr.applier.CalculateMoveTableNextIterationRangeEndValues(mgtr.inspector.db, mt) + if err != nil { + return err // wrapping call will retry + } + if !hasFurtherRange { + mt.SetRowCopyComplete() + return nil + } + if atomic.LoadInt64(&mgtr.rowCopyCompleteFlag) == 1 { + return nil + } + _, rowsAffected, _, err := mgtr.applier.ApplyIterationMoveTableCopyQueries(mgtr.inspector.db, mt) + if err != nil { + return err // wrapping call will retry + } + if mgtr.migrationContext.PanicOnWarnings && len(mgtr.migrationContext.MigrationLastInsertSQLWarnings) > 0 { + for _, warning := range mgtr.migrationContext.MigrationLastInsertSQLWarnings { + mgtr.migrationContext.Log.Infof("move-table copy on %s.%s has SQL warnings! %s", + sql.EscapeName(mt.SourceDatabaseName), sql.EscapeName(mt.SourceTableName), warning) + } + joined := strings.Join(mgtr.migrationContext.MigrationLastInsertSQLWarnings, "; ") + return fmt.Errorf("move-table copy on %s.%s failed because of SQL warnings: [%s]", + sql.EscapeName(mt.SourceDatabaseName), sql.EscapeName(mt.SourceTableName), joined) + } + atomic.AddInt64(&mgtr.migrationContext.TotalRowsCopied, rowsAffected) + atomic.AddInt64(&mt.RowsCopied, rowsAffected) + mt.IncrementIteration() + return nil + } + if err := mgtr.retryBatchCopyWithHooks(applyCopyRowsFunc); err != nil { + return err + } + // Record this table's last successfully-copied range for checkpointing. + // Skip the final completion-detection pass: it advanced the iteration min + // to the previous max without copying anything (and set rowCopyComplete), + // so recording here would overwrite the real [min..max] span of the last + // actual chunk with a degenerate [max..max]. + if !mt.IsRowCopyComplete() { + mt.RecordLastIterationRange() + } + return nil + } + return base.SendWithContext(mgtr.migrationContext.GetContext(), mgtr.copyRowsQueue, copyRowsFunc) + } + + for { + if err := mgtr.checkAbort(); err != nil { + return terminateRowIteration(err) + } + if atomic.LoadInt64(&mgtr.rowCopyCompleteFlag) == 1 || mgtr.migrationContext.AllMoveTablesRowCopyComplete() { + return terminateRowIteration(nil) + } + for _, mt := range tables { + if mt.IsRowCopyComplete() { + continue + } + if err := enqueueChunk(mt); err != nil { + if abortErr := mgtr.checkAbort(); abortErr != nil { + return terminateRowIteration(abortErr) + } + return terminateRowIteration(err) + } + // Mirrors the standard iterateChunks failpoint: fires after a chunk is + // enqueued so resume tests can crash mid-copy (move-tables uses this + // loop, not iterateChunks, so the failpoint must live here too). + mgtr.migrationContext.NewFailPoint("move-tables-panic-after-row-copy", base.WithFailPointWait(2*time.Second)) + if atomic.LoadInt64(&mgtr.rowCopyCompleteFlag) == 1 { + return nil + } + } + } +} + func (mgtr *Migrator) onApplyEventStruct(eventStruct *applyEventStruct) error { + atomic.AddInt64(&mgtr.applyEventsInFlight, 1) + defer atomic.AddInt64(&mgtr.applyEventsInFlight, -1) handleNonDMLEventStruct := func(eventStruct *applyEventStruct) error { if eventStruct.writeFunc != nil { if err := mgtr.retryOperation(*eventStruct.writeFunc); err != nil { @@ -1768,6 +2934,7 @@ func (mgtr *Migrator) onApplyEventStruct(eventStruct *applyEventStruct) error { if eventStruct.dmlEvent != nil { dmlEvents := [](*binlog.BinlogDMLEvent){} dmlEvents = append(dmlEvents, eventStruct.dmlEvent) + lastEventTimestamp := eventStruct.eventTimestamp var nonDmlStructToApply *applyEventStruct availableEvents := len(mgtr.applyEventsQueue) @@ -1785,6 +2952,7 @@ func (mgtr *Migrator) onApplyEventStruct(eventStruct *applyEventStruct) error { break } dmlEvents = append(dmlEvents, additionalStruct.dmlEvent) + lastEventTimestamp = additionalStruct.eventTimestamp } // Create a task to apply the DML event; this will be execute by executeWriteFuncs() var applyEventFunc tableWriteFunc = func() error { @@ -1798,6 +2966,12 @@ func (mgtr *Migrator) onApplyEventStruct(eventStruct *applyEventStruct) error { mgtr.applier.CurrentCoordinates = eventStruct.coords mgtr.applier.CurrentCoordinatesMutex.Unlock() + // In move-tables mode there is no changelog heartbeat; writer lag is derived + // from the binlog-header timestamp of the last event we just applied. + if mgtr.migrationContext.IsMoveTablesMode() && !lastEventTimestamp.IsZero() { + mgtr.migrationContext.UpdateLastAppliedBinlogEventTime(lastEventTimestamp) + } + if nonDmlStructToApply != nil { // We pulled DML events from the queue, and then we hit a non-DML event. Wait! // We need to handle it! @@ -1814,6 +2988,9 @@ func (mgtr *Migrator) onApplyEventStruct(eventStruct *applyEventStruct) error { // applier reaches that trx. At that point it's safe to resume from these coordinates. func (mgtr *Migrator) Checkpoint(ctx context.Context) (*Checkpoint, error) { coords := mgtr.eventsStreamer.GetCurrentBinlogCoordinates() + if mgtr.migrationContext.IsMoveTablesMode() { + return mgtr.checkpointMoveTables(ctx, coords) + } mgtr.applier.LastIterationRangeMutex.Lock() if mgtr.applier.LastIterationRangeMaxValues == nil || mgtr.applier.LastIterationRangeMinValues == nil { mgtr.applier.LastIterationRangeMutex.Unlock() @@ -1841,6 +3018,71 @@ func (mgtr *Migrator) Checkpoint(ctx context.Context) (*Checkpoint, error) { return chk, err } mgtr.applier.CurrentCoordinatesMutex.Unlock() + time.Sleep(500 * time.Millisecond) + } +} + +// buildMoveTableCheckpointRows builds one checkpoint row per migrated table. The +// run-wide fields (coords, total DML, cutover markers, drain GTID) are shared by +// every row; the per-table fields (iteration range, iteration, rows-copied) come +// from each table's own container. There is no representative table. +func (mgtr *Migrator) buildMoveTableCheckpointRows(coords mysql.BinlogCoordinates, isCutover, cutoverStarted bool, drainGTID mysql.BinlogCoordinates) []*Checkpoint { + totalDML := atomic.LoadInt64(&mgtr.migrationContext.TotalDMLEventsApplied) + tables := mgtr.migrationContext.OrderedMoveTables() + rows := make([]*Checkpoint, 0, len(tables)) + for _, mt := range tables { + rangeMin, rangeMax := mt.GetLastIterationRange() + rows = append(rows, &Checkpoint{ + TableName: mt.SourceTableName, + LastTrxCoords: coords, + IterationRangeMin: rangeMin, + IterationRangeMax: rangeMax, + Iteration: mt.GetIteration(), + RowsCopied: mt.GetRowsCopied(), + DMLApplied: totalDML, + IsCutover: isCutover, + MoveTablesCutOverStarted: cutoverStarted, + MoveTablesCutOverDrainGTID: drainGTID, + }) + } + return rows +} + +// moveTablesCheckpointSummary returns a representative-free Checkpoint used only +// for logging a single checkpoint event. Its (empty) range serializes to "". +func (mgtr *Migrator) moveTablesCheckpointSummary(coords mysql.BinlogCoordinates) *Checkpoint { + return &Checkpoint{ + LastTrxCoords: coords, + IterationRangeMin: sql.NewColumnValues(0), + IterationRangeMax: sql.NewColumnValues(0), + RowsCopied: atomic.LoadInt64(&mgtr.migrationContext.TotalRowsCopied), + DMLApplied: atomic.LoadInt64(&mgtr.migrationContext.TotalDMLEventsApplied), + } +} + +// checkpointMoveTables writes one checkpoint row per migrated table once the +// streamer frontier is known to be applied (or, on a quiet source with no +// backlog, treats the frontier as applied since move-tables emits no heartbeat). +func (mgtr *Migrator) checkpointMoveTables(ctx context.Context, coords mysql.BinlogCoordinates) (*Checkpoint, error) { + for { + if err := ctx.Err(); err != nil { + return nil, err + } + mgtr.applier.CurrentCoordinatesMutex.Lock() + applied := coords.SmallerThanOrEquals(mgtr.applier.CurrentCoordinates) + idle := len(mgtr.applyEventsQueue) == 0 && (mgtr.eventsStreamer == nil || len(mgtr.eventsStreamer.eventsChannel) == 0) + if applied || idle { + if !applied { + mgtr.applier.CurrentCoordinates = coords.Clone() + } + mgtr.applier.CurrentCoordinatesMutex.Unlock() + rows := mgtr.buildMoveTableCheckpointRows(coords, false, false, nil) + if err := mgtr.applier.WriteMoveTableCheckpoints(rows); err != nil { + return nil, err + } + return mgtr.moveTablesCheckpointSummary(coords), nil + } + mgtr.applier.CurrentCoordinatesMutex.Unlock() sleepDuration := 500 * time.Millisecond metrics.RecordSleep(mgtr.migrationContext.Metrics, "replica_wait", sleepDuration) time.Sleep(sleepDuration) @@ -1899,6 +3141,13 @@ func (mgtr *Migrator) checkpointLoop() { } else { mgtr.migrationContext.Log.Errorf("error attempting checkpoint: %+v", err) } + } else if mgtr.migrationContext.IsMoveTablesMode() { + // Move-tables writes one checkpoint row per table; the per-table range + // and iteration live in those rows (and the status output). The single + // run-wide summary line has no representative range, so report the + // aggregate progress instead of the (empty) single-table range fields. + mgtr.migrationContext.Log.Infof("checkpoint success at coords=%+v tables=%d rows_copied=%d dml_applied=%d", + chk.LastTrxCoords.DisplayString(), len(mgtr.migrationContext.MoveTables.TableNames), chk.RowsCopied, chk.DMLApplied) } else { mgtr.migrationContext.Log.Infof("checkpoint success at coords=%+v range_min=%+v range_max=%+v iteration=%d", chk.LastTrxCoords.DisplayString(), chk.IterationRangeMin.String(), chk.IterationRangeMax.String(), chk.Iteration) @@ -1998,23 +3247,30 @@ func (mgtr *Migrator) executeDMLWriteFuncs() error { func (mgtr *Migrator) finalCleanup() error { atomic.StoreInt64(&mgtr.migrationContext.CleanupImminentFlag, 1) - mgtr.migrationContext.Log.Infof("Writing changelog state: %+v", Migrated) - if _, err := mgtr.applier.WriteChangelogState(string(Migrated)); err != nil { - return err - } + if !mgtr.migrationContext.IsMoveTablesMode() { + mgtr.migrationContext.Log.Infof("Writing changelog state: %+v", Migrated) + if _, err := mgtr.applier.WriteChangelogState(string(Migrated)); err != nil { + return err + } - if mgtr.migrationContext.Noop { - if createTableStatement, err := mgtr.inspector.showCreateTable(mgtr.migrationContext.GetGhostTableName()); err == nil { - mgtr.migrationContext.Log.Infof("New table structure follows") - fmt.Println(createTableStatement) - } else { - mgtr.migrationContext.Log.Errore(err) + if mgtr.migrationContext.Noop { + if createTableStatement, err := mgtr.inspector.showCreateTable(mgtr.migrationContext.GetGhostTableName()); err == nil { + mgtr.migrationContext.Log.Infof("New table structure follows") + fmt.Println(createTableStatement) + } else { + mgtr.migrationContext.Log.Errore(fmt.Errorf("error showing create table: %w", err)) + } } } + if err := mgtr.eventsStreamer.Close(); err != nil { mgtr.migrationContext.Log.Errore(err) } + if mgtr.migrationContext.IsMoveTablesMode() { + return mgtr.moveTablesFinalCleanup() + } + if err := mgtr.retryOperation(mgtr.applier.DropChangelogTable); err != nil { return err } @@ -2042,6 +3298,96 @@ func (mgtr *Migrator) finalCleanup() error { return nil } +// moveTablesFinalCleanup handles artifact cleanup after a successful move-tables +// run. A successful run leaves two artifacts behind: +// the source `__del` table (the post-cutover rollback handle) and the target +// checkpoint table. There are no `_ghc`/`__gho` tables in move-tables mode. +// +// This runs only on the success path; on failure `__del` is never dropped and +// survives as the rollback handle (see logMoveTablesRollbackHint). +func (mgtr *Migrator) moveTablesFinalCleanup() error { + sourceDatabaseName := mgtr.migrationContext.DatabaseName + targetDatabaseName := mgtr.migrationContext.GetTargetDatabaseName() + checkpointTableName := mgtr.migrationContext.GetCheckpointTableName() + + // A resumed noop reuses the target tables and checkpoint from the interrupted + // migration, so it must preserve both. A fresh noop creates target tables and + // a checkpoint only for schema validation, so it must remove those artifacts + // regardless of --ok-to-drop-table. + if mgtr.migrationContext.Noop { + if mgtr.migrationContext.Resume { + return nil + } + for _, mt := range mgtr.migrationContext.OrderedMoveTables() { + if err := mgtr.retryOperation(func() error { + return mgtr.applier.dropTable(mt.TargetTableName) + }); err != nil { + return err + } + } + if mgtr.migrationContext.Checkpoint { + if err := mgtr.retryOperation(mgtr.applier.DropCheckpointTable); err != nil { + return err + } + } + return nil + } + + if mgtr.migrationContext.OkToDropTable { + // The source `__del` rollback handle only exists after a real cutover, + // never in Noop runs. It must be dropped on the source primary: the + // inspector/streamer source connections may point at a read replica, so the + // drop goes through the dedicated source-primary handle. + if err := mgtr.retryOperation(mgtr.dropMoveTablesSourceOldTables); err != nil { + return err + } + if mgtr.migrationContext.Checkpoint { + if err := mgtr.retryOperation(mgtr.applier.DropCheckpointTable); err != nil { + return err + } + } + return nil + } + + // --ok-to-drop-table not set: log the artifacts left behind and the exact + // commands to drop them. In multi-table mode every migrated table leaves its + // own `_
_del` rollback handle on the source. + mgtr.migrationContext.Log.Infof("Am not dropping move-tables artifacts without `--ok-to-drop-table`. The following are left behind:") + for _, tableName := range mgtr.migrationContext.MoveTables.TableNames { + delTableName := mgtr.migrationContext.MoveTableDelName(tableName) + mgtr.migrationContext.Log.Infof("- source rollback handle %s.%s. To drop it, issue:", sql.EscapeName(sourceDatabaseName), sql.EscapeName(delTableName)) + mgtr.migrationContext.Log.Infof("-- drop table %s.%s", sql.EscapeName(sourceDatabaseName), sql.EscapeName(delTableName)) + } + if mgtr.migrationContext.Checkpoint { + mgtr.migrationContext.Log.Infof("- target checkpoint table %s.%s. To drop it, issue:", sql.EscapeName(targetDatabaseName), sql.EscapeName(checkpointTableName)) + mgtr.migrationContext.Log.Infof("-- drop table %s.%s", sql.EscapeName(targetDatabaseName), sql.EscapeName(checkpointTableName)) + } + return nil +} + +// logMoveTablesRollbackHint prints a clear rollback hint after a failed +// move-tables run in which the source RENAME already happened. Each migrated +// table's source `_
_del` table is intentionally left in place as the +// rollback handle and the operator rolls the source back by renaming every +// `_
_del` back to its original table name. We do NOT drop `__del` on a +// failure path. +func (mgtr *Migrator) logMoveTablesRollbackHint() { + sourceDatabaseName := mgtr.migrationContext.DatabaseName + mgtr.migrationContext.Log.Infof("move-tables run failed after the source rename; leaving the following rollback handle(s) in place:") + rollbackClauses := make([]string, 0, len(mgtr.migrationContext.MoveTables.TableNames)) + for _, tableName := range mgtr.migrationContext.MoveTables.TableNames { + delTableName := mgtr.migrationContext.MoveTableDelName(tableName) + mgtr.migrationContext.Log.Infof("- %s.%s (rollback handle for %s.%s)", + sql.EscapeName(sourceDatabaseName), sql.EscapeName(delTableName), + sql.EscapeName(sourceDatabaseName), sql.EscapeName(tableName)) + rollbackClauses = append(rollbackClauses, fmt.Sprintf("%s.%s to %s.%s", + sql.EscapeName(sourceDatabaseName), sql.EscapeName(delTableName), + sql.EscapeName(sourceDatabaseName), sql.EscapeName(tableName))) + } + mgtr.migrationContext.Log.Infof("To roll back the source table(s), issue:") + mgtr.migrationContext.Log.Infof("-- rename table %s", strings.Join(rollbackClauses, ", ")) +} + func (mgtr *Migrator) teardown() { atomic.StoreInt64(&mgtr.finishedMigrating, 1) @@ -2064,4 +3410,9 @@ func (mgtr *Migrator) teardown() { mgtr.migrationContext.Log.Infof("Tearing down throttler") mgtr.throttler.Teardown() } + + if mgtr.sourcePrimaryDB != nil { + mgtr.migrationContext.Log.Infof("Tearing down source primary connection") + mgtr.sourcePrimaryDB.Close() + } } diff --git a/go/logic/migrator_move_tables_cleanup_test.go b/go/logic/migrator_move_tables_cleanup_test.go new file mode 100644 index 000000000..48b95b0ce --- /dev/null +++ b/go/logic/migrator_move_tables_cleanup_test.go @@ -0,0 +1,83 @@ +package logic + +import ( + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/github/gh-ost/go/base" +) + +// capturingLogger records Infof messages so tests can assert on the operator +// commands emitted by the move-tables cleanup path. It embeds base.Logger so it +// only needs to override the one method the tests care about. +type capturingLogger struct { + base.Logger + infofs []string +} + +func (l *capturingLogger) Infof(format string, args ...interface{}) { + l.infofs = append(l.infofs, fmt.Sprintf(format, args...)) +} + +func (l *capturingLogger) has(substr string) bool { + for _, line := range l.infofs { + if strings.Contains(line, substr) { + return true + } + } + return false +} + +func newCleanupTestMigrator() (*Migrator, *capturingLogger) { + logger := &capturingLogger{Logger: base.NewDefaultLogger()} + mc := base.NewMigrationContext() + mc.Log = logger + mc.DatabaseName = "source_db" + mc.OriginalTableName = "t" + mc.MoveTables.TableNames = []string{"t"} + mc.MoveTables.TargetDatabase = "target_db" + return NewMigrator(mc, "test"), logger +} + +// TestMoveTablesFinalCleanup_EmitsOperatorCommands verifies that without +// --ok-to-drop-table, cleanup drops nothing and instead logs the exact +// `drop table` commands for the source rollback handle and the target +// checkpoint table. The applier/streamer are nil, so any real drop would panic. +func TestMoveTablesFinalCleanup_EmitsOperatorCommands(t *testing.T) { + m, logger := newCleanupTestMigrator() + m.migrationContext.OkToDropTable = false + m.migrationContext.Checkpoint = true + + require.NoError(t, m.moveTablesFinalCleanup()) + + require.True(t, logger.has("-- drop table `source_db`.`_t_del`"), + "must emit the command to drop the source rollback handle") + require.True(t, logger.has(fmt.Sprintf("-- drop table `target_db`.`%s`", m.migrationContext.GetCheckpointTableName())), + "must emit the command to drop the target checkpoint table") +} + +// TestLogMoveTablesRollbackHint_EmitsRenameCommand verifies the failure-path +// hint emits the exact rename command to roll the source table back. +func TestLogMoveTablesRollbackHint_EmitsRenameCommand(t *testing.T) { + m, logger := newCleanupTestMigrator() + + m.logMoveTablesRollbackHint() + + require.True(t, logger.has("-- rename table `source_db`.`_t_del` to `source_db`.`t`"), + "must emit the rename command to roll the source table back") +} + +// TestDropMoveTablesSourceOldTables_NilSourcePrimaryErrors verifies the source +// `__del` drop fails cleanly (rather than panicking) when the source-primary +// connection was never initialized. The drop must never silently no-op. +func TestDropMoveTablesSourceOldTables_NilSourcePrimaryErrors(t *testing.T) { + m, _ := newCleanupTestMigrator() + + err := m.dropMoveTablesSourceOldTables() + + require.Error(t, err) + require.Contains(t, err.Error(), "source primary connection not initialized") +} diff --git a/go/logic/migrator_move_tables_cutover_test.go b/go/logic/migrator_move_tables_cutover_test.go new file mode 100644 index 000000000..1f25cddc5 --- /dev/null +++ b/go/logic/migrator_move_tables_cutover_test.go @@ -0,0 +1,642 @@ +package logic + +import ( + "context" + gosql "database/sql" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/github/gh-ost/go/base" + "github.com/github/gh-ost/go/binlog" + "github.com/github/gh-ost/go/mysql" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + "github.com/testcontainers/testcontainers-go" + testmysql "github.com/testcontainers/testcontainers-go/modules/mysql" +) + +// ----------------------------------------------------------------------------- +// Pure unit tests - no MySQL. These exercise the orchestration branches that +// run BEFORE T1's RENAME, so they do not require a real source-primary DB. The +// "RENAME was not attempted" check is a proxy assertion: m.sourcePrimaryDB is +// nil, so if T1 were reached the test would fail with the source-primary-not- +// initialized error instead of the earlier error it asserts on. +// ----------------------------------------------------------------------------- + +// TestMoveTablesCutOver_NoopShortCircuits maps to the Noop semantics decision +// in #8209-implement-protocol (Noop returns immediately, no postpone gate, no +// hooks, no RENAME). +func TestMoveTablesCutOver_NoopShortCircuits(t *testing.T) { + var calls []string + fakeHooks := &recordingHooks{name: "fake", calls: &calls} + + ctx := base.NewMigrationContext() + ctx.Noop = true + ctx.Hooks = fakeHooks + + m := NewMigrator(ctx, "test") + + require.Equal(t, int64(0), atomic.LoadInt64(&ctx.CutOverCompleteFlag), "pre-state: flag must be 0") + require.Empty(t, calls, "pre-state: no hooks recorded") + + require.NoError(t, m.moveTablesCutOver()) + + require.Equal(t, int64(0), atomic.LoadInt64(&ctx.CutOverCompleteFlag), + "post-state: Noop must not set CutOverCompleteFlag") + require.Empty(t, calls, "post-state: Noop must not fire any hook") +} + +// TestMoveTablesCutOver_OnBeforeCutOverHookAbortsBeforeRename maps to T0 in +// coop_cutover.md section 1.3 ("non-zero return code aborts cutover"). The "aborts +// BEFORE source DDL" assertion is enforced as a proxy: m.sourcePrimaryDB is nil, +// so if T1 RENAME executed it would fail with the source-primary-not-initialized +// error rather than the asserted hook error. +func TestMoveTablesCutOver_OnBeforeCutOverHookAbortsBeforeRename(t *testing.T) { + var calls []string + boom := errors.New("hook says no") + fakeHooks := &recordingHooks{name: "fake", calls: &calls, errOn: "OnBeforeCutOver", errVal: boom} + + ctx := base.NewMigrationContext() + ctx.Hooks = fakeHooks + ctx.DatabaseName = "test" + ctx.OriginalTableName = "t" + + m := NewMigrator(ctx, "test") + + require.Equal(t, int64(0), atomic.LoadInt64(&ctx.CutOverCompleteFlag), "pre-state: flag must be 0") + require.Empty(t, calls, "pre-state: no hooks recorded") + + err := m.moveTablesCutOver() + require.Error(t, err) + require.ErrorIs(t, err, boom) + require.Contains(t, err.Error(), "on-before-cut-over hook failed") + + require.Equal(t, int64(0), atomic.LoadInt64(&ctx.CutOverCompleteFlag), + "post-state: T0 abort must leave CutOverCompleteFlag unset") + require.Equal(t, []string{"fake:OnBeforeCutOver"}, calls, + "post-state: only the failing T0 hook fires; no OnSuccess, no OnBeginPostponed") +} + +type onSuccessCheckHooks struct { + *recordingHooks + onSuccessCheck func() error +} + +func (h *onSuccessCheckHooks) OnSuccess(bool) error { + if err := h.record("OnSuccess"); err != nil { + return err + } + if h.onSuccessCheck != nil { + return h.onSuccessCheck() + } + return nil +} + +// TestMoveTablesCutOver_PostponeGateFiresOnBeginPostponedOnce maps to the +// postpone-gate decision in #8209-implement-protocol (keep OnBeginPostponed +// firing logic with the same once-per-cutover semantics as standard cutOver). +// Also exercises Edge Case Test Quality #5 by asserting both pre-state and +// post-state of IsPostponingCutOver. +func TestMoveTablesCutOver_PostponeGateFiresOnBeginPostponedOnce(t *testing.T) { + flagPath := filepath.Join(t.TempDir(), "postpone.flag") + require.NoError(t, os.WriteFile(flagPath, nil, 0o644)) + + var calls []string + boom := errors.New("we got past the gate") + fakeHooks := &recordingHooks{name: "fake", calls: &calls, errOn: "OnBeforeCutOver", errVal: boom} + + ctx := base.NewMigrationContext() + ctx.PostponeCutOverFlagFile = flagPath + ctx.Hooks = fakeHooks + + m := NewMigrator(ctx, "test") + + require.Equal(t, int64(0), atomic.LoadInt64(&ctx.IsPostponingCutOver), "pre-state: gate flag must be 0") + require.FileExists(t, flagPath, "pre-state: postpone flag file present") + require.Empty(t, calls, "pre-state: no hooks recorded") + + // Remove the flag file during the gate's first 1s sleep so the next + // poll exits cleanly. sleepWhileTrue uses time.Sleep(1 * time.Second). + go func() { + time.Sleep(500 * time.Millisecond) + _ = os.Remove(flagPath) + }() + + err := m.moveTablesCutOver() + require.ErrorIs(t, err, boom, "expect to bail at T0 hook after gate releases") + + onBegin := 0 + for _, c := range calls { + if c == "fake:OnBeginPostponed" { + onBegin++ + } + } + require.Equal(t, 1, onBegin, + "post-state: OnBeginPostponed must fire exactly once per cutover (idempotent via IsPostponingCutOver)") + require.Equal(t, int64(0), atomic.LoadInt64(&ctx.IsPostponingCutOver), + "post-state: gate must reset IsPostponingCutOver to 0 after exit") + require.Contains(t, calls, "fake:OnBeforeCutOver", + "post-state: T0 hook must fire after gate releases") + require.Equal(t, int64(0), atomic.LoadInt64(&ctx.CutOverCompleteFlag), + "post-state: hook failure must leave CutOverCompleteFlag unset") +} + +// TestResumeMoveTablesCutOverFromCheckpointAlreadyDrained verifies the crash- +// safe resume branch skips T1 entirely and proceeds directly to T5 when the +// persisted checkpoint already shows a drain-satisfied position. +func TestResumeMoveTablesCutOverFromCheckpointAlreadyDrained(t *testing.T) { + var calls []string + fakeHooks := &recordingHooks{name: "fake", calls: &calls} + + ctx := base.NewMigrationContext() + ctx.Hooks = fakeHooks + ctx.Checkpoint = false + + m := NewMigrator(ctx, "test") + m.applier = NewApplier(ctx) + + drainGTID, err := mysql.NewGTIDBinlogCoordinates(mysql.MySQLFlavor, "11111111-1111-1111-1111-111111111111:1-10") + require.NoError(t, err) + + chk := &Checkpoint{ + LastTrxCoords: drainGTID, + MoveTablesCutOverStarted: true, + MoveTablesCutOverDrainGTID: drainGTID, + } + + require.Equal(t, int64(0), atomic.LoadInt64(&ctx.CutOverCompleteFlag), "pre-state: flag must be 0") + require.Empty(t, calls, "pre-state: no hooks recorded") + + require.NoError(t, m.resumeMoveTablesCutOverFromCheckpoint(chk)) + + require.Equal(t, int64(1), atomic.LoadInt64(&ctx.CutOverCompleteFlag), + "post-state: resume path must set CutOverCompleteFlag before exiting") + require.NotNil(t, ctx.MoveTables.DrainGTID, + "post-state: resume must set MoveTables.DrainGTID so the on-success hook gets GH_OST_DRAIN_GTID") + require.Equal(t, drainGTID.String(), ctx.MoveTables.DrainGTID.String(), + "post-state: MoveTables.DrainGTID must equal the checkpoint drain GTID") + require.Equal(t, []string{"fake:OnSuccess"}, calls, + "post-state: resume path should jump directly to T5 without rerunning T0/T1") + if m.applier.CurrentCoordinates != nil { + require.Equal(t, drainGTID.String(), m.applier.CurrentCoordinates.String()) + } +} + +// TestResolveSourcePrimaryConnectionConfig_AssumeMasterHostnameOverride verifies +// that --assume-master-host forces the move-tables source primary to the given +// host, and that --master-user/--master-password override the inherited source +// credentials. No DB is required: the override branch builds the config purely +// from the inspector config. +func TestResolveSourcePrimaryConnectionConfig_AssumeMasterHostnameOverride(t *testing.T) { + mc := base.NewMigrationContext() + mc.InspectorConnectionConfig.User = "src_user" + mc.InspectorConnectionConfig.Password = "src_pass" + mc.AssumeMasterHostname = "10.0.0.5:3307" + mc.CliMasterUser = "master_user" + mc.CliMasterPassword = "master_pass" + m := NewMigrator(mc, "test") + + cfg, err := m.resolveSourcePrimaryConnectionConfig("8.0.42") + require.NoError(t, err) + require.Equal(t, "10.0.0.5", cfg.Key.Hostname) + require.Equal(t, 3307, cfg.Key.Port) + require.Equal(t, "master_user", cfg.User, "--master-user must override source credentials") + require.Equal(t, "master_pass", cfg.Password, "--master-password must override source credentials") + + // Without explicit master credentials, the forced primary inherits the source + // (inspector) credentials. + mc.CliMasterUser = "" + mc.CliMasterPassword = "" + cfg, err = m.resolveSourcePrimaryConnectionConfig("8.0.42") + require.NoError(t, err) + require.Equal(t, "src_user", cfg.User) + require.Equal(t, "src_pass", cfg.Password) +} + +// TestValidateMoveTablesSourceReadHost verifies the read-path guard: a replica +// source passes, a source that resolves to the primary is blocked with a hint to +// use a replica or --allow-on-source-primary, and the opt-in flag bypasses it. +func TestValidateMoveTablesSourceReadHost(t *testing.T) { + newMigrator := func(srcKey, primaryKey mysql.InstanceKey, allow bool) *Migrator { + mc := base.NewMigrationContext() + mc.MoveTables.TableNames = []string{"t"} + mc.InspectorConnectionConfig.Key = srcKey + mc.MoveTables.SourcePrimaryConnectionConfig = &mysql.ConnectionConfig{Key: primaryKey} + mc.MoveTables.AllowOnSourcePrimary = allow + return NewMigrator(mc, "test") + } + replica := mysql.InstanceKey{Hostname: "replica.example.com", Port: 3306} + primary := mysql.InstanceKey{Hostname: "primary.example.com", Port: 3306} + + t.Run("source is a replica: passes", func(t *testing.T) { + require.NoError(t, newMigrator(replica, primary, false).validateMoveTablesSourceReadHost()) + }) + + t.Run("source is the primary: blocked", func(t *testing.T) { + err := newMigrator(primary, primary, false).validateMoveTablesSourceReadHost() + require.Error(t, err) + require.Contains(t, err.Error(), "--allow-on-source-primary") + }) + + t.Run("source is the primary but opted in: passes", func(t *testing.T) { + require.NoError(t, newMigrator(primary, primary, true).validateMoveTablesSourceReadHost()) + }) +} + +// ----------------------------------------------------------------------------- +// Integration tests - real MySQL via testcontainers, exercise T1/T2/T3. +// +// These live under a dedicated suite (NOT MigratorTestSuite) so the parent +// function name TestMoveTablesCutOver matches the `-run MoveTablesCutOver` +// verifier alongside the pure unit tests above. SetupSuite duplicates the +// testcontainer setup pattern used by MigratorTestSuite intentionally to +// keep this commit's diff strictly additive (no edits to existing tests or +// shared helpers, per the commit-3 prompt). +// +// Known-environmental flakes when Docker/testcontainers is unavailable mirror +// the same class as #8206; they are not regressions of #8209. +// ----------------------------------------------------------------------------- + +type MoveTablesCutOverSuite struct { + suite.Suite + mysqlContainer testcontainers.Container + db *gosql.DB +} + +func (s *MoveTablesCutOverSuite) SetupSuite() { + ctx := context.Background() + mysqlContainer, err := testmysql.Run(ctx, + testMysqlContainerImage, + testmysql.WithDatabase(testMysqlDatabase), + testmysql.WithUsername(testMysqlUser), + testmysql.WithPassword(testMysqlPass), + testmysql.WithConfigFile("my.cnf.test"), + ) + s.Require().NoError(err) + s.mysqlContainer = mysqlContainer + + dsn, err := mysqlContainer.ConnectionString(ctx) + s.Require().NoError(err) + db, err := gosql.Open("mysql", dsn) + s.Require().NoError(err) + s.db = db +} + +func (s *MoveTablesCutOverSuite) TearDownSuite() { + s.Assert().NoError(s.db.Close()) + s.Assert().NoError(testcontainers.TerminateContainer(s.mysqlContainer)) +} + +func (s *MoveTablesCutOverSuite) SetupTest() { + _, err := s.db.ExecContext(context.Background(), "CREATE DATABASE IF NOT EXISTS "+testMysqlDatabase) + s.Require().NoError(err) +} + +func (s *MoveTablesCutOverSuite) TearDownTest() { + ctx := context.Background() + _, _ = s.db.ExecContext(ctx, "DROP TABLE IF EXISTS "+getTestTableName()) + _, _ = s.db.ExecContext(ctx, "DROP TABLE IF EXISTS "+getTestOldTableName()) +} + +// containingDrainGTID returns a fabricated GTID set guaranteed to contain any +// GTID the test container will assign during the test. RENAME's GTID will be +// a strict subset of this range, so T3's containment poll passes on iteration 1. +func (s *MoveTablesCutOverSuite) containingDrainGTID() *mysql.GTIDBinlogCoordinates { + var serverUUID string + s.Require().NoError(s.db.QueryRow("SELECT @@server_uuid").Scan(&serverUUID)) + g, err := mysql.NewGTIDBinlogCoordinates(mysql.MySQLFlavor, fmt.Sprintf("%s:1-99999999", serverUUID)) + s.Require().NoError(err) + return g +} + +// buildMigrator wires a Migrator with the test container's *sql.DB pinned to +// inspector.db and a fresh Applier. The cutover RENAME + drain-GTID capture run +// on sourcePrimaryDB (a dedicated handle with multiStatements enabled, since +// T1/T2 are issued as a single multi-statement round trip). initialCoords may be +// nil for the drain-timeout case. +func (s *MoveTablesCutOverSuite) buildMigrator(fakeHooks base.Hooks, initialCoords mysql.BinlogCoordinates) (*Migrator, *base.MigrationContext) { + ctx := context.Background() + connectionConfig, err := getTestConnectionConfig(ctx, s.mysqlContainer) + s.Require().NoError(err) + + mc := newTestMigrationContext() + mc.ApplierConnectionConfig = connectionConfig + mc.InspectorConnectionConfig = connectionConfig + mc.MoveTables.SourcePrimaryConnectionConfig = connectionConfig + // Every consumer of buildMigrator is a move-tables cutover test, so put the + // migrator in move-tables mode with the canonical single table. The cutover + // path builds its atomic RENAME from MoveTables.TableNames; leaving it empty + // produces `rename table ;` (Error 1064). + mc.MoveTables.TableNames = []string{testMysqlTableName} + mc.SetConnectionConfig("innodb") + mc.Hooks = fakeHooks + + m := NewMigrator(mc, "test") + m.inspector = &Inspector{db: s.db, migrationContext: mc} + // The source primary handle needs multiStatements enabled for the consolidated + // T1+T2 (RENAME; SELECT @@global.gtid_executed) round trip. + sourcePrimaryDB, _, err := mysql.GetDB(mc.Uuid, connectionConfig.GetDBUri(testMysqlDatabase)+"&multiStatements=true") + s.Require().NoError(err) + m.sourcePrimaryDB = sourcePrimaryDB + m.applier = NewApplier(mc) + if initialCoords != nil { + m.applier.CurrentCoordinatesMutex.Lock() + m.applier.CurrentCoordinates = initialCoords + m.applier.CurrentCoordinatesMutex.Unlock() + } + return m, mc +} + +// TestDropMoveTablesSourceOldTablesUsesSourcePrimary verifies the source `__del` +// rollback handle is dropped through the dedicated source-primary connection. In +// production the inspector/streamer source connections may be a read replica, so +// the drop must not route through them. +func (s *MoveTablesCutOverSuite) TestDropMoveTablesSourceOldTablesUsesSourcePrimary() { + ctx := context.Background() + _, err := s.db.ExecContext(ctx, fmt.Sprintf("CREATE TABLE %s (id INT PRIMARY KEY)", getTestOldTableName())) + s.Require().NoError(err) + + var calls []string + fakeHooks := &recordingHooks{name: "fake", calls: &calls} + m, _ := s.buildMigrator(fakeHooks, s.containingDrainGTID()) + + s.Require().NoError(m.dropMoveTablesSourceOldTables()) + + var name string + err = s.db.QueryRow(fmt.Sprintf("SHOW TABLES IN %s LIKE '_%s_del'", + testMysqlDatabase, testMysqlTableName)).Scan(&name) + s.Require().ErrorIs(err, gosql.ErrNoRows, "source __del handle must be dropped via the source primary") +} + +// TestResolveSourcePrimaryFallsBackToInspectorWhenNoReplica verifies the +// graceful fallback: when the source --host has no upstream primary (the +// standalone test container), master detection returns the inspector connection +// config, so source reads and cutover writes share the one available host. +func (s *MoveTablesCutOverSuite) TestResolveSourcePrimaryFallsBackToInspectorWhenNoReplica() { + var calls []string + m, mc := s.buildMigrator(&recordingHooks{name: "fake", calls: &calls}, nil) + + cfg, err := m.resolveSourcePrimaryConnectionConfig("8.0.42") + s.Require().NoError(err) + s.Require().Equal(mc.InspectorConnectionConfig.Key.Hostname, cfg.Key.Hostname) + s.Require().Equal(mc.InspectorConnectionConfig.Key.Port, cfg.Key.Port) +} + +// TestAssertConnectionWritableRejectsReadOnly verifies the startup writability +// gate: a writable primary passes, and a read_only server is rejected with a +// clear error. The same helper guards both the source primary and the target. +func (s *MoveTablesCutOverSuite) TestAssertConnectionWritableRejectsReadOnly() { + key := mysql.InstanceKey{Hostname: "test-host", Port: 3306} + s.Require().NoError(assertConnectionWritable(s.db, key, "source primary"), + "a writable primary must pass the gate") + + _, err := s.db.Exec("SET GLOBAL read_only = ON") + s.Require().NoError(err) + defer func() { _, _ = s.db.Exec("SET GLOBAL read_only = OFF") }() + + err = assertConnectionWritable(s.db, key, "source primary") + s.Require().Error(err) + s.Require().Contains(err.Error(), "read_only") +} + +// TestHappyPath drives the full T0-T6 protocol against the test container. +// Asserts hook ordering (T0 then T5), T4 flag set, and the source-side rename. +// Maps to acceptance criterion #8209 "RENAME executes; drain completes; +// CutOverCompleteFlag set; on-success hook fires". +func (s *MoveTablesCutOverSuite) TestHappyPath() { + ctx := context.Background() + _, err := s.db.ExecContext(ctx, fmt.Sprintf("CREATE TABLE %s (id INT PRIMARY KEY)", getTestTableName())) + s.Require().NoError(err) + + var calls []string + fakeHooks := &recordingHooks{name: "fake", calls: &calls} + m, mc := s.buildMigrator(fakeHooks, s.containingDrainGTID()) + + s.Require().Equal(int64(0), atomic.LoadInt64(&mc.CutOverCompleteFlag), "pre-state: flag must be 0") + s.Require().Empty(calls, "pre-state: no hooks recorded") + + s.Require().NoError(m.moveTablesCutOver()) + + s.Require().Equal(int64(1), atomic.LoadInt64(&mc.CutOverCompleteFlag), + "post-state: T4 must set CutOverCompleteFlag before T5/T6") + s.Require().Equal([]string{"fake:OnBeforeCutOver", "fake:OnSuccess"}, calls, + "post-state: T0 hook precedes T5 hook") + + // Source-side post-state (Edge Case Test Quality #5: assert both sides). + var renamed string + s.Require().NoError(s.db.QueryRow(fmt.Sprintf("SHOW TABLES IN %s LIKE '_%s_del'", + testMysqlDatabase, testMysqlTableName)).Scan(&renamed)) + s.Require().Equal("_"+testMysqlTableName+"_del", renamed) + + err = s.db.QueryRow(fmt.Sprintf("SHOW TABLES IN %s LIKE '%s'", + testMysqlDatabase, testMysqlTableName)).Scan(&renamed) + s.Require().ErrorIs(err, gosql.ErrNoRows, "original table must no longer exist under its old name") +} + +// TestRenameFailurePropagates maps to T1 failure handling: no source table -> +// RENAME errors. Verify the wrapped error, no CutOverCompleteFlag set, no +// OnSuccess fired. +func (s *MoveTablesCutOverSuite) TestRenameFailurePropagates() { + // Deliberately no CREATE TABLE. + var calls []string + fakeHooks := &recordingHooks{name: "fake", calls: &calls} + m, mc := s.buildMigrator(fakeHooks, s.containingDrainGTID()) + + s.Require().Equal(int64(0), atomic.LoadInt64(&mc.CutOverCompleteFlag), "pre-state: flag must be 0") + + err := m.moveTablesCutOver() + s.Require().Error(err) + s.Require().Contains(err.Error(), "source RENAME + drain GTID capture failed") + + s.Require().Equal(int64(0), atomic.LoadInt64(&mc.CutOverCompleteFlag), + "post-state: RENAME failure must leave CutOverCompleteFlag unset") + for _, c := range calls { + s.Require().NotEqual("fake:OnSuccess", c, "OnSuccess must not fire when RENAME fails") + } + s.Require().Equal([]string{"fake:OnBeforeCutOver"}, calls, + "post-state: only T0 fires before the failed RENAME") +} + +// TestDrainTimeoutPropagates maps to T3 timeout handling: applier coordinates +// never reach the drain GTID -> drain poll bounded by CutOverLockTimeoutSeconds +// returns a wrapped error and the flag is not set. +func (s *MoveTablesCutOverSuite) TestDrainTimeoutPropagates() { + ctx := context.Background() + _, err := s.db.ExecContext(ctx, fmt.Sprintf("CREATE TABLE %s (id INT PRIMARY KEY)", getTestTableName())) + s.Require().NoError(err) + + // Patch poll interval and use a 1-second drain timeout for a bounded test. + origPoll := moveTablesCutOverDrainPollInterval + moveTablesCutOverDrainPollInterval = 50 * time.Millisecond + s.T().Cleanup(func() { + moveTablesCutOverDrainPollInterval = origPoll + }) + + var calls []string + fakeHooks := &recordingHooks{name: "fake", calls: &calls} + // initialCoords nil - drain comparison never satisfies. + m, mc := s.buildMigrator(fakeHooks, nil) + mc.CutOverLockTimeoutSeconds = 1 + + s.Require().Equal(int64(0), atomic.LoadInt64(&mc.CutOverCompleteFlag), "pre-state: flag must be 0") + + err = m.moveTablesCutOver() + s.Require().Error(err) + s.Require().Contains(err.Error(), "drain poll timed out") + s.Require().True(strings.HasPrefix(err.Error(), "drain poll timed out"), + "drain timeout error must name the drain") + + s.Require().Equal(int64(0), atomic.LoadInt64(&mc.CutOverCompleteFlag), + "post-state: drain timeout must abort before T4 flag set") + for _, c := range calls { + s.Require().NotEqual("fake:OnSuccess", c, "OnSuccess must not fire on drain timeout") + } + s.Require().Equal([]string{"fake:OnBeforeCutOver"}, calls, + "post-state: only T0 fires before the drain loop") +} + +// TestDrainWaitsForQueuedDML ensures T3 does not declare success just because +// applier.CurrentCoordinates already contains the drain GTID while there is +// still source-table DML queued for application. +func (s *MoveTablesCutOverSuite) TestDrainWaitsForQueuedDML() { + ctx := context.Background() + _, err := s.db.ExecContext(ctx, fmt.Sprintf("CREATE TABLE %s (id INT PRIMARY KEY)", getTestTableName())) + s.Require().NoError(err) + + origPoll := moveTablesCutOverDrainPollInterval + moveTablesCutOverDrainPollInterval = 50 * time.Millisecond + s.T().Cleanup(func() { + moveTablesCutOverDrainPollInterval = origPoll + }) + + var calls []string + fakeHooks := &recordingHooks{name: "fake", calls: &calls} + m, mc := s.buildMigrator(fakeHooks, s.containingDrainGTID()) + mc.CutOverLockTimeoutSeconds = 1 + m.applyEventsQueue <- newApplyEventStructByDML(&binlog.BinlogEntry{ + DmlEvent: &binlog.BinlogDMLEvent{ + DatabaseName: testMysqlDatabase, + TableName: testMysqlTableName, + DML: binlog.InsertDML, + }, + Coordinates: s.containingDrainGTID(), + }) + + s.Require().Equal(int64(0), atomic.LoadInt64(&mc.CutOverCompleteFlag), "pre-state: flag must be 0") + err = m.moveTablesCutOver() + s.Require().Error(err) + s.Require().Contains(err.Error(), "drain poll timed out") + s.Require().Equal(int64(0), atomic.LoadInt64(&mc.CutOverCompleteFlag), + "post-state: queued DML must keep T3 from reaching T4") + for _, c := range calls { + s.Require().NotEqual("fake:OnSuccess", c, "OnSuccess must not fire while backlog remains") + } + s.Require().Equal([]string{"fake:OnBeforeCutOver"}, calls, + "post-state: only T0 fires before the drain loop times out") +} + +// TestDrainWaitsForInFlightApplyEvent ensures T3 does not complete while an +// apply handler is still running. The blocked handler simulates the last source +// DML not yet landing on the target. If T3 exits early, OnSuccess observes the +// target missing that row and fails immediately. +func (s *MoveTablesCutOverSuite) TestDrainWaitsForInFlightApplyEvent() { + ctx := context.Background() + _, err := s.db.ExecContext(ctx, fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s", testMysqlDatabaseOther)) + s.Require().NoError(err) + _, err = s.db.ExecContext(ctx, fmt.Sprintf("CREATE TABLE %s (id INT PRIMARY KEY)", getTestTableName())) + s.Require().NoError(err) + _, err = s.db.ExecContext(ctx, fmt.Sprintf("CREATE TABLE %s (id INT PRIMARY KEY)", getTestOtherTableName())) + s.Require().NoError(err) + s.T().Cleanup(func() { + _, _ = s.db.ExecContext(context.Background(), "DROP TABLE IF EXISTS "+getTestOtherTableName()) + }) + + drainGTID := s.containingDrainGTID() + const pendingID = 42 + raceErr := errors.New("on-success observed target missing in-flight apply") + + origPoll := moveTablesCutOverDrainPollInterval + moveTablesCutOverDrainPollInterval = 50 * time.Millisecond + s.T().Cleanup(func() { + moveTablesCutOverDrainPollInterval = origPoll + }) + + var calls []string + fakeHooks := &onSuccessCheckHooks{ + recordingHooks: &recordingHooks{name: "fake", calls: &calls}, + onSuccessCheck: func() error { + var count int + query := fmt.Sprintf("SELECT COUNT(*) FROM %s WHERE id = ?", getTestOtherTableName()) + if err := s.db.QueryRowContext(context.Background(), query, pendingID).Scan(&count); err != nil { + return err + } + if count == 0 { + return raceErr + } + return nil + }, + } + m, mc := s.buildMigrator(fakeHooks, drainGTID) + mc.CutOverLockTimeoutSeconds = 1 + m.applier.CurrentCoordinatesMutex.Lock() + m.applier.CurrentCoordinates = drainGTID + m.applier.CurrentCoordinatesMutex.Unlock() + + started := make(chan struct{}) + release := make(chan struct{}) + var startedOnce sync.Once + var releaseOnce sync.Once + releaseApply := func() { + releaseOnce.Do(func() { + close(release) + }) + } + s.T().Cleanup(releaseApply) + blockApply := tableWriteFunc(func() error { + startedOnce.Do(func() { + close(started) + }) + <-release + _, err := s.db.ExecContext(context.Background(), fmt.Sprintf("INSERT INTO %s VALUES (?)", getTestOtherTableName()), pendingID) + return err + }) + go func() { + <-started + time.Sleep(200 * time.Millisecond) + releaseApply() + }() + go func() { + _ = m.onApplyEventStruct(newApplyEventStructByFunc(&blockApply)) + }() + + select { + case <-started: + case <-time.After(time.Second): + s.Require().FailNow("blocked apply event never started") + } + + s.Require().Equal(int64(0), atomic.LoadInt64(&mc.CutOverCompleteFlag), "pre-state: flag must be 0") + err = m.moveTablesCutOver() + s.Require().NoError(err) + + var count int + query := fmt.Sprintf("SELECT COUNT(*) FROM %s WHERE id = ?", getTestOtherTableName()) + s.Require().NoError(s.db.QueryRowContext(context.Background(), query, pendingID).Scan(&count)) + s.Require().Equal(1, count, "post-state: target must contain the pending row by the time cutover succeeds") + s.Require().Equal([]string{"fake:OnBeforeCutOver", "fake:OnSuccess"}, calls, + "post-state: cutover should only reach OnSuccess after the target row is present") +} + +func TestMoveTablesCutOver(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration suite in short mode") + } + suite.Run(t, new(MoveTablesCutOverSuite)) +} diff --git a/go/logic/migrator_test.go b/go/logic/migrator_test.go index ad068691c..8f599280d 100644 --- a/go/logic/migrator_test.go +++ b/go/logic/migrator_test.go @@ -54,6 +54,58 @@ func (buf *buffer) String() string { return buf.Buffer.String() } +func TestMoveTablesWritableColumns(t *testing.T) { + testCases := []struct { + name string + columnNames []string + generatedNames []string + expectedWritable []string + }{ + { + name: "generated columns in middle and end", + columnNames: []string{"id", "virtual_value", "persisted_value", "stored_value"}, + generatedNames: []string{"virtual_value", "stored_value"}, + expectedWritable: []string{"id", "persisted_value"}, + }, + { + name: "generated names match case insensitively", + columnNames: []string{"ID", "Virtual_Value", "persisted_value", "Stored_Value"}, + generatedNames: []string{"virtual_value", "STORED_VALUE"}, + expectedWritable: []string{"ID", "persisted_value"}, + }, + { + name: "no generated columns", + columnNames: []string{"id", "first_value", "second_value"}, + generatedNames: nil, + expectedWritable: []string{"id", "first_value", "second_value"}, + }, + { + name: "all columns generated", + columnNames: []string{"virtual_value", "stored_value"}, + generatedNames: []string{"virtual_value", "stored_value"}, + expectedWritable: []string{}, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + columns := sql.NewColumnList(testCase.columnNames) + generatedColumns := sql.NewColumnList(testCase.generatedNames) + writableColumns := moveTablesWritableColumns(columns, generatedColumns) + + require.Equal(t, testCase.expectedWritable, writableColumns.Names()) + require.NotSame(t, columns, writableColumns) + for ordinal, columnName := range testCase.expectedWritable { + require.Equal(t, ordinal, writableColumns.Ordinals[columnName]) + require.Equal(t, columnName, writableColumns.Columns()[ordinal].Name) + } + + mappedWritableColumns := moveTablesWritableColumns(columns, generatedColumns) + require.NotSame(t, writableColumns, mappedWritableColumns) + }) + } +} + func TestMigratorOnChangelogEvent(t *testing.T) { migrationContext := base.NewMigrationContext() migrator := NewMigrator(migrationContext, "1.2.3") @@ -785,6 +837,134 @@ func (suite *MigratorTestSuite) TestMigrateEmpty() { suite.Require().Equal("_testing_del", tableName) } +func (suite *MigratorTestSuite) TestMoveTablesStateInitializesColumnMetadata() { + ctx := context.Background() + _, err := suite.db.ExecContext(ctx, fmt.Sprintf(` + CREATE TABLE %s ( + id INT PRIMARY KEY, + unsigned_value BIGINT UNSIGNED NOT NULL, + json_value JSON, + virtual_json_value VARCHAR(16) AS ( + COALESCE(JSON_UNQUOTE(JSON_EXTRACT(json_value, '$.value')), 'direct') + ) VIRTUAL, + binary_value BINARY(4) + )`, getTestTableName())) + suite.Require().NoError(err) + + newMoveTablesMigrator := func() (*Migrator, *base.MoveTable) { + migrationContext := newTestMigrationContext() + migrationContext.MoveTables.TableNames = []string{testMysqlTableName} + migrationContext.MoveTables.TargetDatabase = testMysqlDatabase + migrationContext.InitMoveTableContainers() + migrator := NewMigrator(migrationContext, "0.0.0") + return migrator, migrationContext.GetMoveTable(testMysqlTableName) + } + assertHydrated := func(mt *base.MoveTable) { + suite.Require().Equal( + []string{"id", "unsigned_value", "json_value", "binary_value"}, + mt.SharedColumns.Names(), + ) + suite.Require().Equal(sql.JSONColumnType, mt.OriginalTableColumns.GetColumnType("json_value")) + suite.Require().Equal(sql.JSONColumnType, mt.SharedColumns.GetColumnType("json_value")) + suite.Require().Equal(sql.JSONColumnType, mt.MappedSharedColumns.GetColumnType("json_value")) + suite.Require().True(mt.SharedColumns.IsUnsigned("unsigned_value")) + suite.Require().True(mt.MappedSharedColumns.IsUnsigned("unsigned_value")) + suite.Require().Equal(sql.BinaryColumnType, mt.SharedColumns.GetColumnType("binary_value")) + suite.Require().Equal(uint(4), mt.SharedColumns.GetColumn("binary_value").BinaryOctetLength) + } + + suite.Run("fresh preparation", func() { + migrator, mt := newMoveTablesMigrator() + migrator.inspector = &Inspector{db: suite.db, migrationContext: migrator.migrationContext} + suite.Require().NoError(migrator.prepareMoveTablesCopyState()) + assertHydrated(mt) + }) + + suite.Run("resume hydration", func() { + migrator, mt := newMoveTablesMigrator() + migrator.applier = NewApplier(migrator.migrationContext) + migrator.applier.moveTablesTargetDB = suite.db + suite.Require().NoError(migrator.hydrateMoveTablesStateFromTarget()) + assertHydrated(mt) + }) +} + +func (suite *MigratorTestSuite) TestMoveTablesNoopDropsTargetTables() { + ctx := context.Background() + targetTableNames := []string{testMysqlTableName, "test_noop_target"} + + migrationContext := newTestMigrationContext() + migrationContext.Noop = true + migrationContext.Checkpoint = true + migrationContext.MoveTables.TableNames = targetTableNames + migrationContext.MoveTables.TargetDatabase = testMysqlDatabase + migrationContext.InitMoveTableContainers() + + migrator := NewMigrator(migrationContext, "test") + migrator.applier = NewApplier(migrationContext) + migrator.applier.moveTablesTargetDB = suite.db + suite.Require().NoError(migrator.applier.CreateCheckpointTable()) + + for _, tableName := range targetTableNames { + _, err := suite.db.ExecContext(ctx, fmt.Sprintf("CREATE TABLE %s.%s (id INT PRIMARY KEY)", + sql.EscapeName(testMysqlDatabase), sql.EscapeName(tableName))) + suite.Require().NoError(err) + } + + suite.Require().NoError(migrator.moveTablesFinalCleanup()) + + for _, tableName := range targetTableNames { + exists, err := migrator.applier.targetTableExists(tableName) + suite.Require().NoError(err) + suite.Require().False(exists, "noop cleanup must drop target table %s", tableName) + } + checkpointExists, err := migrator.applier.targetTableExists(migrationContext.GetCheckpointTableName()) + suite.Require().NoError(err) + suite.Require().False(checkpointExists, "fresh noop cleanup must drop its checkpoint table") +} + +func (suite *MigratorTestSuite) TestMoveTablesResumedNoopPreservesTargetTables() { + ctx := context.Background() + targetTableNames := []string{testMysqlTableName, "test_noop_resume_target"} + + migrationContext := newTestMigrationContext() + migrationContext.Noop = true + migrationContext.Resume = true + migrationContext.Checkpoint = true + migrationContext.MoveTables.TableNames = targetTableNames + migrationContext.MoveTables.TargetDatabase = testMysqlDatabase + migrationContext.InitMoveTableContainers() + + migrator := NewMigrator(migrationContext, "test") + migrator.applier = NewApplier(migrationContext) + migrator.applier.moveTablesTargetDB = suite.db + suite.Require().NoError(migrator.applier.CreateCheckpointTable()) + + for _, tableName := range targetTableNames { + _, err := suite.db.ExecContext(ctx, fmt.Sprintf("CREATE TABLE %s.%s (id INT PRIMARY KEY)", + sql.EscapeName(testMysqlDatabase), sql.EscapeName(tableName))) + suite.Require().NoError(err) + } + defer func() { + for _, tableName := range append(targetTableNames, migrationContext.GetCheckpointTableName()) { + _, err := suite.db.ExecContext(ctx, fmt.Sprintf("DROP TABLE IF EXISTS %s.%s", + sql.EscapeName(testMysqlDatabase), sql.EscapeName(tableName))) + suite.Require().NoError(err) + } + }() + + suite.Require().NoError(migrator.moveTablesFinalCleanup()) + + for _, tableName := range targetTableNames { + exists, err := migrator.applier.targetTableExists(tableName) + suite.Require().NoError(err) + suite.Require().True(exists, "resumed noop cleanup must preserve target table %s", tableName) + } + checkpointExists, err := migrator.applier.targetTableExists(migrationContext.GetCheckpointTableName()) + suite.Require().NoError(err) + suite.Require().True(checkpointExists, "resumed noop cleanup must preserve its checkpoint table") +} + func (suite *MigratorTestSuite) TestRetryBatchCopyWithHooks() { ctx := context.Background() @@ -932,7 +1112,7 @@ func (suite *MigratorTestSuite) TestCopierIntPK() { suite.Require().NoError(migrator.initiateApplier()) defer migrator.applier.Teardown() suite.Require().NoError(migrator.applier.prepareQueries()) - suite.Require().NoError(migrator.applier.ReadMigrationRangeValues()) + suite.Require().NoError(migrator.applier.ReadMigrationRangeValues(nil)) go migrator.iterateChunks() go func() { @@ -1004,7 +1184,7 @@ func (suite *MigratorTestSuite) TestCopierCompositePK() { suite.Require().NoError(migrator.initiateApplier()) defer migrator.applier.Teardown() suite.Require().NoError(migrator.applier.prepareQueries()) - suite.Require().NoError(migrator.applier.ReadMigrationRangeValues()) + suite.Require().NoError(migrator.applier.ReadMigrationRangeValues(nil)) go migrator.iterateChunks() go func() { diff --git a/go/logic/progress_snapshot.go b/go/logic/progress_snapshot.go index 375532cca..f5aa5af91 100644 --- a/go/logic/progress_snapshot.go +++ b/go/logic/progress_snapshot.go @@ -28,6 +28,7 @@ type migrationProgressSnapshot struct { streamerBinlogPosition string replicationLagSeconds float64 heartbeatLagSeconds float64 + writerLagSeconds float64 } func (mgtr *Migrator) migrationProgressSnapshot() migrationProgressSnapshot { @@ -63,6 +64,7 @@ func (mgtr *Migrator) migrationProgressSnapshot() migrationProgressSnapshot { streamerBinlogPosition: streamerBinlogPosition, replicationLagSeconds: mgtr.migrationContext.GetCurrentLagDuration().Seconds(), heartbeatLagSeconds: mgtr.migrationContext.TimeSinceLastHeartbeatOnChangelog().Seconds(), + writerLagSeconds: mgtr.migrationContext.GetBinlogWriterLag().Seconds(), } } diff --git a/go/logic/server.go b/go/logic/server.go index 4705ba9b9..ecdf75522 100644 --- a/go/logic/server.go +++ b/go/logic/server.go @@ -192,6 +192,33 @@ func (srv *Server) onServerCommand(command string, writer *bufio.Writer) (err er return srv.migrationContext.Log.Errore(err) } +// commandArgMatchesMigration reports whether a table-name argument supplied with +// an interactive command refers to this migration. The argument is optional and +// acts as a courtesy safety check, so an operator who is connected to the wrong +// gh-ost socket is rejected. In standard mode it must equal the single migrated +// table; in move-tables mode it may be any one of the migrated tables. +func (srv *Server) commandArgMatchesMigration(arg string) bool { + if srv.migrationContext.IsMoveTablesMode() { + for _, tableName := range srv.migrationContext.MoveTables.TableNames { + if arg == tableName { + return true + } + } + return false + } + return arg == srv.migrationContext.OriginalTableName +} + +// migrationTargetDescription returns a human-readable description of the migrated +// table(s), used in interactive-command messages. In move-tables mode it is the +// comma-joined list of migrated tables; otherwise the single table name. +func (srv *Server) migrationTargetDescription() string { + if srv.migrationContext.IsMoveTablesMode() { + return strings.Join(srv.migrationContext.MoveTables.TableNames, ",") + } + return srv.migrationContext.OriginalTableName +} + // applyServerCommand parses and executes commands by user func (srv *Server) applyServerCommand(command string, writer *bufio.Writer) (printStatusRule PrintStatusRule, err error) { tokens := strings.SplitN(command, "=", 2) @@ -387,9 +414,9 @@ help # This message } case "throttle", "pause", "suspend": { - if arg != "" && arg != srv.migrationContext.OriginalTableName { + if arg != "" && !srv.commandArgMatchesMigration(arg) { // User explicitly provided table name. This is a courtesy protection mechanism - err := fmt.Errorf("user commanded 'throttle' on %s, but migrated table is %s; ignoring request", arg, srv.migrationContext.OriginalTableName) + err := fmt.Errorf("user commanded 'throttle' on %s, but migrated table is %s; ignoring request", arg, srv.migrationTargetDescription()) return NoPrintStatusRule, err } atomic.StoreInt64(&srv.migrationContext.ThrottleCommandedByUser, 1) @@ -398,9 +425,9 @@ help # This message } case "no-throttle", "unthrottle", "resume", "continue": { - if arg != "" && arg != srv.migrationContext.OriginalTableName { + if arg != "" && !srv.commandArgMatchesMigration(arg) { // User explicitly provided table name. This is a courtesy protection mechanism - err := fmt.Errorf("user commanded 'no-throttle' on %s, but migrated table is %s; ignoring request", arg, srv.migrationContext.OriginalTableName) + err := fmt.Errorf("user commanded 'no-throttle' on %s, but migrated table is %s; ignoring request", arg, srv.migrationTargetDescription()) return NoPrintStatusRule, err } atomic.StoreInt64(&srv.migrationContext.ThrottleCommandedByUser, 0) @@ -425,9 +452,9 @@ help # This message err := fmt.Errorf("user commanded 'unpostpone' without specifying table name, but --force-named-cut-over is set") return NoPrintStatusRule, err } - if arg != "" && arg != srv.migrationContext.OriginalTableName { + if arg != "" && !srv.commandArgMatchesMigration(arg) { // User explicitly provided table name. This is a courtesy protection mechanism - err := fmt.Errorf("user commanded 'unpostpone' on %s, but migrated table is %s; ignoring request", arg, srv.migrationContext.OriginalTableName) + err := fmt.Errorf("user commanded 'unpostpone' on %s, but migrated table is %s; ignoring request", arg, srv.migrationTargetDescription()) return NoPrintStatusRule, err } if atomic.LoadInt64(&srv.migrationContext.IsPostponingCutOver) > 0 { @@ -444,9 +471,9 @@ help # This message err := fmt.Errorf("user commanded 'panic' without specifying table name, but --force-named-panic is set") return NoPrintStatusRule, err } - if arg != "" && arg != srv.migrationContext.OriginalTableName { + if arg != "" && !srv.commandArgMatchesMigration(arg) { // User explicitly provided table name. This is a courtesy protection mechanism - err := fmt.Errorf("user commanded 'panic' on %s, but migrated table is %s; ignoring request", arg, srv.migrationContext.OriginalTableName) + err := fmt.Errorf("user commanded 'panic' on %s, but migrated table is %s; ignoring request", arg, srv.migrationTargetDescription()) return NoPrintStatusRule, err } err := fmt.Errorf("user commanded 'panic'. The migration will be aborted without cleanup. Please drop the gh-ost tables before trying again") diff --git a/go/logic/test_utils_test.go b/go/logic/test_utils_test.go index 6012d4556..b334830ec 100644 --- a/go/logic/test_utils_test.go +++ b/go/logic/test_utils_test.go @@ -17,6 +17,7 @@ var ( testMysqlUser = "root" testMysqlPass = "root-password" testMysqlDatabase = "test" + testMysqlDatabaseOther = "test_other" testMysqlTableName = "testing" ) @@ -36,6 +37,10 @@ func getTestOldTableName() string { return fmt.Sprintf("`%s`.`_%s_del`", testMysqlDatabase, testMysqlTableName) } +func getTestOtherTableName() string { + return fmt.Sprintf("`%s`.`%s`", testMysqlDatabaseOther, testMysqlTableName) +} + func getTestConnectionConfig(ctx context.Context, container testcontainers.Container) (*mysql.ConnectionConfig, error) { host, err := container.Host(ctx) if err != nil { diff --git a/go/logic/throttler.go b/go/logic/throttler.go index ee6e3d132..a1995fba3 100644 --- a/go/logic/throttler.go +++ b/go/logic/throttler.go @@ -184,18 +184,32 @@ func (thlr *Throttler) collectReplicationLag(firstThrottlingCollected chan<- boo } } +// controlReplicaConnectionConfig returns the connection config used to read +// replication lag from a control replica. In move-tables mode the lag is read +// from the target cluster's replicas, otherwise from the source (inspector) +// cluster's replicas. +func (thlr *Throttler) controlReplicaConnectionConfig(replicaKey mysql.InstanceKey) *mysql.ConnectionConfig { + if thlr.migrationContext.IsMoveTablesMode() { + return thlr.migrationContext.MoveTables.ConnectionConfig.DuplicateCredentials(replicaKey) + } + return thlr.migrationContext.InspectorConnectionConfig.DuplicateCredentials(replicaKey) +} + // collectControlReplicasLag polls all the control replicas to get maximum lag value func (thlr *Throttler) collectControlReplicasLag() { if atomic.LoadInt64(&thlr.migrationContext.HibernateUntil) > 0 { return } - replicationLagQuery := fmt.Sprintf(` - select value from %s.%s where hint = 'heartbeat' and id <= 255 - `, - sql.EscapeName(thlr.migrationContext.DatabaseName), - sql.EscapeName(thlr.migrationContext.GetChangelogTableName()), - ) + var replicationLagQuery string + if !thlr.migrationContext.IsMoveTablesMode() { + replicationLagQuery = fmt.Sprintf(` + select value from %s.%s where hint = 'heartbeat' and id <= 255 + `, + sql.EscapeName(thlr.migrationContext.DatabaseName), + sql.EscapeName(thlr.migrationContext.GetChangelogTableName()), + ) + } readReplicaLag := func(connectionConfig *mysql.ConnectionConfig) (lag time.Duration, err error) { dbUri := connectionConfig.GetDBUri("information_schema") @@ -206,6 +220,14 @@ func (thlr *Throttler) collectControlReplicasLag() { return lag, err } + if thlr.migrationContext.IsMoveTablesMode() { + dbVersion, err := mysql.GetDBVersion(thlr.migrationContext.Uuid, dbUri) + if err != nil { + return lag, err + } + return mysql.GetReplicationLagFromSlaveStatus(dbVersion, db) + } + if err := db.QueryRow(replicationLagQuery).Scan(&heartbeatValue); err != nil { return lag, err } @@ -221,7 +243,8 @@ func (thlr *Throttler) collectControlReplicasLag() { } lagResults := make(chan *mysql.ReplicationLagResult, instanceKeyMap.Len()) for replicaKey := range *instanceKeyMap { - connectionConfig := thlr.migrationContext.InspectorConnectionConfig.DuplicateCredentials(replicaKey) + connectionConfig := thlr.controlReplicaConnectionConfig(replicaKey) + if err := connectionConfig.RegisterTLSConfig(); err != nil { return &mysql.ReplicationLagResult{Err: err} } @@ -451,7 +474,9 @@ func (thlr *Throttler) collectGeneralThrottleMetrics() error { // that may affect throttling. There are several components, all running independently, // that collect such metrics. func (thlr *Throttler) initiateThrottlerCollection(firstThrottlingCollected chan<- bool) { - go thlr.collectReplicationLag(firstThrottlingCollected) + if !thlr.migrationContext.IsMoveTablesMode() { + go thlr.collectReplicationLag(firstThrottlingCollected) + } go thlr.collectControlReplicasLag() go thlr.collectThrottleHTTPStatus(firstThrottlingCollected) diff --git a/go/logic/throttler_test.go b/go/logic/throttler_test.go index 01104805b..7a9b77e2e 100644 --- a/go/logic/throttler_test.go +++ b/go/logic/throttler_test.go @@ -14,6 +14,7 @@ import ( "github.com/stretchr/testify/require" "github.com/github/gh-ost/go/base" + "github.com/github/gh-ost/go/mysql" ) func newTestThrottler() *Throttler { @@ -87,6 +88,40 @@ func TestThrottleReturnsOnContextCancellation(t *testing.T) { } } +func TestControlReplicaConnectionConfig(t *testing.T) { + replicaKey := mysql.InstanceKey{Hostname: "replica-host", Port: 3307} + + t.Run("uses inspector connection config when not in move-tables mode", func(t *testing.T) { + thlr := newTestThrottler() + thlr.migrationContext.InspectorConnectionConfig.Key = mysql.InstanceKey{Hostname: "source-host", Port: 3306} + thlr.migrationContext.InspectorConnectionConfig.User = "source-user" + thlr.migrationContext.InspectorConnectionConfig.Password = "source-pass" + + connectionConfig := thlr.controlReplicaConnectionConfig(replicaKey) + + assert.False(t, thlr.migrationContext.IsMoveTablesMode()) + assert.Equal(t, replicaKey, connectionConfig.Key) + assert.Equal(t, "source-user", connectionConfig.User) + assert.Equal(t, "source-pass", connectionConfig.Password) + }) + + t.Run("uses move-tables connection config when in move-tables mode", func(t *testing.T) { + thlr := newTestThrottler() + thlr.migrationContext.MoveTables.TableNames = []string{"my_table"} + thlr.migrationContext.MoveTables.ConnectionConfig = mysql.NewConnectionConfig() + thlr.migrationContext.MoveTables.ConnectionConfig.Key = mysql.InstanceKey{Hostname: "target-host", Port: 3306} + thlr.migrationContext.MoveTables.ConnectionConfig.User = "target-user" + thlr.migrationContext.MoveTables.ConnectionConfig.Password = "target-pass" + + connectionConfig := thlr.controlReplicaConnectionConfig(replicaKey) + + assert.True(t, thlr.migrationContext.IsMoveTablesMode()) + assert.Equal(t, replicaKey, connectionConfig.Key) + assert.Equal(t, "target-user", connectionConfig.User) + assert.Equal(t, "target-pass", connectionConfig.Password) + }) +} + func TestThrottleCallsOnThrottledCallback(t *testing.T) { thlr := newTestThrottler() thlr.migrationContext.SetThrottled(true, "test", base.NoThrottleReasonHint) diff --git a/go/mysql/utils.go b/go/mysql/utils.go index ad3c09487..0169c4861 100644 --- a/go/mysql/utils.go +++ b/go/mysql/utils.go @@ -50,6 +50,8 @@ func (rlg *ReplicationLagResult) HasLag() bool { // knownDBs is a DB cache by uri var knownDBs map[string]*gosql.DB = make(map[string]*gosql.DB) var knownDBsMutex = &sync.Mutex{} +var knownDBsVersions map[string]string = make(map[string]string) +var knownDBsVersionsMutex = &sync.Mutex{} // initConnector wraps a driver.Connector to run a fixed set of statements on // every newly established connection (e.g. setting the transaction isolation @@ -123,6 +125,30 @@ func GetDB(migrationUuid string, mysql_uri string) (db *gosql.DB, exists bool, e return db, exists, nil } +// GetDBVersion returns the MySQL version for a given mysql_uri, and caches it for future calls. +// Uses GetDB to get a connection to the database. +func GetDBVersion(migrationUuid string, mysql_uri string) (dbVersion string, err error) { + cacheKey := migrationUuid + ":" + mysql_uri + + knownDBsVersionsMutex.Lock() + defer knownDBsVersionsMutex.Unlock() + + if dbVersion, exists := knownDBsVersions[cacheKey]; exists { + return dbVersion, nil + } + + db, _, err := GetDB(migrationUuid, mysql_uri) + if err != nil { + return "", err + } + var version string + if err := db.QueryRow(`select @@global.version`).Scan(&version); err != nil { + return "", err + } + knownDBsVersions[cacheKey] = version + return version, nil +} + // GetReplicationLagFromSlaveStatus returns replication lag for a given db; via SHOW SLAVE STATUS func GetReplicationLagFromSlaveStatus(dbVersion string, informationSchemaDb *gosql.DB) (replicationLag time.Duration, err error) { showReplicaStatusQuery := fmt.Sprintf("show %s", ReplicaTermFor(dbVersion, `slave status`)) diff --git a/go/sql/builder.go b/go/sql/builder.go index 7d0864601..3f4c375ef 100644 --- a/go/sql/builder.go +++ b/go/sql/builder.go @@ -7,6 +7,7 @@ package sql import ( "fmt" + "slices" "strconv" "strings" ) @@ -119,11 +120,12 @@ func BuildEqualsPreparedComparison(columns []string) (result string, err error) // It holds the prepared query statement so it doesn't need to be recreated every time. type CheckpointInsertQueryBuilder struct { - uniqueKeyColumns *ColumnList - preparedStatement string + uniqueKeyColumns *ColumnList + preparedStatement string + includeMoveTablesCutOverColumns bool } -func NewCheckpointQueryBuilder(databaseName, tableName string, uniqueKeyColumns *ColumnList) (*CheckpointInsertQueryBuilder, error) { +func NewCheckpointQueryBuilder(databaseName, tableName string, uniqueKeyColumns *ColumnList, includeMoveTablesCutOverColumns bool) (*CheckpointInsertQueryBuilder, error) { if uniqueKeyColumns.Len() == 0 { return nil, fmt.Errorf("got 0 columns in BuildSetCheckpointInsertQuery") } @@ -138,26 +140,48 @@ func NewCheckpointQueryBuilder(databaseName, tableName string, uniqueKeyColumns } databaseName = EscapeName(databaseName) tableName = EscapeName(tableName) - stmt := fmt.Sprintf(` - insert /* gh-ost */ - into %s.%s - (gh_ost_chk_timestamp, gh_ost_chk_coords, gh_ost_chk_iteration, - gh_ost_rows_copied, gh_ost_dml_applied, gh_ost_is_cutover, - %s, %s) - values - (unix_timestamp(now()), ?, ?, - ?, ?, ?, - %s, %s)`, - databaseName, tableName, - strings.Join(minUniqueColNames, ", "), - strings.Join(maxUniqueColNames, ", "), - strings.Join(values, ", "), - strings.Join(values, ", "), - ) - b := &CheckpointInsertQueryBuilder{ - uniqueKeyColumns: uniqueKeyColumns, - preparedStatement: stmt, + uniqueKeyColumns: uniqueKeyColumns, + preparedStatement: func() string { + if includeMoveTablesCutOverColumns { + return fmt.Sprintf(` + insert /* gh-ost */ + into %s.%s + (gh_ost_chk_timestamp, gh_ost_chk_coords, gh_ost_chk_iteration, + gh_ost_rows_copied, gh_ost_dml_applied, gh_ost_is_cutover, + gh_ost_move_tables_cutover_started, gh_ost_move_tables_drain_gtid, + %s, %s) + values + (unix_timestamp(now()), ?, ?, + ?, ?, ?, + ?, ?, + %s, %s)`, + databaseName, tableName, + strings.Join(minUniqueColNames, ", "), + strings.Join(maxUniqueColNames, ", "), + strings.Join(values, ", "), + strings.Join(values, ", "), + ) + } + + return fmt.Sprintf(` + insert /* gh-ost */ + into %s.%s + (gh_ost_chk_timestamp, gh_ost_chk_coords, gh_ost_chk_iteration, + gh_ost_rows_copied, gh_ost_dml_applied, gh_ost_is_cutover, + %s, %s) + values + (unix_timestamp(now()), ?, ?, + ?, ?, ?, + %s, %s)`, + databaseName, tableName, + strings.Join(minUniqueColNames, ", "), + strings.Join(maxUniqueColNames, ", "), + strings.Join(values, ", "), + strings.Join(values, ", "), + ) + }(), + includeMoveTablesCutOverColumns: includeMoveTablesCutOverColumns, } return b, nil } @@ -425,6 +449,146 @@ func BuildRangeInsertPreparedQuery(databaseName, originalTableName, ghostTableNa return BuildRangeInsertQuery(databaseName, originalTableName, ghostTableName, sharedColumns, mappedSharedColumns, uniqueKey, uniqueKeyColumns, rangeStartValues, rangeEndValues, rangeStartArgs, rangeEndArgs, includeRangeStartValues, transactionalTable, noWait) } +type MoveTableCopySelectQueryBuilder struct { + preparedStatement string + argsMapping []int + argsCount int +} + +func NewMoveTableCopySelectQueryBuilder(sourceDatabaseName, sourceTableName string, columns *ColumnList, uniqueKey string, uniqueKeyColumns *ColumnList, includeRangeStartValues bool) (*MoveTableCopySelectQueryBuilder, error) { + sourceDatabaseName = EscapeName(sourceDatabaseName) + sourceTableName = EscapeName(sourceTableName) + columnNames := columns.Names() + for i := range columnNames { + columnNames[i] = EscapeName(columnNames[i]) + } + sharedColumnsListing := strings.Join(columnNames, ", ") + uniqueKey = EscapeName(uniqueKey) + var minRangeComparisonSign = GreaterThanComparisonSign + if includeRangeStartValues { + minRangeComparisonSign = GreaterThanOrEqualsComparisonSign + } + rangeStartValues := buildColumnsPreparedValues(uniqueKeyColumns) + rangeEndValues := buildColumnsPreparedValues(uniqueKeyColumns) + dummyArgs := make([]any, len(uniqueKeyColumns.Columns())) + for i := range dummyArgs { + dummyArgs[i] = i + } + var argsMapping []int + + rangeStartComparison, rangeExplodedArgs, err := BuildRangeComparison(uniqueKeyColumns.Names(), rangeStartValues, dummyArgs, minRangeComparisonSign) + if err != nil { + return nil, err + } + for _, a := range rangeExplodedArgs { + idx := slices.Index(dummyArgs, a) + if idx == -1 { + return nil, fmt.Errorf("failed to build args mapping, missing argument pointer %v", a) + } + argsMapping = append(argsMapping, idx) + } + + rangeEndComparison, rangeExplodedArgs, err := BuildRangeComparison(uniqueKeyColumns.Names(), rangeEndValues, dummyArgs, LessThanOrEqualsComparisonSign) + if err != nil { + return nil, err + } + for _, a := range rangeExplodedArgs { + idx := slices.Index(dummyArgs, a) + if idx == -1 { + return nil, fmt.Errorf("failed to build args mapping, missing argument pointer %v", a) + } + argsMapping = append(argsMapping, idx+len(dummyArgs)) + } + + stmt := fmt.Sprintf(` + select /* gh-ost %s.%s */ %s + from + %s.%s + force index (%s) + where + (%s and %s) + `, + sourceDatabaseName, sourceTableName, sharedColumnsListing, + sourceDatabaseName, sourceTableName, + uniqueKey, + rangeStartComparison, rangeEndComparison, + ) + return &MoveTableCopySelectQueryBuilder{ + preparedStatement: stmt, + argsMapping: argsMapping, + argsCount: len(dummyArgs) * 2, + }, nil +} + +func (b *MoveTableCopySelectQueryBuilder) BuildQuery(rangeStartArgs, rangeEndArgs []any) (string, []any, error) { + if len(rangeStartArgs)+len(rangeEndArgs) != b.argsCount { + return "", nil, fmt.Errorf("got %d args but expected %d", len(rangeStartArgs)+len(rangeEndArgs), b.argsCount) + } + if len(rangeStartArgs) != len(rangeEndArgs) { + return "", nil, fmt.Errorf("mismatched number of start and end args: %d != %d", len(rangeStartArgs), len(rangeEndArgs)) + } + explodedArgs := make([]any, 0, len(b.argsMapping)) + for _, idx := range b.argsMapping { + if idx < len(rangeStartArgs) { + explodedArgs = append(explodedArgs, rangeStartArgs[idx]) + } else { + explodedArgs = append(explodedArgs, rangeEndArgs[idx-len(rangeStartArgs)]) + } + } + return b.preparedStatement, explodedArgs, nil +} + +type MoveTableCopyInsertQueryBuilder struct { + preparedStatement string + valueListPlaceholder string + valueListSize int +} + +func NewMoveTableCopyInsertQueryBuilder(targetDatabaseName, targetTableName string, columns *ColumnList) (*MoveTableCopyInsertQueryBuilder, error) { + targetDatabaseName = EscapeName(targetDatabaseName) + targetTableName = EscapeName(targetTableName) + columnsNames := columns.Names() + for i := range columnsNames { + columnsNames[i] = EscapeName(columnsNames[i]) + } + sharedColumnsListing := strings.Join(columnsNames, ", ") + valueListPlaceholder := "(" + strings.Join(buildColumnsPreparedValues(columns), ", ") + ")" + valueListSize := len(columnsNames) + stmt := fmt.Sprintf(` + insert /* gh-ost %s.%s */ ignore + into + %s.%s + (%s) + values + `, + targetDatabaseName, targetTableName, + targetDatabaseName, targetTableName, + sharedColumnsListing, + ) + return &MoveTableCopyInsertQueryBuilder{ + preparedStatement: stmt, + valueListPlaceholder: valueListPlaceholder, + valueListSize: valueListSize, + }, nil +} + +func (b *MoveTableCopyInsertQueryBuilder) BuildQuery(values []*ColumnValues) (string, []any, error) { + var explodedArgs []any + var builder strings.Builder + builder.WriteString(b.preparedStatement) + for i, value := range values { + if len(value.AbstractValues()) != b.valueListSize { + return "", nil, fmt.Errorf("got %d column values but expected %d", len(value.AbstractValues()), b.valueListSize) + } + if i > 0 { + builder.WriteString(",\n") + } + builder.WriteString(b.valueListPlaceholder) + explodedArgs = append(explodedArgs, value.AbstractValues()...) + } + return builder.String(), explodedArgs, nil +} + func BuildUniqueKeyRangeEndPreparedQueryViaOffset(databaseName, tableName string, uniqueKeyColumns *ColumnList, rangeStartArgs, rangeEndArgs []interface{}, chunkSize int64, includeRangeStartValues bool, hint string) (result string, explodedArgs []interface{}, err error) { if uniqueKeyColumns.Len() == 0 { return "", explodedArgs, fmt.Errorf("got 0 columns in BuildUniqueKeyRangeEndPreparedQuery") diff --git a/go/sql/builder_test.go b/go/sql/builder_test.go index be7075927..38b5043ab 100644 --- a/go/sql/builder_test.go +++ b/go/sql/builder_test.go @@ -1102,12 +1102,277 @@ func TestBuildDMLUpdateQuerySignedUnsigned(t *testing.T) { } } +func TestMoveTableCopySelectQueryBuilder(t *testing.T) { + t.Run("single column unique key", func(t *testing.T) { + sharedColumns := NewColumnList([]string{"id", "name", "position"}) + uniqueKeyColumns := NewColumnList([]string{"id"}) + + builder, err := NewMoveTableCopySelectQueryBuilder("mydb", "tbl", sharedColumns, "PRIMARY", uniqueKeyColumns, true) + require.NoError(t, err) + + query, args, err := builder.BuildQuery([]any{3}, []any{103}) + require.NoError(t, err) + + expected := ` + select /* gh-ost mydb.tbl */ id, name, position + from + mydb.tbl + force index (PRIMARY) + where + (((id > ?) or ((id = ?))) and ((id < ?) or ((id = ?)))) + ` + require.Equal(t, normalizeQuery(expected), normalizeQuery(query)) + require.Equal(t, []any{3, 3, 103, 103}, args) + }) + + t.Run("single column unique key without range start", func(t *testing.T) { + sharedColumns := NewColumnList([]string{"id", "name", "position"}) + uniqueKeyColumns := NewColumnList([]string{"id"}) + + builder, err := NewMoveTableCopySelectQueryBuilder("mydb", "tbl", sharedColumns, "PRIMARY", uniqueKeyColumns, false) + require.NoError(t, err) + + query, args, err := builder.BuildQuery([]any{3}, []any{103}) + require.NoError(t, err) + + expected := ` + select /* gh-ost mydb.tbl */ id, name, position + from + mydb.tbl + force index (PRIMARY) + where + (((id > ?)) and ((id < ?) or ((id = ?)))) + ` + require.Equal(t, normalizeQuery(expected), normalizeQuery(query)) + require.Equal(t, []any{3, 103, 103}, args) + }) + + t.Run("compound unique key", func(t *testing.T) { + sharedColumns := NewColumnList([]string{"id", "name", "position"}) + uniqueKeyColumns := NewColumnList([]string{"name", "position"}) + + builder, err := NewMoveTableCopySelectQueryBuilder("mydb", "tbl", sharedColumns, "name_position_uidx", uniqueKeyColumns, true) + require.NoError(t, err) + + query, args, err := builder.BuildQuery([]any{3, 17}, []any{103, 117}) + require.NoError(t, err) + + expected := ` + select /* gh-ost mydb.tbl */ id, name, position + from + mydb.tbl + force index (name_position_uidx) + where + (((name > ?) or (((name = ?)) AND (position > ?)) or ((name = ?) and (position = ?))) + and ((name < ?) or (((name = ?)) AND (position < ?)) or ((name = ?) and (position = ?)))) + ` + require.Equal(t, normalizeQuery(expected), normalizeQuery(query)) + require.Equal(t, []any{3, 3, 17, 3, 17, 103, 103, 117, 103, 117}, args) + }) + + t.Run("reuses prepared statement across calls", func(t *testing.T) { + sharedColumns := NewColumnList([]string{"id", "name"}) + uniqueKeyColumns := NewColumnList([]string{"id"}) + + builder, err := NewMoveTableCopySelectQueryBuilder("mydb", "tbl", sharedColumns, "PRIMARY", uniqueKeyColumns, true) + require.NoError(t, err) + + query1, args1, err := builder.BuildQuery([]any{1}, []any{10}) + require.NoError(t, err) + query2, args2, err := builder.BuildQuery([]any{11}, []any{20}) + require.NoError(t, err) + + require.Equal(t, query1, query2) + require.Equal(t, []any{1, 1, 10, 10}, args1) + require.Equal(t, []any{11, 11, 20, 20}, args2) + }) + + t.Run("wrong args count", func(t *testing.T) { + sharedColumns := NewColumnList([]string{"id", "name"}) + uniqueKeyColumns := NewColumnList([]string{"id"}) + + builder, err := NewMoveTableCopySelectQueryBuilder("mydb", "tbl", sharedColumns, "PRIMARY", uniqueKeyColumns, true) + require.NoError(t, err) + + _, _, err = builder.BuildQuery([]any{1, 2}, []any{10}) + require.Error(t, err) + }) + + t.Run("mismatched start and end args count", func(t *testing.T) { + sharedColumns := NewColumnList([]string{"id", "name", "position"}) + uniqueKeyColumns := NewColumnList([]string{"name", "position"}) + + builder, err := NewMoveTableCopySelectQueryBuilder("mydb", "tbl", sharedColumns, "name_position_uidx", uniqueKeyColumns, true) + require.NoError(t, err) + + // Total args count matches argsCount (4), but start and end counts differ. + _, _, err = builder.BuildQuery([]any{1, 2, 3}, []any{10}) + require.Error(t, err) + require.Contains(t, err.Error(), "mismatched number of start and end args") + }) +} + +func BenchmarkMoveTableCopySelectQueryBuilderBuildQuery(b *testing.B) { + sharedColumns := NewColumnList([]string{"id", "name", "position"}) + uniqueKeyColumns := NewColumnList([]string{"name", "position"}) + + builder, err := NewMoveTableCopySelectQueryBuilder("mydb", "tbl", sharedColumns, "name_position_uidx", uniqueKeyColumns, true) + if err != nil { + b.Fatal(err) + } + + rangeStartArgs := []any{3, 17} + rangeEndArgs := []any{103, 117} + + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, _, err := builder.BuildQuery(rangeStartArgs, rangeEndArgs) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMoveTableCopyInsertQueryBuilder(t *testing.T) { + t.Run("single row", func(t *testing.T) { + sharedColumns := NewColumnList([]string{"id", "name", "position"}) + + builder, err := NewMoveTableCopyInsertQueryBuilder("mydb", "ghost", sharedColumns) + require.NoError(t, err) + + values := []*ColumnValues{ + ToColumnValues([]interface{}{1, "alice", 10}), + } + query, args, err := builder.BuildQuery(values) + require.NoError(t, err) + + expected := ` + insert /* gh-ost mydb.ghost */ ignore + into + mydb.ghost + (id, name, position) + values + (?, ?, ?) + ` + require.Equal(t, normalizeQuery(expected), normalizeQuery(query)) + require.Equal(t, []any{1, "alice", 10}, args) + }) + + t.Run("multiple rows", func(t *testing.T) { + sharedColumns := NewColumnList([]string{"id", "name", "position"}) + + builder, err := NewMoveTableCopyInsertQueryBuilder("mydb", "ghost", sharedColumns) + require.NoError(t, err) + + values := []*ColumnValues{ + ToColumnValues([]interface{}{1, "alice", 10}), + ToColumnValues([]interface{}{2, "bob", 20}), + ToColumnValues([]interface{}{3, "carol", 30}), + } + query, args, err := builder.BuildQuery(values) + require.NoError(t, err) + + expected := ` + insert /* gh-ost mydb.ghost */ ignore + into + mydb.ghost + (id, name, position) + values + (?, ?, ?), + (?, ?, ?), + (?, ?, ?) + ` + require.Equal(t, normalizeQuery(expected), normalizeQuery(query)) + require.Equal(t, []any{1, "alice", 10, 2, "bob", 20, 3, "carol", 30}, args) + }) + + t.Run("wrong column count", func(t *testing.T) { + sharedColumns := NewColumnList([]string{"id", "name", "position"}) + + builder, err := NewMoveTableCopyInsertQueryBuilder("mydb", "ghost", sharedColumns) + require.NoError(t, err) + + values := []*ColumnValues{ + ToColumnValues([]interface{}{1, "alice"}), + } + _, _, err = builder.BuildQuery(values) + require.Error(t, err) + }) + + t.Run("reuses prepared statement", func(t *testing.T) { + sharedColumns := NewColumnList([]string{"id", "name"}) + + builder, err := NewMoveTableCopyInsertQueryBuilder("mydb", "ghost", sharedColumns) + require.NoError(t, err) + + values1 := []*ColumnValues{ToColumnValues([]interface{}{1, "a"})} + values2 := []*ColumnValues{ToColumnValues([]interface{}{2, "b"})} + + query1, args1, err := builder.BuildQuery(values1) + require.NoError(t, err) + query2, args2, err := builder.BuildQuery(values2) + require.NoError(t, err) + + require.Equal(t, query1, query2) + require.Equal(t, []any{1, "a"}, args1) + require.Equal(t, []any{2, "b"}, args2) + }) +} + +func BenchmarkMoveTableCopyInsertQueryBuilderBuildQuery(b *testing.B) { + sharedColumns := NewColumnList([]string{"id", "name", "position"}) + + builder, err := NewMoveTableCopyInsertQueryBuilder("mydb", "ghost", sharedColumns) + if err != nil { + b.Fatal(err) + } + + values := []*ColumnValues{ + ToColumnValues([]interface{}{1, "alice", 10}), + ToColumnValues([]interface{}{2, "bob", 20}), + ToColumnValues([]interface{}{3, "carol", 30}), + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, _, err := builder.BuildQuery(values) + if err != nil { + b.Fatal(err) + } + } +} + func TestCheckpointQueryBuilder(t *testing.T) { databaseName := "mydb" tableName := "_tbl_ghk" valueArgs := []interface{}{"mona", "mascot", int8(-17), "anothername", "anotherposition", int8(-2)} uniqueKeyColumns := NewColumnList([]string{"name", "position", "my_very_long_column_that_is_64_utf8_characters_long_很长很长很长很长很长很长"}) - builder, err := NewCheckpointQueryBuilder(databaseName, tableName, uniqueKeyColumns) + builder, err := NewCheckpointQueryBuilder(databaseName, tableName, uniqueKeyColumns, false) + require.NoError(t, err) + query, uniqueKeyArgs, err := builder.BuildQuery(valueArgs) + require.NoError(t, err) + expected := ` + insert /* gh-ost */ into mydb._tbl_ghk + (gh_ost_chk_timestamp, gh_ost_chk_coords, gh_ost_chk_iteration, + gh_ost_rows_copied, gh_ost_dml_applied, gh_ost_is_cutover, + name_min, position_min, my_very_long_column_that_is_64_utf8_characters_long_很长很长很长很长_min, + name_max, position_max, my_very_long_column_that_is_64_utf8_characters_long_很长很长很长很长_max) + values + (unix_timestamp(now()), ?, ?, + ?, ?, ?, + ?, ?, ?, + ?, ?, ?) + ` + require.Equal(t, normalizeQuery(expected), normalizeQuery(query)) + require.Equal(t, []interface{}{"mona", "mascot", int8(-17), "anothername", "anotherposition", int8(-2)}, uniqueKeyArgs) +} + +func TestMoveTablesCheckpointQueryBuilder(t *testing.T) { + databaseName := "mydb" + tableName := "_tbl_ghk" + valueArgs := []interface{}{"mona", "mascot", int8(-17), "anothername", "anotherposition", int8(-2)} + uniqueKeyColumns := NewColumnList([]string{"name", "position", "my_very_long_column_that_is_64_utf8_characters_long_很长很长很长很长很长很长"}) + builder, err := NewCheckpointQueryBuilder(databaseName, tableName, uniqueKeyColumns, true) require.NoError(t, err) query, uniqueKeyArgs, err := builder.BuildQuery(valueArgs) require.NoError(t, err) @@ -1115,11 +1380,13 @@ func TestCheckpointQueryBuilder(t *testing.T) { insert /* gh-ost */ into mydb._tbl_ghk (gh_ost_chk_timestamp, gh_ost_chk_coords, gh_ost_chk_iteration, gh_ost_rows_copied, gh_ost_dml_applied, gh_ost_is_cutover, + gh_ost_move_tables_cutover_started, gh_ost_move_tables_drain_gtid, name_min, position_min, my_very_long_column_that_is_64_utf8_characters_long_很长很长很长很长_min, name_max, position_max, my_very_long_column_that_is_64_utf8_characters_long_很长很长很长很长_max) values (unix_timestamp(now()), ?, ?, ?, ?, ?, + ?, ?, ?, ?, ?, ?, ?, ?) ` diff --git a/localtests/docker-compose-move-tables.yml b/localtests/docker-compose-move-tables.yml new file mode 100644 index 000000000..8ba8647ea --- /dev/null +++ b/localtests/docker-compose-move-tables.yml @@ -0,0 +1,57 @@ +services: + mysql-source-primary: + image: $TEST_MYSQL_IMAGE + container_name: mysql-source-primary + command: --server-id=1 --log-bin=mysql-bin --binlog-format=row --gtid-mode=ON --enforce-gtid-consistency=ON --character-set-server=utf8mb4 $MYSQL_NATIVE_PASSWORD_FLAG + environment: + MYSQL_ROOT_PASSWORD: opensesame + MYSQL_ROOT_HOST: '%' + MYSQL_DATABASE: test + MYSQL_TCP_PORT: 3307 + INIT_ROCKSDB: 1 # for percona-server + ports: + - '3307:3307' + expose: + - '3307' + mysql-source-replica: + image: $TEST_MYSQL_IMAGE + container_name: mysql-source-replica + command: --server-id=2 --log-bin=mysql-bin --binlog-format=row --gtid-mode=ON --enforce-gtid-consistency=ON --log-slave-updates=ON --character-set-server=utf8mb4 $MYSQL_NATIVE_PASSWORD_FLAG + environment: + MYSQL_ROOT_PASSWORD: opensesame + MYSQL_ROOT_HOST: '%' + MYSQL_DATABASE: test + MYSQL_TCP_PORT: 3308 + INIT_ROCKSDB: 1 # for percona-server + ports: + - '3308:3308' + expose: + - '3308' + mysql-target-primary: + image: $TEST_MYSQL_IMAGE + container_name: mysql-target-primary + command: --server-id=3 --log-bin=mysql-bin --binlog-format=row --gtid-mode=ON --enforce-gtid-consistency=ON --character-set-server=utf8mb4 $MYSQL_NATIVE_PASSWORD_FLAG + environment: + MYSQL_ROOT_PASSWORD: opensesame + MYSQL_ROOT_HOST: '%' + MYSQL_DATABASE: test + MYSQL_TCP_PORT: 3309 + INIT_ROCKSDB: 1 # for percona-server + ports: + - '3309:3309' + expose: + - '3309' + mysql-target-replica: + image: $TEST_MYSQL_IMAGE + container_name: mysql-target-replica + command: --server-id=4 --log-bin=mysql-bin --binlog-format=row --gtid-mode=ON --enforce-gtid-consistency=ON --log-slave-updates=ON --character-set-server=utf8mb4 $MYSQL_NATIVE_PASSWORD_FLAG + environment: + MYSQL_ROOT_PASSWORD: opensesame + MYSQL_ROOT_HOST: '%' + MYSQL_DATABASE: test + MYSQL_TCP_PORT: 3310 + INIT_ROCKSDB: 1 # for percona-server + ports: + - '3310:3310' + expose: + - '3310' \ No newline at end of file diff --git a/localtests/move-tables-test.sh b/localtests/move-tables-test.sh new file mode 100755 index 000000000..a3b2ada46 --- /dev/null +++ b/localtests/move-tables-test.sh @@ -0,0 +1,545 @@ +#!/bin/bash + +# Local integration tests. To be used by CI. +# See https://github.com/github/gh-ost/tree/doc/local-tests.md +# + +# Usage: localtests/test/sh [filter] +# By default, runs all move-tables tests. Given filter, will only run tests matching given regep + +repo_root=$(git rev-parse --show-toplevel) +script_path="$repo_root/script/move-tables" +tests_path=$(dirname $0)/move-tables +test_logfile=/tmp/gh-ost-test.log +default_ghost_binary=/tmp/gh-ost-test +ghost_binary="" +database=test +docker=false +gtid=false +storage_engine=innodb +exec_command_file=/tmp/gh-ost-test.bash +orig_structure_output_file=/tmp/gh-ost-test.orig.structure.sql +ghost_structure_output_file=/tmp/gh-ost-test.ghost.structure.sql +orig_content_output_file=/tmp/gh-ost-test.orig.content.csv +ghost_content_output_file=/tmp/gh-ost-test.ghost.content.csv +postpone_cutover_flag_file=/tmp/gh-ost-test.ghost.postpone.flag + +source_master_host= +source_master_port= +source_replica_host= +source_replica_port= +target_master_host= +target_master_port= +target_replica_host= +target_replica_port= +original_sql_mode= +current_gtid_mode= +test_timeout=120 +test_failure_log_tail_lines=50 +tables_to_migrate=() + +OPTIND=1 +while getopts "b:s:dg" OPTION; do + case $OPTION in + b) + ghost_binary="$OPTARG" + ;; + s) + storage_engine="$OPTARG" + ;; + d) + docker=true + ;; + g) + gtid=true + ;; + esac +done +shift $((OPTIND - 1)) + +test_pattern="${1:-.}" + +mysql-exec() { + cluster=$1 + role=$2 + shift 2 + + if [[ $TEST_MYSQL_IMAGE =~ "mysql:8.4" ]]; then + $script_path/mysql-$cluster-$role --ssl-mode=required "$@" + else + $script_path/mysql-$cluster-$role "$@" + fi +} + +verify_master_and_replica() { + cluster=$1 + echo "Verifying $cluster cluster..." + + if [ "$(mysql-exec $cluster primary -e "select 1" -ss)" != "1" ]; then + echo "Cannot verify $cluster primary" + exit 1 + fi + read master_host master_port <<<$(mysql-exec $cluster primary -e "select @@hostname, @@port" -ss) + [ "$master_host" == "$(hostname)" ] && master_host="127.0.0.1" + echo "# master verified at $master_host:$master_port" + if ! mysql-exec $cluster primary -e "set global event_scheduler := 1"; then + echo "Cannot enable event_scheduler on master" + exit 1 + fi + original_sql_mode="$(mysql-exec $cluster primary -e "select @@global.sql_mode" -s -s)" + echo "sql_mode on master is ${original_sql_mode}" + + current_gtid_mode=$(mysql-exec $cluster primary -s -s -e "select @@global.gtid_mode" 2>/dev/null || echo unsupported) + current_enforce_gtid_consistency=$(mysql-exec $cluster primary -s -s -e "select @@global.enforce_gtid_consistency" 2>/dev/null || echo unsupported) + current_master_server_uuid=$(mysql-exec $cluster primary -s -s -e "select @@global.server_uuid" 2>/dev/null || echo unsupported) + current_replica_server_uuid=$(mysql-exec $cluster replica -s -s -e "select @@global.server_uuid" 2>/dev/null || echo unsupported) + echo "gtid_mode on master is ${current_gtid_mode} with enforce_gtid_consistency=${current_enforce_gtid_consistency}" + echo "server_uuid on master is ${current_master_server_uuid}, replica is ${current_replica_server_uuid}" + + echo "Gracefully sleeping for 3 seconds while replica is setting up..." + sleep 3 + + if [ "$(mysql-exec $cluster replica -e "select 1" -ss)" != "1" ]; then + echo "Cannot verify gh-ost-test-mysql-replica" + exit 1 + fi + if [ "$(mysql-exec $cluster replica -e "select @@global.binlog_format" -ss)" != "ROW" ]; then + echo "Expecting test replica to have binlog_format=ROW" + exit 1 + fi + read replica_host replica_port <<<$(mysql-exec $cluster replica -e "select @@hostname, @@port" -ss) + [ "$replica_host" == "$(hostname)" ] && replica_host="127.0.0.1" + echo "# replica verified at $replica_host:$replica_port" + + if [ "$docker" = true ]; then + master_host="0.0.0.0" + if [ "$cluster" == "source" ]; then + master_port="3307" + elif [ "$cluster" == "target" ]; then + master_port="3309" + fi + echo "# using docker master at $master_host:$master_port" + replica_host="0.0.0.0" + if [ "$cluster" == "source" ]; then + replica_port="3308" + elif [ "$cluster" == "target" ]; then + replica_port="3310" + fi + echo "# using docker replica at $replica_host:$replica_port" + fi + + if [ "$cluster" == "source" ]; then + source_master_host=$master_host + source_master_port=$master_port + source_replica_host=$replica_host + source_replica_port=$replica_port + elif [ "$cluster" == "target" ]; then + target_master_host=$master_host + target_master_port=$master_port + target_replica_host=$replica_host + target_replica_port=$replica_port + fi +} + +echo_dot() { + echo -n "." +} + +start_replication() { + cluster=$1 + + echo "Starting replication for $cluster..." + mysql_version="$(mysql-exec $cluster replica -e "select @@version")" + if [[ $mysql_version =~ "8.4" ]]; then + seconds_behind_source="Seconds_Behind_Source" + replica_terminology="replica" + else + seconds_behind_source="Seconds_Behind_Master" + replica_terminology="slave" + fi + mysql-exec $cluster replica -e "stop $replica_terminology; start $replica_terminology;" + + num_attempts=0 + while mysql-exec $cluster replica -e "show $replica_terminology status\G" | grep $seconds_behind_source | grep -q NULL; do + ((num_attempts = num_attempts + 1)) + if [ $num_attempts -gt 10 ]; then + echo + echo "ERROR replication failure" + exit 1 + fi + echo_dot + sleep 1 + done +} + +build_ghost_command() { + # Build gh-ost command with all standard options + # + # expected $1 to be a comma-separated list of tables to move + + # build comma-separated list of tables to move + move_tables_arg=$(IFS=, ; echo "${tables_to_migrate[*]}") + + # NOTE(chriskirkland): fully qualified package name + failpoint name + cmd="GOTRACEBACK=crash $ghost_binary \ + --move-tables=$move_tables_arg \ + --user=root \ + --password=opensesame \ + --host=$source_replica_host \ + --port=$source_replica_port \ + --assume-master-host=${source_master_host}:${source_master_port} \ + --database=$database \ + --target-user=root \ + --target-password=opensesame \ + --target-host=$target_master_host \ + --target-port=$target_master_port \ + --target-database=$database \ + --serve-socket-file=/tmp/gh-ost.test.sock \ + --initially-drop-socket-file \ + --default-retries=3 \ + --chunk-size=10 \ + --verbose \ + --debug \ + --stack \ + --checkpoint \ + --postpone-cut-over-flag-file=$postpone_cutover_flag_file \ + --checkpoint-seconds=1 \ + --unsafe-fail-points-enabled \ + --execute ${extra_args[@]}" + + if [ -n "$GO_FAILPOINTS" ]; then + cmd="GO_FAILPOINTS=\"$GO_FAILPOINTS\" $cmd" + fi +} + +print_log_excerpt() { + echo "=== Last $test_failure_log_tail_lines lines of $test_logfile ===" + tail -n $test_failure_log_tail_lines $test_logfile + echo "=== End log excerpt ===" +} + +validate_expected_failure() { + # Check if test expected to fail and validate error message + # Expects: tests_path, test_name, execution_result, test_logfile + if [ -f $tests_path/$test_name/expect_failure ]; then + if [ $execution_result -eq 0 ]; then + echo + echo "ERROR $test_name execution was expected to exit on error but did not." + print_log_excerpt + return 1 + fi + if [ -s $tests_path/$test_name/expect_failure ]; then + # 'expect_failure' file has content. We expect to find this content in the log. + expected_error_message="$(cat $tests_path/$test_name/expect_failure)" + if grep -q "$expected_error_message" $test_logfile; then + return 0 + fi + echo + echo "ERROR $test_name execution was expected to exit with error message '${expected_error_message}' but did not." + print_log_excerpt + return 1 + fi + # 'expect_failure' file has no content. We generally agree that the failure is correct + return 0 + fi + + if [ $execution_result -ne 0 ]; then + echo + echo "ERROR $test_name execution failure. cat $test_logfile:" + cat $test_logfile + return 1 + fi + return 0 +} + +cleanup() { + # reset test database + mysql-exec source primary --default-character-set=utf8mb4 $database -e "drop database if exists $database; create database $database;" + mysql-exec target primary --default-character-set=utf8mb4 $database -e "drop database if exists $database; create database $database;" +} + +test_single() { + local test_name + test_name="$1" + + # Read the list of tables to migrate from the test's tables.txt + if [ ! -f $tests_path/$test_name/tables.txt ]; then + echo "🐛 ERROR: $tests_path/$test_name/tables.txt not found" + return 1 + fi + echo "----" + cat $tests_path/$test_name/tables.txt + echo "----" + tables_to_migrate=() + while IFS='' read -r line; do + tables_to_migrate+=("$line") + done < <(cat $tests_path/$test_name/tables.txt) + + if [ -f $tests_path/$test_name/ignore_versions ]; then + ignore_versions=$(cat $tests_path/$test_name/ignore_versions) + mysql_version=$(mysql-exec source primary -s -s -e "select @@version") + mysql_version_comment=$(mysql-exec source primary -s -s -e "select @@version_comment") + if echo "$mysql_version" | egrep -q "^${ignore_versions}"; then + echo -n "Skipping: $test_name" + return 0 + elif echo "$mysql_version_comment" | egrep -i -q "^${ignore_versions}"; then + echo -n "Skipping: $test_name" + return 0 + fi + fi + + echo -n "Testing: $test_name (${#tables_to_migrate[@]} table(s))" + + echo_dot + start_replication source + start_replication target + echo_dot + + if [ -f $tests_path/$test_name/gtid_mode ]; then + target_gtid_mode=$(cat $tests_path/$test_name/gtid_mode) + if [ "$current_gtid_mode" != "$target_gtid_mode" ]; then + echo "gtid_mode is ${current_gtid_mode}, expected ${target_gtid_mode}" + exit 1 + fi + fi + + if [ -f $tests_path/$test_name/sql_mode ]; then + mysql-exec source primary --default-character-set=utf8mb4 $database -e "set @@global.sql_mode='$(cat $tests_path/$test_name/sql_mode)'" + mysql-exec source replica --default-character-set=utf8mb4 $database -e "set @@global.sql_mode='$(cat $tests_path/$test_name/sql_mode)'" + mysql-exec target primary --default-character-set=utf8mb4 $database -e "set @@global.sql_mode='$(cat $tests_path/$test_name/sql_mode)'" + mysql-exec target replica --default-character-set=utf8mb4 $database -e "set @@global.sql_mode='$(cat $tests_path/$test_name/sql_mode)'" + fi + + mysql-exec source primary --default-character-set=utf8mb4 $database <$tests_path/$test_name/create.sql + test_create_result=$? + + if [ $test_create_result -ne 0 ]; then + echo + echo "ERROR $test_name create failure. cat $tests_path/$test_name/create.sql:" + cat $tests_path/$test_name/create.sql + return 1 + fi + + extra_args="" + if [ -f $tests_path/$test_name/extra_args ]; then + extra_args=$(cat $tests_path/$test_name/extra_args) + fi + if [ "$gtid" = true ]; then + extra_args+=" --gtid" + fi + + # graceful sleep for replica to catch up + echo_dot + sleep 1 + + # Check for custom test script + if [ -f $tests_path/$test_name/test.sh ]; then + # Run the custom test script in a subshell with timeout monitoring + # The subshell inherits all functions and variables from the current shell + (source $tests_path/$test_name/test.sh) & + test_pid=$! + + # Monitor the test with timeout + timeout_counter=0 + while kill -0 $test_pid 2>/dev/null; do + if [ $timeout_counter -ge $test_timeout ]; then + kill -TERM $test_pid 2>/dev/null + sleep 1 + kill -KILL $test_pid 2>/dev/null + wait $test_pid 2>/dev/null + echo + echo "ERROR $test_name execution timed out" + print_log_excerpt + return 1 + fi + sleep 1 + ((timeout_counter++)) + done + + # Get the exit code + wait $test_pid 2>/dev/null + execution_result=$? + return $execution_result + + else + + # kick off the on_test script for the test. this enables arbitrary custom logic + # concurrent with the gh-ost process. this enables additional scenarios like + # streaming of writes prior to the write cutover. + # + # IMPORTANT: The on-test script is executed in the background and will be killed as soon + # as the gh-ost process terminates. + if [ -f $tests_path/$test_name/on_test.sh ]; then + $tests_path/$test_name/on_test.sh &> /dev/null & + on_test_pid=$! + fi + + # queue up removal of the postpone cutover flag, otherwise gh-ost hangs on the cutover + ( + sleep 1; + echo "⏩ Sending unpostpone cutover" + rm $postpone_cutover_flag_file &> /dev/null; + ) & + + # Build and execute gh-ost command + build_ghost_command + echo_dot + echo $cmd >$exec_command_file + echo_dot + timeout $test_timeout bash $exec_command_file >$test_logfile 2>&1 + + execution_result=$? + + if [ -n "$on_test_pid" ]; then + kill -KILL $on_test_pid &>/dev/null + fi + + # Check for timeout (exit code 124) + if [ $execution_result -eq 124 ]; then + echo + echo "ERROR $test_name execution timed out" + print_log_excerpt + return 1 + fi + fi + + if [ -f $tests_path/$test_name/sql_mode ]; then + mysql-exec source primary --default-character-set=utf8mb4 $database -e "set @@global.sql_mode='${original_sql_mode}'" + mysql-exec source replica --default-character-set=utf8mb4 $database -e "set @@global.sql_mode='${original_sql_mode}'" + mysql-exec target primary --default-character-set=utf8mb4 $database -e "set @@global.sql_mode='${original_sql_mode}'" + mysql-exec target replica --default-character-set=utf8mb4 $database -e "set @@global.sql_mode='${original_sql_mode}'" + fi + + # Validate expected failure or success + if ! validate_expected_failure; then + return 1 + fi + + # If this was an expected failure test, we're done (no need to validate structure/checksums) + if [ -f $tests_path/$test_name/expect_failure ]; then + return 0 + fi + + # Test succeeded - now validate structure/checksums and contents + for table_name in "${tables_to_migrate[@]}"; do + echo "⚙️ Validating table: $table_name" + + # Validate that the structure of the table matches on the source and target clusters (accounting for table rename on source) + mysql-exec source replica --default-character-set=utf8mb4 $database -e "show create table _${table_name}_del\G" -ss | sed -e "s/_${table_name}_del/${table_name}/g" >$orig_structure_output_file + sed -i "s/_${table_name}_del/${table_name}/g" $orig_structure_output_file + mysql-exec target replica --default-character-set=utf8mb4 $database -e "show create table ${table_name}\G" -ss >$ghost_structure_output_file + + if ! diff $orig_structure_output_file $ghost_structure_output_file > /dev/null 2>&1 ; then + echo "ERROR $test_name: structure mismatch on table $table_name" + echo "---" + diff $orig_structure_output_file $ghost_structure_output_file + + echo "diff $orig_structure_output_file $ghost_structure_output_file" + + return 1 + fi + echo "✅ Table $table_name: structures match" + + echo_dot + + # validate contents match + mysql-exec source replica --default-character-set=utf8mb4 $database -e "select * from _${table_name}_del" -ss >$orig_content_output_file + mysql-exec target replica --default-character-set=utf8mb4 $database -e "select * from ${table_name}" -ss >$ghost_content_output_file + orig_content_checksum=$(cat $orig_content_output_file | md5sum) + ghost_content_checksum=$(cat $ghost_content_output_file | md5sum) + + if [ "$orig_content_checksum" != "$ghost_content_checksum" ]; then + mysql-exec source replica --default-character-set=utf8mb4 $database -e "select * from _${table_name}_del" -ss >$orig_content_output_file + mysql-exec target replica --default-character-set=utf8mb4 $database -e "select * from ${table_name}" -ss >$ghost_content_output_file + echo "ERROR $test_name: checksum mismatch on table $table_name" + echo "---" + diff $orig_content_output_file $ghost_content_output_file + + echo "diff $orig_content_output_file $ghost_content_output_file" + + return 1 + fi + echo "✅ Table $table_name: content checksums match" + + echo_dot + echo_dot + done +} + +enable_failpoint() { + mkdir -p $repo_root/tools/bin + if [ ! -f $repo_root/tools/bin/failpoint-ctl ]; then + echo "⚙️ Installing failpoint" + GOBIN=$repo_root/tools/bin go install github.com/pingcap/failpoint/failpoint-ctl@v0.0.0-20220801062533-2eaa32854a6c + fi + + echo "⚙️ Enabling failpoint" + $repo_root/tools/bin/failpoint-ctl enable go + + echo "✅ Successfully enabled failpoint" +} + +disable_failpoint() { + echo "⚙️ Disabling failpoint" + $repo_root/tools/bin/failpoint-ctl disable go + + echo "✅ Successfully disabled failpoint" +} + +build_binary() { + enable_failpoint + + echo "Building" + rm -f $default_ghost_binary + [ "$ghost_binary" == "" ] && ghost_binary="$default_ghost_binary" + if [ -f "$ghost_binary" ]; then + echo "Using binary: $ghost_binary" + return 0 + fi + + go build -o $ghost_binary go/cmd/gh-ost/main.go + + if [ $? -ne 0 ]; then + echo "Build failure" + exit 1 + fi + + disable_failpoint +} + +test_all() { + build_binary + test_dirs=$(find "$tests_path" -mindepth 1 -maxdepth 1 ! -path . -type d | grep "$test_pattern" | sort) + while read -r test_dir; do + test_name=$(basename "$test_dir") + local test_start_time=$(date +%s) + if ! test_single "$test_name"; then + local test_end_time=$(date +%s) + local test_duration=$((test_end_time - test_start_time)) + echo "+ FAIL (${test_duration}s)" + return 1 + else + local test_end_time=$(date +%s) + local test_duration=$((test_end_time - test_start_time)) + echo + echo "+ pass (${test_duration}s)" + fi + + cleanup + + for cluster in source target; do + mysql_version="$(mysql-exec $cluster replica -e "select @@version")" + replica_terminology="slave" + if [[ $mysql_version =~ "8.4" ]]; then + replica_terminology="replica" + fi + mysql-exec $cluster replica -e "start $replica_terminology" + done + done <<<"$test_dirs" + + echo "✅ All tests completed." +} + +verify_master_and_replica source +verify_master_and_replica target +test_all diff --git a/localtests/move-tables/atomic-multi-table-cutover/create.sql b/localtests/move-tables/atomic-multi-table-cutover/create.sql new file mode 100644 index 000000000..0209fb320 --- /dev/null +++ b/localtests/move-tables/atomic-multi-table-cutover/create.sql @@ -0,0 +1,33 @@ +-- Atomic multi-table cutover test. +-- +-- Two tables of identical shape with a correlation column `txn_id`. The test +-- workload (see test.sh) commits transactions that write the SAME txn_id into +-- BOTH tables. Because all migrated tables are renamed in a single atomic +-- `RENAME TABLE t1 TO ..., t2 TO ...` at cutover, every such transaction lands on +-- the target entirely or not at all -- so the target tables must hold exactly the +-- same set of txn_ids. A regression to per-table sequential RENAME would split a +-- boundary transaction and leave an orphan. + +drop table if exists gh_ost_test; +create table gh_ost_test ( + id int(11) NOT NULL AUTO_INCREMENT, + txn_id int(11) NOT NULL, + payload varchar(32) NOT NULL, + PRIMARY KEY (id), + KEY txn_ix (txn_id) +); + +insert into gh_ost_test (txn_id, payload) values + (0, 'seed'), (0, 'seed'), (0, 'seed'), (0, 'seed'), (0, 'seed'); + +drop table if exists gh_ost_test_other; +create table gh_ost_test_other ( + id int(11) NOT NULL AUTO_INCREMENT, + txn_id int(11) NOT NULL, + payload varchar(32) NOT NULL, + PRIMARY KEY (id), + KEY txn_ix (txn_id) +); + +insert into gh_ost_test_other (txn_id, payload) values + (0, 'seed'), (0, 'seed'), (0, 'seed'), (0, 'seed'), (0, 'seed'); diff --git a/localtests/move-tables/atomic-multi-table-cutover/tables.txt b/localtests/move-tables/atomic-multi-table-cutover/tables.txt new file mode 100644 index 000000000..30fa51c70 --- /dev/null +++ b/localtests/move-tables/atomic-multi-table-cutover/tables.txt @@ -0,0 +1,2 @@ +gh_ost_test +gh_ost_test_other diff --git a/localtests/move-tables/atomic-multi-table-cutover/test.sh b/localtests/move-tables/atomic-multi-table-cutover/test.sh new file mode 100755 index 000000000..ae5cf4fd3 --- /dev/null +++ b/localtests/move-tables/atomic-multi-table-cutover/test.sh @@ -0,0 +1,108 @@ +#!/bin/bash +# Atomic multi-table cutover test. +# +# A workload commits transactions that each write the SAME txn_id into BOTH +# migrated tables, in a tight loop, right up to the cutover. Because gh-ost +# renames every migrated table in ONE atomic `RENAME TABLE t1 TO ..., t2 TO ...`, +# each cross-table transaction lands on the target entirely or not at all. +# +# Verification is deterministic (final-state set + checksum comparison, no timing +# assertions): the set of txn_ids in target gh_ost_test must exactly equal the set +# in target gh_ost_test_other. A regression from the atomic multi-table RENAME to +# a per-table sequential RENAME splits a boundary transaction across the two +# tables and leaves an orphan txn_id, failing this test. + +database=test + +build_binary +build_ghost_command + +###################################################################################################### +### Drive cross-table transactions, then cut over while they are still committing +###################################################################################################### + +echo "⚙️ Starting cross-table transaction workload..." + +# Each iteration commits one transaction touching BOTH tables with a shared +# txn_id. The loop runs with no delay until the tables are renamed at cutover +# (the INSERT then errors and the loop exits), so cross-table transactions are +# committing continuously while the cutover happens. Note: we do NOT (and cannot, +# from a shell) control whether a transaction is literally mid-commit at the +# RENAME instant -- that is timing-dependent. Correctness is asserted +# deterministically on the final state below (no orphaned txn_id across the pair +# + per-table checksums), not on hitting that instant. +( + n=1 + while true; do + mysql-exec source primary $database -e \ + "START TRANSACTION; \ + INSERT INTO gh_ost_test (txn_id, payload) VALUES ($n, 'a'); \ + INSERT INTO gh_ost_test_other (txn_id, payload) VALUES ($n, 'b'); \ + COMMIT;" 2>/dev/null || break + n=$((n + 1)) + done +) & +workload_pid=$! + +# Remove the postpone flag so cutover proceeds while the workload is still +# committing cross-table transactions. This is a best-effort time-based overlap, +# not a guarantee that a transaction is mid-commit at the exact RENAME; the +# atomicity guarantee is verified on the final target state below. +( + sleep 4 + echo "⏩ Sending unpostpone cutover" + rm $postpone_cutover_flag_file &> /dev/null +) & + +echo > $test_logfile +bash -c "$cmd" >> $test_logfile 2>&1 +ghost_result=$? + +kill $workload_pid &> /dev/null + +if [ $ghost_result -ne 0 ]; then + echo "ERROR: gh-ost should have succeeded but did not. ($ghost_result)" + return 1 +fi + +echo -e "\n\n\n\n\n" + +###################################################################################################### +### Validate atomicity + data integrity (read primaries to avoid replication lag) +###################################################################################################### + +echo "⚙️ Validating atomic multi-table cutover..." + +# Sanity: the workload must have landed cross-table rows on the target, otherwise +# the atomicity assertion below would be vacuously true. +paired=$(mysql-exec target primary $database -sNe "SELECT COUNT(*) FROM gh_ost_test WHERE txn_id > 0;") +if [ -z "$paired" ] || [ "$paired" -lt 1 ]; then + echo "ERROR: workload produced no cross-table rows on target; test would be vacuous." + return 1 +fi + +# Atomicity invariant: every cross-table transaction landed entirely or not at +# all, i.e. the txn_id sets match across the two target tables (no orphans). +orphans_a=$(mysql-exec target primary $database -sNe \ + "SELECT COUNT(*) FROM gh_ost_test t1 WHERE t1.txn_id > 0 \ + AND NOT EXISTS (SELECT 1 FROM gh_ost_test_other t2 WHERE t2.txn_id = t1.txn_id);") +orphans_b=$(mysql-exec target primary $database -sNe \ + "SELECT COUNT(*) FROM gh_ost_test_other t2 WHERE t2.txn_id > 0 \ + AND NOT EXISTS (SELECT 1 FROM gh_ost_test t1 WHERE t1.txn_id = t2.txn_id);") + +if [ "$orphans_a" != "0" ] || [ "$orphans_b" != "0" ]; then + echo "ERROR: non-atomic cutover: ${orphans_a} txn_id(s) in gh_ost_test missing from gh_ost_test_other; ${orphans_b} the other way." + return 1 +fi + +# Full data integrity: each migrated table matches its source rollback handle. +for table_name in gh_ost_test gh_ost_test_other; do + src_checksum=$(mysql-exec source primary $database -ss -e "SELECT * FROM _${table_name}_del ORDER BY id" | md5sum) + dst_checksum=$(mysql-exec target primary $database -ss -e "SELECT * FROM ${table_name} ORDER BY id" | md5sum) + if [ "$src_checksum" != "$dst_checksum" ]; then + echo "ERROR: checksum mismatch on ${table_name} between source _del and target." + return 1 + fi +done + +echo "✅ Atomic multi-table cutover validated: ${paired} cross-table transactions, no orphans, checksums match." diff --git a/localtests/move-tables/generated-columns/create.sql b/localtests/move-tables/generated-columns/create.sql new file mode 100644 index 000000000..c326893c3 --- /dev/null +++ b/localtests/move-tables/generated-columns/create.sql @@ -0,0 +1,36 @@ +drop table if exists gh_ost_test; +create table gh_ost_test ( + id int auto_increment, + a int not null, + virtual_sum int as (a + 10) virtual not null, + b int not null, + stored_sum int as (a + b) stored not null, + json_value json default null, + virtual_json_value varchar(16) as ( + coalesce(json_unquote(json_extract(json_value, '$.value')), 'direct') + ) virtual, + primary key(id) +) auto_increment=1; + +insert into gh_ost_test (a, b, json_value) values + (1, 2, json_object('value', 'team')), + (3, 5, json_object('value', 'project')), + (8, 13, null); + +drop event if exists gh_ost_test; +delimiter ;; +create event gh_ost_test + on schedule every 1 second + starts current_timestamp + ends current_timestamp + interval 60 second + on completion not preserve + enable + do +begin + insert into gh_ost_test (a, b, json_value) values (2, 3, json_object('value', 'team')); + insert into gh_ost_test (a, b, json_value) values (5, 8, json_object('value', 'project')); + insert into gh_ost_test (a, b, json_value) values (13, 21, null); + update gh_ost_test set a=a+1, b=b+2, json_value=json_object('value', 'updated') where id <= 3; + update gh_ost_test set b=b+1 where id > 3; + delete from gh_ost_test where id > 3 order by id limit 1; +end ;; diff --git a/localtests/move-tables/generated-columns/tables.txt b/localtests/move-tables/generated-columns/tables.txt new file mode 100644 index 000000000..11fc5eef8 --- /dev/null +++ b/localtests/move-tables/generated-columns/tables.txt @@ -0,0 +1 @@ +gh_ost_test diff --git a/localtests/move-tables/json/create.sql b/localtests/move-tables/json/create.sql new file mode 100644 index 000000000..ef1d2726b --- /dev/null +++ b/localtests/move-tables/json/create.sql @@ -0,0 +1,27 @@ +create table gh_ost_test ( + id int auto_increment, + json_value json not null, + primary key(id) +) auto_increment=1; + +insert into gh_ost_test (json_value) values + (json_object('message', 'first', 'nested', json_object('enabled', true))), + (json_object('message', 'second', 'items', json_array(1, 2, 3))), + (json_object('message', 'third', 'value', 42)); + +drop event if exists gh_ost_test; +delimiter ;; +create event gh_ost_test + on schedule every 1 second + starts current_timestamp + ends current_timestamp + interval 60 second + on completion not preserve + enable + do +begin + insert into gh_ost_test (json_value) values + (json_object('message', 'inserted', 'items', json_array('a', 'b'))); + update gh_ost_test + set json_value=json_set(json_value, '$.updated', true) + where id <= 3; +end ;; \ No newline at end of file diff --git a/localtests/move-tables/json/tables.txt b/localtests/move-tables/json/tables.txt new file mode 100644 index 000000000..11fc5eef8 --- /dev/null +++ b/localtests/move-tables/json/tables.txt @@ -0,0 +1 @@ +gh_ost_test diff --git a/localtests/move-tables/multiple-three-concurrent-writes/create.sql b/localtests/move-tables/multiple-three-concurrent-writes/create.sql new file mode 100644 index 000000000..f92cef32e --- /dev/null +++ b/localtests/move-tables/multiple-three-concurrent-writes/create.sql @@ -0,0 +1,92 @@ +-- Three tables with distinct schemas, primary-key types, and row counts. This +-- exercises the multi-table move-tables path at its widest: per-table +-- runtime state, per-table query builders, interleaved row copy where the tables +-- finish at different times, and a single atomic multi-table RENAME at cutover. +-- +-- These three tables are the canonical superset used by the manual harness +-- (script/move-tables/setup, reset, insert-source-primary-loop). The `single` +-- and `multiple-two` localtest fixtures move subsets of them. + +drop table if exists gh_ost_test; +create table gh_ost_test ( + id bigint(20) NOT NULL AUTO_INCREMENT, + column1 int(11) NOT NULL, + column2 smallint(5) unsigned NOT NULL, + column3 mediumint(8) unsigned NOT NULL, + column4 tinyint(3) unsigned NOT NULL, + column5 int(11) NOT NULL, + column6 int(11) NOT NULL, + PRIMARY KEY (id), + KEY c12_ix (column1, column2) +) auto_increment=1; + +insert into gh_ost_test values + (NULL, 1001, 100, 500000, 10, 1700000001, 1700000002), + (NULL, 1002, 200, 600000, 20, 1700000003, 1700000004), + (NULL, 1003, 300, 700000, 30, 1700000005, 1700000006), + (NULL, 1004, 400, 800000, 40, 1700000007, 1700000008), + (NULL, 1005, 500, 900000, 50, 1700000009, 1700000010), + (NULL, 1006, 600, 1000000, 60, 1700000011, 1700000012), + (NULL, 1007, 700, 1100000, 70, 1700000013, 1700000014), + (NULL, 1008, 800, 1200000, 80, 1700000015, 1700000016), + (NULL, 1009, 900, 1300000, 90, 1700000017, 1700000018), + (NULL, 1010, 1000, 1400000, 100, 1700000019, 1700000020), + (NULL, 1011, 1100, 1500000, 110, 1700000021, 1700000022), + (NULL, 1012, 1200, 1600000, 120, 1700000023, 1700000024), + (NULL, 1013, 1300, 1700000, 130, 1700000025, 1700000026), + (NULL, 1014, 1400, 1800000, 140, 1700000027, 1700000028), + (NULL, 1015, 1500, 1900000, 150, 1700000029, 1700000030), + (NULL, 1016, 1600, 2000000, 160, 1700000031, 1700000032), + (NULL, 1017, 1700, 2100000, 170, 1700000033, 1700000034), + (NULL, 1018, 1800, 2200000, 180, 1700000035, 1700000036), + (NULL, 1019, 1900, 2300000, 190, 1700000037, 1700000038), + (NULL, 1020, 2000, 2400000, 200, 1700000039, 1700000040), + (NULL, 1021, 2100, 2500000, 210, 1700000041, 1700000042), + (NULL, 1022, 2200, 2600000, 220, 1700000043, 1700000044), + (NULL, 1023, 2300, 2700000, 230, 1700000045, 1700000046), + (NULL, 1024, 2400, 2800000, 240, 1700000047, 1700000048), + (NULL, 1025, 2500, 2900000, 250, 1700000049, 1700000050); + +drop table if exists gh_ost_test_other; +create table gh_ost_test_other ( + uid int(11) NOT NULL, + name varchar(64) NOT NULL, + amount decimal(10,2) NOT NULL, + created_at datetime NOT NULL, + PRIMARY KEY (uid), + UNIQUE KEY name_uq (name) +); + +insert into gh_ost_test_other values + (1, 'alpha', 10.50, '2024-01-01 10:00:00'), + (2, 'bravo', 20.75, '2024-01-02 11:00:00'), + (3, 'charlie', 30.00, '2024-01-03 12:00:00'), + (4, 'delta', 40.25, '2024-01-04 13:00:00'), + (5, 'echo', 50.50, '2024-01-05 14:00:00'), + (6, 'foxtrot', 60.75, '2024-01-06 15:00:00'), + (7, 'golf', 70.00, '2024-01-07 16:00:00'), + (8, 'hotel', 80.25, '2024-01-08 17:00:00'), + (9, 'india', 90.50, '2024-01-09 18:00:00'), + (10, 'juliet', 100.75, '2024-01-10 19:00:00'), + (11, 'kilo', 110.00, '2024-01-11 20:00:00'), + (12, 'lima', 120.25, '2024-01-12 21:00:00'); + +drop table if exists gh_ost_test_third; +create table gh_ost_test_third ( + code varchar(32) NOT NULL, + label varchar(128) NOT NULL, + score double NOT NULL, + updated_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (code), + KEY score_ix (score) +); + +insert into gh_ost_test_third (code, label, score) values + ('code_1', 'label_1', 1.5), + ('code_2', 'label_2', 2.5), + ('code_3', 'label_3', 3.5), + ('code_4', 'label_4', 4.5), + ('code_5', 'label_5', 5.5), + ('code_6', 'label_6', 6.5), + ('code_7', 'label_7', 7.5), + ('code_8', 'label_8', 8.5); diff --git a/localtests/move-tables/multiple-three-concurrent-writes/on_test.sh b/localtests/move-tables/multiple-three-concurrent-writes/on_test.sh new file mode 100755 index 000000000..06d703361 --- /dev/null +++ b/localtests/move-tables/multiple-three-concurrent-writes/on_test.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +# Three-table move with sustained DML on all three tables during the +# copy. insert-source-primary-loop auto-detects every seeded fixture +# (gh_ost_test, gh_ost_test_other, gh_ost_test_third) and writes to all of them, +# so each migrated table sees concurrent inserts while gh-ost copies and drains. +# The harness then validates per-table structure + content checksums (source +# `_
_del` vs target), which deterministically proves every concurrent +# write was captured on the target. +DATABASE=test script/move-tables/insert-source-primary-loop 100 0.01 100 & +sleep 5 && kill $! diff --git a/localtests/move-tables/multiple-three-concurrent-writes/tables.txt b/localtests/move-tables/multiple-three-concurrent-writes/tables.txt new file mode 100644 index 000000000..72f7ba8f6 --- /dev/null +++ b/localtests/move-tables/multiple-three-concurrent-writes/tables.txt @@ -0,0 +1,3 @@ +gh_ost_test +gh_ost_test_other +gh_ost_test_third diff --git a/localtests/move-tables/multiple-three/create.sql b/localtests/move-tables/multiple-three/create.sql new file mode 100644 index 000000000..631653315 --- /dev/null +++ b/localtests/move-tables/multiple-three/create.sql @@ -0,0 +1,92 @@ +-- Three tables with distinct schemas, primary-key types, and row counts. This +-- exercises the multi-table move-tables path (§2.1-2.4) at its widest: per-table +-- runtime state, per-table query builders, interleaved row copy where the tables +-- finish at different times, and a single atomic multi-table RENAME at cutover. +-- +-- These three tables are the canonical superset used by the manual harness +-- (script/move-tables/setup, reset, insert-source-primary-loop). The `single` +-- and `multiple-two` localtest fixtures move subsets of them. + +drop table if exists gh_ost_test; +create table gh_ost_test ( + id bigint(20) NOT NULL AUTO_INCREMENT, + column1 int(11) NOT NULL, + column2 smallint(5) unsigned NOT NULL, + column3 mediumint(8) unsigned NOT NULL, + column4 tinyint(3) unsigned NOT NULL, + column5 int(11) NOT NULL, + column6 int(11) NOT NULL, + PRIMARY KEY (id), + KEY c12_ix (column1, column2) +) auto_increment=1; + +insert into gh_ost_test values + (NULL, 1001, 100, 500000, 10, 1700000001, 1700000002), + (NULL, 1002, 200, 600000, 20, 1700000003, 1700000004), + (NULL, 1003, 300, 700000, 30, 1700000005, 1700000006), + (NULL, 1004, 400, 800000, 40, 1700000007, 1700000008), + (NULL, 1005, 500, 900000, 50, 1700000009, 1700000010), + (NULL, 1006, 600, 1000000, 60, 1700000011, 1700000012), + (NULL, 1007, 700, 1100000, 70, 1700000013, 1700000014), + (NULL, 1008, 800, 1200000, 80, 1700000015, 1700000016), + (NULL, 1009, 900, 1300000, 90, 1700000017, 1700000018), + (NULL, 1010, 1000, 1400000, 100, 1700000019, 1700000020), + (NULL, 1011, 1100, 1500000, 110, 1700000021, 1700000022), + (NULL, 1012, 1200, 1600000, 120, 1700000023, 1700000024), + (NULL, 1013, 1300, 1700000, 130, 1700000025, 1700000026), + (NULL, 1014, 1400, 1800000, 140, 1700000027, 1700000028), + (NULL, 1015, 1500, 1900000, 150, 1700000029, 1700000030), + (NULL, 1016, 1600, 2000000, 160, 1700000031, 1700000032), + (NULL, 1017, 1700, 2100000, 170, 1700000033, 1700000034), + (NULL, 1018, 1800, 2200000, 180, 1700000035, 1700000036), + (NULL, 1019, 1900, 2300000, 190, 1700000037, 1700000038), + (NULL, 1020, 2000, 2400000, 200, 1700000039, 1700000040), + (NULL, 1021, 2100, 2500000, 210, 1700000041, 1700000042), + (NULL, 1022, 2200, 2600000, 220, 1700000043, 1700000044), + (NULL, 1023, 2300, 2700000, 230, 1700000045, 1700000046), + (NULL, 1024, 2400, 2800000, 240, 1700000047, 1700000048), + (NULL, 1025, 2500, 2900000, 250, 1700000049, 1700000050); + +drop table if exists gh_ost_test_other; +create table gh_ost_test_other ( + uid int(11) NOT NULL, + name varchar(64) NOT NULL, + amount decimal(10,2) NOT NULL, + created_at datetime NOT NULL, + PRIMARY KEY (uid), + UNIQUE KEY name_uq (name) +); + +insert into gh_ost_test_other values + (1, 'alpha', 10.50, '2024-01-01 10:00:00'), + (2, 'bravo', 20.75, '2024-01-02 11:00:00'), + (3, 'charlie', 30.00, '2024-01-03 12:00:00'), + (4, 'delta', 40.25, '2024-01-04 13:00:00'), + (5, 'echo', 50.50, '2024-01-05 14:00:00'), + (6, 'foxtrot', 60.75, '2024-01-06 15:00:00'), + (7, 'golf', 70.00, '2024-01-07 16:00:00'), + (8, 'hotel', 80.25, '2024-01-08 17:00:00'), + (9, 'india', 90.50, '2024-01-09 18:00:00'), + (10, 'juliet', 100.75, '2024-01-10 19:00:00'), + (11, 'kilo', 110.00, '2024-01-11 20:00:00'), + (12, 'lima', 120.25, '2024-01-12 21:00:00'); + +drop table if exists gh_ost_test_third; +create table gh_ost_test_third ( + code varchar(32) NOT NULL, + label varchar(128) NOT NULL, + score double NOT NULL, + updated_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (code), + KEY score_ix (score) +); + +insert into gh_ost_test_third (code, label, score) values + ('code_1', 'label_1', 1.5), + ('code_2', 'label_2', 2.5), + ('code_3', 'label_3', 3.5), + ('code_4', 'label_4', 4.5), + ('code_5', 'label_5', 5.5), + ('code_6', 'label_6', 6.5), + ('code_7', 'label_7', 7.5), + ('code_8', 'label_8', 8.5); diff --git a/localtests/move-tables/multiple-three/tables.txt b/localtests/move-tables/multiple-three/tables.txt new file mode 100644 index 000000000..72f7ba8f6 --- /dev/null +++ b/localtests/move-tables/multiple-three/tables.txt @@ -0,0 +1,3 @@ +gh_ost_test +gh_ost_test_other +gh_ost_test_third diff --git a/localtests/move-tables/multiple-two/create.sql b/localtests/move-tables/multiple-two/create.sql new file mode 100644 index 000000000..3d4a5d70b --- /dev/null +++ b/localtests/move-tables/multiple-two/create.sql @@ -0,0 +1,68 @@ +-- Two tables with different schemas, primary-key types, and row counts. This +-- exercises the multi-table move-tables path (§2.1-2.4): per-table runtime +-- state, per-table query builders, interleaved row copy where the tables finish +-- at different times, and a single atomic multi-table RENAME at cutover. + +drop table if exists gh_ost_test; +create table gh_ost_test ( + id bigint(20) NOT NULL AUTO_INCREMENT, + column1 int(11) NOT NULL, + column2 smallint(5) unsigned NOT NULL, + column3 mediumint(8) unsigned NOT NULL, + column4 tinyint(3) unsigned NOT NULL, + column5 int(11) NOT NULL, + column6 int(11) NOT NULL, + PRIMARY KEY (id), + KEY c12_ix (column1, column2) +) auto_increment=1; + +insert into gh_ost_test values + (NULL, 1001, 100, 500000, 10, 1700000001, 1700000002), + (NULL, 1002, 200, 600000, 20, 1700000003, 1700000004), + (NULL, 1003, 300, 700000, 30, 1700000005, 1700000006), + (NULL, 1004, 400, 800000, 40, 1700000007, 1700000008), + (NULL, 1005, 500, 900000, 50, 1700000009, 1700000010), + (NULL, 1006, 600, 1000000, 60, 1700000011, 1700000012), + (NULL, 1007, 700, 1100000, 70, 1700000013, 1700000014), + (NULL, 1008, 800, 1200000, 80, 1700000015, 1700000016), + (NULL, 1009, 900, 1300000, 90, 1700000017, 1700000018), + (NULL, 1010, 1000, 1400000, 100, 1700000019, 1700000020), + (NULL, 1011, 1100, 1500000, 110, 1700000021, 1700000022), + (NULL, 1012, 1200, 1600000, 120, 1700000023, 1700000024), + (NULL, 1013, 1300, 1700000, 130, 1700000025, 1700000026), + (NULL, 1014, 1400, 1800000, 140, 1700000027, 1700000028), + (NULL, 1015, 1500, 1900000, 150, 1700000029, 1700000030), + (NULL, 1016, 1600, 2000000, 160, 1700000031, 1700000032), + (NULL, 1017, 1700, 2100000, 170, 1700000033, 1700000034), + (NULL, 1018, 1800, 2200000, 180, 1700000035, 1700000036), + (NULL, 1019, 1900, 2300000, 190, 1700000037, 1700000038), + (NULL, 1020, 2000, 2400000, 200, 1700000039, 1700000040), + (NULL, 1021, 2100, 2500000, 210, 1700000041, 1700000042), + (NULL, 1022, 2200, 2600000, 220, 1700000043, 1700000044), + (NULL, 1023, 2300, 2700000, 230, 1700000045, 1700000046), + (NULL, 1024, 2400, 2800000, 240, 1700000047, 1700000048), + (NULL, 1025, 2500, 2900000, 250, 1700000049, 1700000050); + +drop table if exists gh_ost_test_other; +create table gh_ost_test_other ( + uid int(11) NOT NULL, + name varchar(64) NOT NULL, + amount decimal(10,2) NOT NULL, + created_at datetime NOT NULL, + PRIMARY KEY (uid), + UNIQUE KEY name_uq (name) +); + +insert into gh_ost_test_other values + (1, 'alpha', 10.50, '2024-01-01 10:00:00'), + (2, 'bravo', 20.75, '2024-01-02 11:00:00'), + (3, 'charlie', 30.00, '2024-01-03 12:00:00'), + (4, 'delta', 40.25, '2024-01-04 13:00:00'), + (5, 'echo', 50.50, '2024-01-05 14:00:00'), + (6, 'foxtrot', 60.75, '2024-01-06 15:00:00'), + (7, 'golf', 70.00, '2024-01-07 16:00:00'), + (8, 'hotel', 80.25, '2024-01-08 17:00:00'), + (9, 'india', 90.50, '2024-01-09 18:00:00'), + (10, 'juliet', 100.75, '2024-01-10 19:00:00'), + (11, 'kilo', 110.00, '2024-01-11 20:00:00'), + (12, 'lima', 120.25, '2024-01-12 21:00:00'); diff --git a/localtests/move-tables/multiple-two/tables.txt b/localtests/move-tables/multiple-two/tables.txt new file mode 100644 index 000000000..30fa51c70 --- /dev/null +++ b/localtests/move-tables/multiple-two/tables.txt @@ -0,0 +1,2 @@ +gh_ost_test +gh_ost_test_other diff --git a/localtests/move-tables/resume-panic-before-drain-complete/create.sql b/localtests/move-tables/resume-panic-before-drain-complete/create.sql new file mode 100644 index 000000000..46e919003 --- /dev/null +++ b/localtests/move-tables/resume-panic-before-drain-complete/create.sql @@ -0,0 +1,34 @@ +drop table if exists gh_ost_test; +create table gh_ost_test ( + id bigint(20) NOT NULL AUTO_INCREMENT, + column1 int(11) NOT NULL, + column2 smallint(5) unsigned NOT NULL, + column3 mediumint(8) unsigned NOT NULL, + column4 tinyint(3) unsigned NOT NULL, + column5 int(11) NOT NULL, + column6 int(11) NOT NULL, + PRIMARY KEY (id), + KEY c12_ix (column1, column2) +) auto_increment=1; + +insert into gh_ost_test values + (NULL, 1001, 100, 500000, 10, 1700000001, 1700000002), + (NULL, 1002, 200, 600000, 20, 1700000003, 1700000004), + (NULL, 1003, 300, 700000, 30, 1700000005, 1700000006), + (NULL, 1004, 400, 800000, 40, 1700000007, 1700000008), + (NULL, 1005, 500, 900000, 50, 1700000009, 1700000010), + (NULL, 1006, 600, 1000000, 60, 1700000011, 1700000012), + (NULL, 1007, 700, 1100000, 70, 1700000013, 1700000014), + (NULL, 1008, 800, 1200000, 80, 1700000015, 1700000016), + (NULL, 1009, 900, 1300000, 90, 1700000017, 1700000018), + (NULL, 1010, 1000, 1400000, 100, 1700000019, 1700000020), + (NULL, 1011, 1100, 1500000, 110, 1700000021, 1700000022), + (NULL, 1012, 1200, 1600000, 120, 1700000023, 1700000024), + (NULL, 1013, 1300, 1700000, 130, 1700000025, 1700000026), + (NULL, 1014, 1400, 1800000, 140, 1700000027, 1700000028), + (NULL, 1015, 1500, 1900000, 150, 1700000029, 1700000030), + (NULL, 1016, 1600, 2000000, 160, 1700000031, 1700000032), + (NULL, 1017, 1700, 2100000, 170, 1700000033, 1700000034), + (NULL, 1018, 1800, 2200000, 180, 1700000035, 1700000036), + (NULL, 1019, 1900, 2300000, 190, 1700000037, 1700000038), + (NULL, 1020, 2000, 2400000, 200, 1700000039, 1700000040); \ No newline at end of file diff --git a/localtests/move-tables/resume-panic-before-drain-complete/tables.txt b/localtests/move-tables/resume-panic-before-drain-complete/tables.txt new file mode 100644 index 000000000..11fc5eef8 --- /dev/null +++ b/localtests/move-tables/resume-panic-before-drain-complete/tables.txt @@ -0,0 +1 @@ +gh_ost_test diff --git a/localtests/move-tables/resume-panic-before-drain-complete/test.sh b/localtests/move-tables/resume-panic-before-drain-complete/test.sh new file mode 100644 index 000000000..d73436f3d --- /dev/null +++ b/localtests/move-tables/resume-panic-before-drain-complete/test.sh @@ -0,0 +1,129 @@ + +#!/bin/bash +# Custom test: +# - panic after RENAME (T1) and prior to drain completion (T3), prior to cutover completion +# - validate RENAME and source writes are not possible +# - resume and complete the migration + +set -x + +database=test +table_name=gh_ost_test + +# Build gh-ost command from scratch using framework function (required to inject failpoints) +rm $ghost_binary +build_binary + +###################################################################################################### +### Run #1: Should panic after RENAME (T1) and before drain completion (T3) +###################################################################################################### + +echo "⚙️ Starting migration with failpoint (run #1)..." + +# Build the gh-ost command using the framework function +GO_FAILPOINTS="github.com/github/gh-ost/go/base/move-tables-panic-before-drain-completion=return(true)" build_ghost_command + +# queue up removal of the postpone cutover flag, otherwise gh-ost hangs on the cutover +( + sleep 2; + echo "⏩ Sending unpostpone cutover" + rm $postpone_cutover_flag_file &> /dev/null; +) & + +# Run the gh-ost command, expecting panic on the failpoint the first time +echo_dot +echo > $test_logfile +bash -c "$cmd" >>$test_logfile 2>&1 +ghost_result=$? + +if [ $ghost_result -eq 0 ]; then + echo "ERROR: gh-ost should have failed but did not." + return 1 +fi + +echo -e "\n\n\n\n\n" + +###################################################################################################### +### Intermediate validation +###################################################################################################### + +echo "⚙️ Validating checkpointed state on unexpected exit..." + +# Table was renamed on source +mysql-exec source primary $database -sNe "SELECT 1 FROM ${table_name} LIMIT 1;" +if [ $? -eq 0 ]; then + echo "ERROR: Table '${table_name}' exists on source but show have been renamed." + return 1 +fi + +mysql-exec source primary $database -sNe "SELECT 1 FROM _${table_name}_del LIMIT 1;" +if [ $? -gt 0 ]; then + echo "ERROR: Renamed table '_${table_name}_del' does not exist on source." + return 1 +fi + +# Table not writeable on source +mysql-exec source primary $database -e "INSERT INTO ${table_name} VALUES (NULL, 1021, 2001, 2400001, 201, 1700000041, 1700000041);" +if [ $? -eq 0 ]; then + echo "ERROR: Table '${table_name}' was writeable on source but should not be!." + return 1 +fi + +# Table still exists on target +mysql-exec target primary $database -sNe "SELECT 1 FROM ${table_name} LIMIT 1;" +if [ $? -gt 0 ]; then + echo "ERROR: Table '${table_name}' does not exist on target." + return 1 +fi + +# validate last checkpoint (cutover started and drain GTID are set). The +# checkpoint table is named from the run token (_gho__ghk), so look it up +# by pattern rather than a static per-table name. +checkpoint_table=$(mysql-exec target primary $database -sNe "SELECT table_name FROM information_schema.tables WHERE table_schema='${database}' AND table_name LIKE '\\_gho\\_%\\_ghk' LIMIT 1;") +if [ -z "$checkpoint_table" ]; then + echo "ERROR: Checkpoint table does not exist." + return 1 +fi + +cutover_started=$(mysql-exec target primary $database -Ne "SELECT gh_ost_move_tables_cutover_started FROM \`${checkpoint_table}\` ORDER BY gh_ost_chk_id DESC LIMIT 1;") +if [ "$cutover_started" != 1 ]; then + echo "ERROR: Expected cutover started to be set in last checkpoint." + return 1 +fi + +drain_gtid=$(mysql-exec target primary $database -Ne "SELECT gh_ost_move_tables_drain_gtid FROM \`${checkpoint_table}\` ORDER BY gh_ost_chk_id DESC LIMIT 1;") +if [ "$drain_gtid" == "" ]; then + echo "ERROR: Expected drain GTID to be set in last checkpoint." + return 1 +fi + +echo "✅ Validated checkpointed state on unexpected exit..." + +echo -e "\n\n\n\n\n" + +###################################################################################################### +### Run #2: Resume and complete the migration +###################################################################################################### + +echo "⚙️ Resuming migration (run #2)..." + +# resume migration +build_ghost_command +cmd="$cmd --resume" + +# queue up removal of the postpone cutover flag, otherwise gh-ost hangs on the cutover +( + sleep 2; + echo "⏩ Sending unpostpone cutover" + rm $postpone_cutover_flag_file &> /dev/null; +) & + +bash -c "$cmd" >>$test_logfile 2>&1 +ghost_result=$? + +if [ $ghost_result -ne 0 ]; then + echo "ERROR: gh-ost should have succeeded but did not. ($ghost_result)" + return 1 +fi + +echo -e "\n\n\n\n\n" diff --git a/localtests/move-tables/resume-panic-before-on-success-hook/create.sql b/localtests/move-tables/resume-panic-before-on-success-hook/create.sql new file mode 100644 index 000000000..46e919003 --- /dev/null +++ b/localtests/move-tables/resume-panic-before-on-success-hook/create.sql @@ -0,0 +1,34 @@ +drop table if exists gh_ost_test; +create table gh_ost_test ( + id bigint(20) NOT NULL AUTO_INCREMENT, + column1 int(11) NOT NULL, + column2 smallint(5) unsigned NOT NULL, + column3 mediumint(8) unsigned NOT NULL, + column4 tinyint(3) unsigned NOT NULL, + column5 int(11) NOT NULL, + column6 int(11) NOT NULL, + PRIMARY KEY (id), + KEY c12_ix (column1, column2) +) auto_increment=1; + +insert into gh_ost_test values + (NULL, 1001, 100, 500000, 10, 1700000001, 1700000002), + (NULL, 1002, 200, 600000, 20, 1700000003, 1700000004), + (NULL, 1003, 300, 700000, 30, 1700000005, 1700000006), + (NULL, 1004, 400, 800000, 40, 1700000007, 1700000008), + (NULL, 1005, 500, 900000, 50, 1700000009, 1700000010), + (NULL, 1006, 600, 1000000, 60, 1700000011, 1700000012), + (NULL, 1007, 700, 1100000, 70, 1700000013, 1700000014), + (NULL, 1008, 800, 1200000, 80, 1700000015, 1700000016), + (NULL, 1009, 900, 1300000, 90, 1700000017, 1700000018), + (NULL, 1010, 1000, 1400000, 100, 1700000019, 1700000020), + (NULL, 1011, 1100, 1500000, 110, 1700000021, 1700000022), + (NULL, 1012, 1200, 1600000, 120, 1700000023, 1700000024), + (NULL, 1013, 1300, 1700000, 130, 1700000025, 1700000026), + (NULL, 1014, 1400, 1800000, 140, 1700000027, 1700000028), + (NULL, 1015, 1500, 1900000, 150, 1700000029, 1700000030), + (NULL, 1016, 1600, 2000000, 160, 1700000031, 1700000032), + (NULL, 1017, 1700, 2100000, 170, 1700000033, 1700000034), + (NULL, 1018, 1800, 2200000, 180, 1700000035, 1700000036), + (NULL, 1019, 1900, 2300000, 190, 1700000037, 1700000038), + (NULL, 1020, 2000, 2400000, 200, 1700000039, 1700000040); \ No newline at end of file diff --git a/localtests/move-tables/resume-panic-before-on-success-hook/hooks/gh-ost-on-success b/localtests/move-tables/resume-panic-before-on-success-hook/hooks/gh-ost-on-success new file mode 100755 index 000000000..a2c0b6926 --- /dev/null +++ b/localtests/move-tables/resume-panic-before-on-success-hook/hooks/gh-ost-on-success @@ -0,0 +1,4 @@ +#!/bin/bash + +# touch file to mark completion of on-success hook +touch /tmp/gh-ost-hooks/on-success \ No newline at end of file diff --git a/localtests/move-tables/resume-panic-before-on-success-hook/tables.txt b/localtests/move-tables/resume-panic-before-on-success-hook/tables.txt new file mode 100644 index 000000000..11fc5eef8 --- /dev/null +++ b/localtests/move-tables/resume-panic-before-on-success-hook/tables.txt @@ -0,0 +1 @@ +gh_ost_test diff --git a/localtests/move-tables/resume-panic-before-on-success-hook/test.sh b/localtests/move-tables/resume-panic-before-on-success-hook/test.sh new file mode 100644 index 000000000..14c68bd67 --- /dev/null +++ b/localtests/move-tables/resume-panic-before-on-success-hook/test.sh @@ -0,0 +1,149 @@ + +#!/bin/bash +# Custom test: +# - panic after drain (T4) and prior to on-success (T5), prior to cutover completion +# - validate RENAME and source writes are not possible +# - validate contents of source and target are the same +# - resume and complete the migration + +database=test +table_name=gh_ost_test + +# Build gh-ost command from scratch using framework function (required to inject failpoints) +rm $ghost_binary +build_binary + +# ensure hook files are executable +chmod +x $tests_path/$test_name/hooks/* + +# clean up any existing test hook files +rm -rf /tmp/gh-ost-hooks/ +mkdir -p /tmp/gh-ost-hooks/ + +###################################################################################################### +### Run #1: Should panic after drain (T4) and before on-success (T5) +###################################################################################################### + + +echo "⚙️ Starting migration with failpoint (run #1)..." + +# Build the gh-ost command using the framework function +GO_FAILPOINTS="github.com/github/gh-ost/go/base/move-tables-panic-before-on-success-hook=return(true)" build_ghost_command +cmd="$cmd --hooks-path=$tests_path/$test_name/hooks" + +# queue up removal of the postpone cutover flag, otherwise gh-ost hangs on the cutover +( + sleep 2; + echo "⏩ Sending unpostpone cutover" + rm $postpone_cutover_flag_file &> /dev/null; +) & + +# drive some concurrent writes to the table to exercise queue drain (T3/T4) +( + DATABASE=test script/move-tables/insert-source-primary-loop 100 0.1 10 &>/dev/null & + writes_pid=$! + sleep 3 + kill $writes_pid +) & + +# Run the gh-ost command, expecting panic on the failpoint the first time +echo_dot +echo > $test_logfile +bash -c "$cmd" >>$test_logfile 2>&1 +ghost_result=$? + +if [ $ghost_result -eq 0 ]; then + echo "ERROR: gh-ost should have failed but did not." + return 1 +fi + +echo -e "\n\n\n\n\n" + +###################################################################################################### +### Intermediate validation +###################################################################################################### + +echo "⚙️ Validating checkpointed state on unexpected exit..." + +# Table was renamed on source +mysql-exec source primary $database -sNe "SELECT 1 FROM ${table_name} LIMIT 1;" +if [ $? -eq 0 ]; then + echo "ERROR: Table '${table_name}' exists on source but show have been renamed." + return 1 +fi + +mysql-exec source primary $database -sNe "SELECT 1 FROM _${table_name}_del LIMIT 1;" +if [ $? -gt 0 ]; then + echo "ERROR: Renamed table '_${table_name}_del' does not exist on source." + return 1 +fi + +# Table not writeable on source +mysql-exec source primary $database -sNe "INSERT INTO ${table_name} VALUES (NULL, 1021, 2001, 2400001, 201, 1700000041, 1700000041);" +if [ $? -eq 0 ]; then + echo "ERROR: Table '${table_name}' was writeable on source but should not be!." + return 1 +fi + +# Table still exists on target +mysql-exec target primary $database -sNe "SELECT 1 FROM ${table_name} LIMIT 1;" +if [ $? -gt 0 ]; then + echo "ERROR: Table '${table_name}' does not exist on target." + return 1 +fi + +# contents of table on source and target are the same +source_contents_file=/tmp/gh-ost-test.resume-move-tables-panic-before-on-success-hook-source_contents.txt +target_contents_file=/tmp/gh-ost-test.resume-move-tables-panic-before-on-success-hook-target_contents.txt +mysql-exec source primary $database -sNe "SELECT * FROM _${table_name}_del;" > $source_contents_file +mysql-exec target primary $database -sNe "SELECT * FROM ${table_name};" > $target_contents_file + +if ! diff $source_contents_file $target_contents_file; then + echo "ERROR: Contents of table '${table_name}' are not the same on source and target." + echo "---- DIFF -----" + diff --side-by-side $source_contents_file $target_contents_file + echo "---------------" + return 1 +fi + +# validate on-success hook was not called +if [ -f /tmp/gh-ost-hooks/on-success ]; then + echo "ERROR: on-success hook was called when it should not have been." + return 1 +fi + +echo "✅ Validated checkpointed state on unexpected exit..." + +echo -e "\n\n\n\n\n" + +###################################################################################################### +### Run #2: Resume and complete the migration +###################################################################################################### + +echo "⚙️ Resuming migration (run #2)..." + +# resume migration +build_ghost_command +cmd="$cmd --resume --hooks-path=$tests_path/$test_name/hooks" + +# queue up removal of the postpone cutover flag, otherwise gh-ost hangs on the cutover +( + sleep 2; + echo "⏩ Sending unpostpone cutover" + rm $postpone_cutover_flag_file &> /dev/null; +) & + +bash -c "$cmd" >>$test_logfile 2>&1 +ghost_result=$? + +if [ $ghost_result -ne 0 ]; then + echo "ERROR: gh-ost should have succeeded but did not. ($ghost_result)" + return 1 +fi + +# validate on-success hook was was called +if [ ! -f /tmp/gh-ost-hooks/on-success ]; then + echo "ERROR: on-success hook was not called when it should have been." +fi + +echo -e "\n\n\n\n\n" diff --git a/localtests/move-tables/resume-panic-on-row-copy/create.sql b/localtests/move-tables/resume-panic-on-row-copy/create.sql new file mode 100644 index 000000000..46e919003 --- /dev/null +++ b/localtests/move-tables/resume-panic-on-row-copy/create.sql @@ -0,0 +1,34 @@ +drop table if exists gh_ost_test; +create table gh_ost_test ( + id bigint(20) NOT NULL AUTO_INCREMENT, + column1 int(11) NOT NULL, + column2 smallint(5) unsigned NOT NULL, + column3 mediumint(8) unsigned NOT NULL, + column4 tinyint(3) unsigned NOT NULL, + column5 int(11) NOT NULL, + column6 int(11) NOT NULL, + PRIMARY KEY (id), + KEY c12_ix (column1, column2) +) auto_increment=1; + +insert into gh_ost_test values + (NULL, 1001, 100, 500000, 10, 1700000001, 1700000002), + (NULL, 1002, 200, 600000, 20, 1700000003, 1700000004), + (NULL, 1003, 300, 700000, 30, 1700000005, 1700000006), + (NULL, 1004, 400, 800000, 40, 1700000007, 1700000008), + (NULL, 1005, 500, 900000, 50, 1700000009, 1700000010), + (NULL, 1006, 600, 1000000, 60, 1700000011, 1700000012), + (NULL, 1007, 700, 1100000, 70, 1700000013, 1700000014), + (NULL, 1008, 800, 1200000, 80, 1700000015, 1700000016), + (NULL, 1009, 900, 1300000, 90, 1700000017, 1700000018), + (NULL, 1010, 1000, 1400000, 100, 1700000019, 1700000020), + (NULL, 1011, 1100, 1500000, 110, 1700000021, 1700000022), + (NULL, 1012, 1200, 1600000, 120, 1700000023, 1700000024), + (NULL, 1013, 1300, 1700000, 130, 1700000025, 1700000026), + (NULL, 1014, 1400, 1800000, 140, 1700000027, 1700000028), + (NULL, 1015, 1500, 1900000, 150, 1700000029, 1700000030), + (NULL, 1016, 1600, 2000000, 160, 1700000031, 1700000032), + (NULL, 1017, 1700, 2100000, 170, 1700000033, 1700000034), + (NULL, 1018, 1800, 2200000, 180, 1700000035, 1700000036), + (NULL, 1019, 1900, 2300000, 190, 1700000037, 1700000038), + (NULL, 1020, 2000, 2400000, 200, 1700000039, 1700000040); \ No newline at end of file diff --git a/localtests/move-tables/resume-panic-on-row-copy/tables.txt b/localtests/move-tables/resume-panic-on-row-copy/tables.txt new file mode 100644 index 000000000..11fc5eef8 --- /dev/null +++ b/localtests/move-tables/resume-panic-on-row-copy/tables.txt @@ -0,0 +1 @@ +gh_ost_test diff --git a/localtests/move-tables/resume-panic-on-row-copy/test.sh b/localtests/move-tables/resume-panic-on-row-copy/test.sh new file mode 100644 index 000000000..583d8ea4b --- /dev/null +++ b/localtests/move-tables/resume-panic-on-row-copy/test.sh @@ -0,0 +1,122 @@ + +#!/bin/bash +# Custom test: +# - panic during row copy stage, prior to cutover +# - resume and complete the migration + +database=test +table_name=gh_ost_test + +# Build gh-ost command from scratch using framework function (required to inject failpoints) +rm $ghost_binary +build_binary + +###################################################################################################### +### Run #1: Should panic after first row copy and migration will not complete +###################################################################################################### + +echo "⚙️ Starting migration with failpoint (run #1)..." + +# Build the gh-ost command using the framework function +GO_FAILPOINTS="github.com/github/gh-ost/go/base/move-tables-panic-after-row-copy=return(true)" build_ghost_command + +# Run the gh-ost command, expecting panic on the failpoint the first time +echo_dot +echo > $test_logfile +bash -c "$cmd" >>$test_logfile 2>&1 +ghost_result=$? + +if [ $ghost_result -eq 0 ]; then + echo "ERROR: gh-ost should have failed but did not." + return 1 +fi + +echo -e "\n\n\n\n\n" + +###################################################################################################### +### Intermediate validation +###################################################################################################### + +echo "⚙️ Validating checkpointed state on unexpected exit..." + +# checkpoint table exists on target and is non-empty +checkpoint_table=$(mysql-exec target primary $database -sNe "SELECT table_name FROM information_schema.tables WHERE table_schema='${database}' AND table_name LIKE '\\_gho\\_%\\_ghk' LIMIT 1;") +if [ -z "$checkpoint_table" ]; then + echo "ERROR: Checkpoint table does not exist." + return 1 +fi +mysql-exec target primary $database -sNe "SELECT 1 FROM \`${checkpoint_table}\` LIMIT 1;" +if [ $? -gt 0 ]; then + echo "ERROR: Checkpoint table is empty or does not exist." + return 1 +fi + +# original table still exists on source +mysql-exec source replica $database -sNe "SELECT 1 FROM ${table_name} LIMIT 1;" +if [ $? -gt 0 ]; then + echo "ERROR: Table '${table_name}' does not exist on the source cluster." + return 1 +fi + +# original table exists on the target +mysql-exec target replica $database -sNe "SELECT 1 FROM ${table_name} LIMIT 1;" +if [ $? -gt 0 ]; then + echo "ERROR: Table '${table_name}' does not exist on the target cluster." + return 1 +fi + +# validate we processed a single row-copy chunk (10 rows) and there are 20 total to process +rows_copied=$(mysql-exec target primary $database -Ne "SELECT gh_ost_rows_copied FROM \`${checkpoint_table}\` ORDER BY gh_ost_chk_id DESC LIMIT 1;") +if [ $rows_copied -ne 10 ]; then + echo "ERROR: Expected last checkpoint to show 10 rows copied." + return 1 +fi + +echo "✅ Validating checkpointed state on unexpected exit..." + +echo -e "\n\n\n\n\n" + +###################################################################################################### +### Run #2: Resume and complete the migration +###################################################################################################### + +echo "⚙️ Resuming migration (run #2)..." + +# resume migration +build_ghost_command +cmd="$cmd --resume" + +# queue up removal of the postpone cutover flag, otherwise gh-ost hangs on the cutover +( + sleep 2; + echo "⏩ Sending unpostpone cutover" + rm $postpone_cutover_flag_file &> /dev/null; +) & + +bash -c "$cmd" >>$test_logfile 2>&1 +ghost_result=$? + +if [ $ghost_result -ne 0 ]; then + echo "ERROR: gh-ost should have succeeded but did not. ($ghost_result)" + return 1 +fi + +echo -e "\n\n\n\n\n" + +###################################################################################################### +### post-migration validation +###################################################################################################### + +echo "⚙️ Validating checkpointed state after resumed migration..." + +# validate we processed the rest of the 20 rows to copy +rows_copied=$(mysql-exec target primary $database -Ne "SELECT gh_ost_rows_copied FROM \`${checkpoint_table}\` ORDER BY gh_ost_chk_id DESC LIMIT 1;") +if [ $rows_copied -ne 20 ]; then + echo "ERROR: Expected last checkpoint to show 20 rows copied." + return 1 +fi + +echo "✅ Validating checkpointed state on resumed migration." + +echo -e "\n\n\n\n\n" + diff --git a/localtests/move-tables/single-concurrent-writes/create.sql b/localtests/move-tables/single-concurrent-writes/create.sql new file mode 100644 index 000000000..46e919003 --- /dev/null +++ b/localtests/move-tables/single-concurrent-writes/create.sql @@ -0,0 +1,34 @@ +drop table if exists gh_ost_test; +create table gh_ost_test ( + id bigint(20) NOT NULL AUTO_INCREMENT, + column1 int(11) NOT NULL, + column2 smallint(5) unsigned NOT NULL, + column3 mediumint(8) unsigned NOT NULL, + column4 tinyint(3) unsigned NOT NULL, + column5 int(11) NOT NULL, + column6 int(11) NOT NULL, + PRIMARY KEY (id), + KEY c12_ix (column1, column2) +) auto_increment=1; + +insert into gh_ost_test values + (NULL, 1001, 100, 500000, 10, 1700000001, 1700000002), + (NULL, 1002, 200, 600000, 20, 1700000003, 1700000004), + (NULL, 1003, 300, 700000, 30, 1700000005, 1700000006), + (NULL, 1004, 400, 800000, 40, 1700000007, 1700000008), + (NULL, 1005, 500, 900000, 50, 1700000009, 1700000010), + (NULL, 1006, 600, 1000000, 60, 1700000011, 1700000012), + (NULL, 1007, 700, 1100000, 70, 1700000013, 1700000014), + (NULL, 1008, 800, 1200000, 80, 1700000015, 1700000016), + (NULL, 1009, 900, 1300000, 90, 1700000017, 1700000018), + (NULL, 1010, 1000, 1400000, 100, 1700000019, 1700000020), + (NULL, 1011, 1100, 1500000, 110, 1700000021, 1700000022), + (NULL, 1012, 1200, 1600000, 120, 1700000023, 1700000024), + (NULL, 1013, 1300, 1700000, 130, 1700000025, 1700000026), + (NULL, 1014, 1400, 1800000, 140, 1700000027, 1700000028), + (NULL, 1015, 1500, 1900000, 150, 1700000029, 1700000030), + (NULL, 1016, 1600, 2000000, 160, 1700000031, 1700000032), + (NULL, 1017, 1700, 2100000, 170, 1700000033, 1700000034), + (NULL, 1018, 1800, 2200000, 180, 1700000035, 1700000036), + (NULL, 1019, 1900, 2300000, 190, 1700000037, 1700000038), + (NULL, 1020, 2000, 2400000, 200, 1700000039, 1700000040); \ No newline at end of file diff --git a/localtests/move-tables/single-concurrent-writes/on_test.sh b/localtests/move-tables/single-concurrent-writes/on_test.sh new file mode 100755 index 000000000..46849a09b --- /dev/null +++ b/localtests/move-tables/single-concurrent-writes/on_test.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +# insert data into the source primary, starting at ID 100 in batches of 10. kill +# the process after 5 seconds +DATABASE=test script/move-tables/insert-source-primary-loop 100 0.01 100 & +sleep 5 && kill $! diff --git a/localtests/move-tables/single-concurrent-writes/tables.txt b/localtests/move-tables/single-concurrent-writes/tables.txt new file mode 100644 index 000000000..11fc5eef8 --- /dev/null +++ b/localtests/move-tables/single-concurrent-writes/tables.txt @@ -0,0 +1 @@ +gh_ost_test diff --git a/localtests/move-tables/single-with-hooks/create.sql b/localtests/move-tables/single-with-hooks/create.sql new file mode 100644 index 000000000..46e919003 --- /dev/null +++ b/localtests/move-tables/single-with-hooks/create.sql @@ -0,0 +1,34 @@ +drop table if exists gh_ost_test; +create table gh_ost_test ( + id bigint(20) NOT NULL AUTO_INCREMENT, + column1 int(11) NOT NULL, + column2 smallint(5) unsigned NOT NULL, + column3 mediumint(8) unsigned NOT NULL, + column4 tinyint(3) unsigned NOT NULL, + column5 int(11) NOT NULL, + column6 int(11) NOT NULL, + PRIMARY KEY (id), + KEY c12_ix (column1, column2) +) auto_increment=1; + +insert into gh_ost_test values + (NULL, 1001, 100, 500000, 10, 1700000001, 1700000002), + (NULL, 1002, 200, 600000, 20, 1700000003, 1700000004), + (NULL, 1003, 300, 700000, 30, 1700000005, 1700000006), + (NULL, 1004, 400, 800000, 40, 1700000007, 1700000008), + (NULL, 1005, 500, 900000, 50, 1700000009, 1700000010), + (NULL, 1006, 600, 1000000, 60, 1700000011, 1700000012), + (NULL, 1007, 700, 1100000, 70, 1700000013, 1700000014), + (NULL, 1008, 800, 1200000, 80, 1700000015, 1700000016), + (NULL, 1009, 900, 1300000, 90, 1700000017, 1700000018), + (NULL, 1010, 1000, 1400000, 100, 1700000019, 1700000020), + (NULL, 1011, 1100, 1500000, 110, 1700000021, 1700000022), + (NULL, 1012, 1200, 1600000, 120, 1700000023, 1700000024), + (NULL, 1013, 1300, 1700000, 130, 1700000025, 1700000026), + (NULL, 1014, 1400, 1800000, 140, 1700000027, 1700000028), + (NULL, 1015, 1500, 1900000, 150, 1700000029, 1700000030), + (NULL, 1016, 1600, 2000000, 160, 1700000031, 1700000032), + (NULL, 1017, 1700, 2100000, 170, 1700000033, 1700000034), + (NULL, 1018, 1800, 2200000, 180, 1700000035, 1700000036), + (NULL, 1019, 1900, 2300000, 190, 1700000037, 1700000038), + (NULL, 1020, 2000, 2400000, 200, 1700000039, 1700000040); \ No newline at end of file diff --git a/localtests/move-tables/single-with-hooks/hooks/gh-ost-on-before-cut-over b/localtests/move-tables/single-with-hooks/hooks/gh-ost-on-before-cut-over new file mode 100755 index 000000000..5c7a5b872 --- /dev/null +++ b/localtests/move-tables/single-with-hooks/hooks/gh-ost-on-before-cut-over @@ -0,0 +1,14 @@ +#!/bin/bash + +repo_root=$(git rev-parse --show-toplevel) +source $repo_root/localtests/move-tables/single-with-hooks/hooks/util.sh + +# dump environment variables on dirty exit +trap '[[ $? -eq 0 ]] || dump_env' EXIT + +set -e + +assert_common_envs + +# touch file to mark completion of on-before-cut-over hook +touch /tmp/gh-ost-hooks/on-before-cut-over diff --git a/localtests/move-tables/single-with-hooks/hooks/gh-ost-on-row-copy-complete b/localtests/move-tables/single-with-hooks/hooks/gh-ost-on-row-copy-complete new file mode 100755 index 000000000..0b2ea7d4c --- /dev/null +++ b/localtests/move-tables/single-with-hooks/hooks/gh-ost-on-row-copy-complete @@ -0,0 +1,14 @@ +#!/bin/bash + +repo_root=$(git rev-parse --show-toplevel) +source $repo_root/localtests/move-tables/single-with-hooks/hooks/util.sh + +# dump environment variables on dirty exit +trap '[[ $? -eq 0 ]] || dump_env' EXIT + +set -e + +assert_common_envs + +# touch file to mark completion of on-row-copy-complete hook +touch /tmp/gh-ost-hooks/on-row-copy-complete \ No newline at end of file diff --git a/localtests/move-tables/single-with-hooks/hooks/gh-ost-on-success b/localtests/move-tables/single-with-hooks/hooks/gh-ost-on-success new file mode 100755 index 000000000..2397c8aaa --- /dev/null +++ b/localtests/move-tables/single-with-hooks/hooks/gh-ost-on-success @@ -0,0 +1,16 @@ +#!/bin/bash + +repo_root=$(git rev-parse --show-toplevel) +source $repo_root/localtests/move-tables/single-with-hooks/hooks/util.sh + +# dump environment variables on dirty exit +trap '[[ $? -eq 0 ]] || dump_env' EXIT + +set -e + +assert_common_envs + +assert_env_present "GH_OST_DRAIN_GTID" + +# touch file to mark completion of on-success hook +touch /tmp/gh-ost-hooks/on-success \ No newline at end of file diff --git a/localtests/move-tables/single-with-hooks/hooks/util.sh b/localtests/move-tables/single-with-hooks/hooks/util.sh new file mode 100755 index 000000000..d5820acfc --- /dev/null +++ b/localtests/move-tables/single-with-hooks/hooks/util.sh @@ -0,0 +1,42 @@ +#/bin/bash + +assert_env_equal() { + env_name=$1 + expected=$2 + + if [ "${!env_name}" != "${expected}" ]; then + echo "ERROR: Expected '${expected}' for ${env_name}, but got '${!env_name}'" + exit 1 + fi +} + +assert_env_present() { + env_name=$1 + + echo "checking '${env_name}=${!env_name}'" + if [[ -z "${!env_name}" ]]; then + echo "ERROR: Expected '${env_name}' to be set but not present" + exit 1 + fi +} + +assert_common_envs() { + assert_env_present "GH_OST_TARGET_HOST" + + assert_env_equal "GH_OST_TARGET_DATABASE_NAME" "test" + assert_env_equal "GH_OST_TABLE_NAME" "gh_ost_test" + assert_env_equal "GH_OST_OLD_TABLE_NAME" "_gh_ost_test_del" + assert_env_equal "GH_OST_TARGET_TABLE_NAME" "gh_ost_test" + assert_env_equal "GH_OST_MOVE_TABLES" "true" + assert_env_equal "GH_OST_REVERT" "false" +} + +dump_env() { + echo "-----------------------------------------------------" + echo "----------------- ENVIRONS --------------------------" + echo "-----------------------------------------------------" + env | grep "GH_OST_" + echo "-----------------------------------------------------" + echo "-----------------------------------------------------" + echo "-----------------------------------------------------" +} \ No newline at end of file diff --git a/localtests/move-tables/single-with-hooks/tables.txt b/localtests/move-tables/single-with-hooks/tables.txt new file mode 100644 index 000000000..11fc5eef8 --- /dev/null +++ b/localtests/move-tables/single-with-hooks/tables.txt @@ -0,0 +1 @@ +gh_ost_test diff --git a/localtests/move-tables/single-with-hooks/test.sh b/localtests/move-tables/single-with-hooks/test.sh new file mode 100644 index 000000000..73fbb7771 --- /dev/null +++ b/localtests/move-tables/single-with-hooks/test.sh @@ -0,0 +1,67 @@ + +#!/bin/bash +# Custom test: +# Executes migration with custom hooks (on-row-copy-complete, on-before-cut-over, on-success) +# which are executed at different stages of the migration and validate the environment variables +# expected to be available to the respective hooks. + +database=test +table_name=gh_ost_test + +# Build gh-ost command from scratch using framework function +build_binary + +###################################################################################################### +### Run gh-ost with custom hooks neabled +###################################################################################################### + +echo "⚙️ Running gh-ost with custom hooks..." + +# ensure hook files are executable +chmod +x $tests_path/$test_name/hooks/* + +# clean up any existing test hook files +rm -rf /tmp/gh-ost-hooks/ +mkdir -p /tmp/gh-ost-hooks/ + +# Build the gh-ost command using the framework function +build_ghost_command +cmd="$cmd --hooks-path=$tests_path/$test_name/hooks" + +# queue up removal of the postpone cutover flag, otherwise gh-ost hangs on the cutover +( + sleep 2; + echo "⏩ Sending unpostpone cutover" + rm $postpone_cutover_flag_file &> /dev/null; +) & + +# Run the gh-ost command +echo_dot +echo > $test_logfile +bash -c "$cmd" >>$test_logfile 2>&1 +ghost_result=$? + +if [ $ghost_result -ne 0 ]; then + echo "ERROR: gh-ost failed unexpectedly." + return 1 +fi + +echo "✅ gh-ost move-tables succeeded!" + +echo -e "\n\n\n\n\n" + + +###################################################################################################### +### Validate hook status +###################################################################################################### + +echo "⚙️ Validating hook status after execution..." + +for expected in on-row-copy-complete on-before-cut-over on-success; do + if [ ! -f "/tmp/gh-ost-hooks/$expected" ]; then + echo "ERROR: Expected test hook file '/tmp/gh-ost-hooks/$expected' was not found." + return 1 + fi +done + +echo "✅ Hook status validated successfully." diff --git a/localtests/move-tables/single/create.sql b/localtests/move-tables/single/create.sql new file mode 100644 index 000000000..46e919003 --- /dev/null +++ b/localtests/move-tables/single/create.sql @@ -0,0 +1,34 @@ +drop table if exists gh_ost_test; +create table gh_ost_test ( + id bigint(20) NOT NULL AUTO_INCREMENT, + column1 int(11) NOT NULL, + column2 smallint(5) unsigned NOT NULL, + column3 mediumint(8) unsigned NOT NULL, + column4 tinyint(3) unsigned NOT NULL, + column5 int(11) NOT NULL, + column6 int(11) NOT NULL, + PRIMARY KEY (id), + KEY c12_ix (column1, column2) +) auto_increment=1; + +insert into gh_ost_test values + (NULL, 1001, 100, 500000, 10, 1700000001, 1700000002), + (NULL, 1002, 200, 600000, 20, 1700000003, 1700000004), + (NULL, 1003, 300, 700000, 30, 1700000005, 1700000006), + (NULL, 1004, 400, 800000, 40, 1700000007, 1700000008), + (NULL, 1005, 500, 900000, 50, 1700000009, 1700000010), + (NULL, 1006, 600, 1000000, 60, 1700000011, 1700000012), + (NULL, 1007, 700, 1100000, 70, 1700000013, 1700000014), + (NULL, 1008, 800, 1200000, 80, 1700000015, 1700000016), + (NULL, 1009, 900, 1300000, 90, 1700000017, 1700000018), + (NULL, 1010, 1000, 1400000, 100, 1700000019, 1700000020), + (NULL, 1011, 1100, 1500000, 110, 1700000021, 1700000022), + (NULL, 1012, 1200, 1600000, 120, 1700000023, 1700000024), + (NULL, 1013, 1300, 1700000, 130, 1700000025, 1700000026), + (NULL, 1014, 1400, 1800000, 140, 1700000027, 1700000028), + (NULL, 1015, 1500, 1900000, 150, 1700000029, 1700000030), + (NULL, 1016, 1600, 2000000, 160, 1700000031, 1700000032), + (NULL, 1017, 1700, 2100000, 170, 1700000033, 1700000034), + (NULL, 1018, 1800, 2200000, 180, 1700000035, 1700000036), + (NULL, 1019, 1900, 2300000, 190, 1700000037, 1700000038), + (NULL, 1020, 2000, 2400000, 200, 1700000039, 1700000040); \ No newline at end of file diff --git a/localtests/move-tables/single/tables.txt b/localtests/move-tables/single/tables.txt new file mode 100644 index 000000000..11fc5eef8 --- /dev/null +++ b/localtests/move-tables/single/tables.txt @@ -0,0 +1 @@ +gh_ost_test diff --git a/localtests/move-tables/unsigned/create.sql b/localtests/move-tables/unsigned/create.sql new file mode 100644 index 000000000..2c809d7ca --- /dev/null +++ b/localtests/move-tables/unsigned/create.sql @@ -0,0 +1,32 @@ +drop table if exists gh_ost_test; +create table gh_ost_test ( + id int auto_increment, + signed_value bigint not null, + unsigned_value int unsigned not null, + unsigned_big_value bigint unsigned not null, + primary key(id) +) auto_increment=1; + +insert into gh_ost_test (signed_value, unsigned_value, unsigned_big_value) values + (-9223372036854775807, 4294967295, 18446744073709551615), + (-9223372036854775806, 4294967294, 18446744073709551614), + (-9223372036854775805, 4294967293, 18446744073709551613); + +drop event if exists gh_ost_test; +delimiter ;; +create event gh_ost_test + on schedule every 1 second + starts current_timestamp + ends current_timestamp + interval 60 second + on completion not preserve + enable + do +begin + insert into gh_ost_test (signed_value, unsigned_value, unsigned_big_value) values + (-9223372036854775804, 4294967292, 18446744073709551612); + update gh_ost_test + set signed_value=signed_value+1, + unsigned_value=unsigned_value-1, + unsigned_big_value=unsigned_big_value-1 + where id <= 3; +end ;; \ No newline at end of file diff --git a/localtests/move-tables/unsigned/tables.txt b/localtests/move-tables/unsigned/tables.txt new file mode 100644 index 000000000..11fc5eef8 --- /dev/null +++ b/localtests/move-tables/unsigned/tables.txt @@ -0,0 +1 @@ +gh_ost_test diff --git a/localtests/test.sh b/localtests/test.sh index dfc9bee6a..dd9ae722d 100755 --- a/localtests/test.sh +++ b/localtests/test.sh @@ -522,7 +522,7 @@ build_binary() { test_all() { build_binary - test_dirs=$(find "$tests_path" -mindepth 1 -maxdepth 1 ! -path . -type d | grep "$test_pattern" | sort) + test_dirs=$(find "$tests_path" -mindepth 1 -maxdepth 1 ! -path . -type d | grep "$test_pattern" | grep -v move-tables | sort) # Read the test list on FD 3, not stdin: the mysql wrappers may run # `docker exec -i`, which attaches and drains stdin. On stdin (FD 0) the # first such call inside the loop would swallow the remaining test-dir diff --git a/script/docker-gh-ost-move-tables-tests b/script/docker-gh-ost-move-tables-tests new file mode 100755 index 000000000..4e7f09c6a --- /dev/null +++ b/script/docker-gh-ost-move-tables-tests @@ -0,0 +1,126 @@ +#!/bin/bash + +# This script starts four MySQL docker containers in a primary-replica setup +# for two distinct clusters which can be used to run move-tables tests +# in localtests/ . +# Set the environment var TEST_MYSQL_IMAGE to change the docker image. +# +# Usage: +# docker-gh-ost-move-tables-tests up [-t] start the containers +# docker-gh-ost-move-tables-tests down remove the containers +# docker-gh-ost-move-tables-tests run [-t] run move-tables tests on the containers + +set -e + +GH_OST_ROOT=$(git rev-parse --show-toplevel) +if [[ ":$PATH:" != *":$GH_OST_ROOT:"* ]]; then + export PATH="${PATH}:${GH_OST_ROOT}/script" +fi +SCRIPT_PATH="${GH_OST_ROOT}/script/move-tables" + +poll_mysql() { + cluster=$1 + role=$2 + + echo "polling $role for $cluster..." + + CTR=0 + while ! mysql-exec "$cluster" "$role" -e "select 1;" >/dev/null 2>&1; do + sleep 1 + CTR=$((CTR + 1)) + if [ $CTR -gt 30 ]; then + echo " ❌ MySQL $cluster/$role failed to start" + return 1 + fi + done + echo " ✔ MySQL $cluster/$role OK" + return 0 +} + +mysql-exec() { + cluster=$1 + role=$2 + shift 2 + + if [[ $TEST_MYSQL_IMAGE =~ "mysql:8.4" ]]; then + $SCRIPT_PATH/mysql-$cluster-$role --ssl-mode=required "$@" + else + $SCRIPT_PATH/mysql-$cluster-$role "$@" + fi +} + +setup() { + [ -z "$TEST_MYSQL_IMAGE" ] && TEST_MYSQL_IMAGE="mysql:8.0.41" + + echo "Starting MySQL $TEST_MYSQL_IMAGE containers..." + compose_file="$GH_OST_ROOT/localtests/docker-compose-move-tables.yml" + MYSQL_SHA2_RSA_KEYS_FLAG="" + MYSQL_PASSWORD_HASHING_ALGORITHM="mysql_native_password" + if [[ $TEST_MYSQL_IMAGE =~ "mysql:8.4" ]]; then + MYSQL_PASSWORD_HASHING_ALGORITHM="caching_sha2_password" + MYSQL_SHA2_RSA_KEYS_FLAG="--caching-sha2-password-auto-generate-rsa-keys=ON" + fi + (TEST_MYSQL_IMAGE="$TEST_MYSQL_IMAGE" MYSQL_SHA2_RSA_KEYS_FLAG="$MYSQL_SHA2_RSA_KEYS_FLAG" envsubst <"$compose_file") >"$compose_file.tmp" + + docker compose -f "$compose_file.tmp" up -d --wait + + for cluster in "source" "target"; do + echo "-----------------------------------" + echo "--- Setting up $cluster cluster ---" + echo "-----------------------------------" + + echo "Waiting for MySQL..." + poll_mysql "$cluster" "primary" || exit 1 + poll_mysql "$cluster" "replica" || exit 1 + + echo -n "Setting up replication..." + mysql-exec "$cluster" "primary" -e "create user if not exists 'repl'@'%' identified with $MYSQL_PASSWORD_HASHING_ALGORITHM by 'repl';" + mysql-exec "$cluster" "primary" -e "grant replication slave on *.* to 'repl'@'%'; flush privileges;" + mysql-exec "$cluster" "primary" -e "create user if not exists 'gh-ost'@'%' identified with $MYSQL_PASSWORD_HASHING_ALGORITHM by 'gh-ost';" + mysql-exec "$cluster" "primary" -e "grant all on *.* to 'gh-ost'@'%';" + + primary_port=3307 + if [ "$cluster" = "target" ]; then + primary_port=3309 + fi + + sleep 1 + if [[ $TEST_MYSQL_IMAGE =~ "mysql:8.4" ]]; then + mysql-exec "$cluster" "replica" -e "change replication source to source_host='mysql-$cluster-primary', source_port=$primary_port, source_user='repl', source_password='repl', source_auto_position=1, source_ssl=1;" + mysql-exec "$cluster" "replica" -e "start replica;" + else + mysql-exec "$cluster" "replica" -e "change master to master_host='mysql-$cluster-primary', master_port=$primary_port, master_user='repl', master_password='repl', master_auto_position=1;" + mysql-exec "$cluster" "replica" -e "start slave;" + fi + echo "OK" + done +} + +teardown() { + echo "Tearing down..." + compose_file="$GH_OST_ROOT/localtests/docker-compose-move-tables.yml" + docker compose -f "$compose_file.tmp" down --volumes --remove-orphans +} + +main() { + local cmd="$1" + if [[ "$cmd" == "up" ]]; then + setup + elif [[ "$cmd" == "down" ]]; then + teardown + elif [[ "$cmd" == "run" ]]; then + shift 1 + + if [ "$1" == "--rm" ]; then + # run teardown, setup,a nd run for convenience + echo "⚙️ Bootstrapping environment from scratch..." + teardown + setup + shift 1 + fi + + "$GH_OST_ROOT/localtests/move-tables-test.sh" -d "$@" + fi +} + +main "$@" diff --git a/script/move-tables/README.md b/script/move-tables/README.md new file mode 100644 index 000000000..398957821 --- /dev/null +++ b/script/move-tables/README.md @@ -0,0 +1,116 @@ +### Setup + +Setup the multi-cluster topology and seed the data. This always seeds the same +canonical **three** tables on the source — `gh_ost_test`, `gh_ost_test_other`, +and `gh_ost_test_third` (see `localtests/move-tables/multiple-three/create.sql`) — into +the `test` database. You then choose how many of them to move via `--move-tables`, +so `setup`/`reset`/`teardown` behave identically regardless of which scenario you +run. +```bash +script/move-tables/setup +``` + +Verify data is present in the source cluster. +```bash +script/move-tables/mysql-source-primary -D test -e "SELECT * FROM gh_ost_test; SELECT * FROM gh_ost_test_other; SELECT * FROM gh_ost_test_third;" +``` + +Verify the empty database is present in the target cluster. +```bash +script/move-tables/mysql-target-primary -D test -e "SHOW TABLES;" +``` + +### Testing `gh-ost` + +Checkout your branch of `github/gh-ost` and build the binaries: +```bash +script/build --cli +``` + +Run gh-ost to move tables. Pick **one**, **two**, or **three** tables by changing +the `--move-tables` list — everything else stays the same: +```bash +# one table +./bin/gh-ost --move-tables=gh_ost_test --host=localhost --port=3308 --user root --password opensesame --database=test --target-host=localhost --target-port=3309 --target-user root --target-password opensesame --target-database=test --postpone-cut-over-flag-file=/tmp/ghost-move-tables.postpone.flag --execute --verbose --checkpoint --checkpoint-seconds 10 --initially-drop-socket-file + +# two tables +./bin/gh-ost --move-tables=gh_ost_test,gh_ost_test_other ... (same flags) + +# three tables +./bin/gh-ost --move-tables=gh_ost_test,gh_ost_test_other,gh_ost_test_third ... (same flags) +``` + +You'll see per-table row-copy progress in the status output, with all moved +tables advancing concurrently. + +Start continuous inserts against the source. No arguments required: it detects +which of the three fixtures exist and writes to all of them. +```bash +script/move-tables/insert-source-primary-loop +``` + +Check the target - it should have the initial data from the source and should be receiving the new data. +```bash +script/move-tables/mysql-target-primary -D test -e "SELECT * FROM gh_ost_test;" +``` + +Remove the cutover flag file. +```bash +rm /tmp/ghost-move-tables.postpone.flag +``` + +The continuous inserts stop because the moved tables are renamed. When you move +multiple tables, they are all renamed together in a single atomic `RENAME TABLE`. + +Check the source - each moved table has been renamed to its `_
_del` +rollback handle (only the tables you moved are renamed): +```bash +script/move-tables/mysql-source-primary -D test -e "SELECT * FROM _gh_ost_test_del;" +``` + +Check the target has the same set of data. +```bash +script/move-tables/mysql-target-primary -D test -e "SELECT * FROM gh_ost_test;" +``` + +### Resetting between runs + +Drop and re-seed all three source tables (and clean up the moved target tables + +checkpoint table) so you can run again without a full teardown. It works the same +no matter how many tables you just moved: +```bash +script/move-tables/reset +``` + +### Teardown + +Remove the docker containers: +```bash +script/move-tables/teardown +``` + +### CI integration tests + +The same fixtures back the CI integration tests, run via +`localtests/move-tables-test.sh [filter]`. Each test directory under +`localtests/move-tables/` is self-contained (its own `create.sql` + `tables.txt`, +plus an optional `on_test.sh` for concurrent workload or `test.sh` for a fully +custom scenario): + +- `single` — moves 1 table, idle source +- `single-concurrent-writes` — moves 1 table with sustained DML during copy +- `single-with-hooks` — moves 1 table and asserts the hook env vars +- `multiple-two` — moves 2 tables, idle source +- `multiple-three` — moves 3 tables, idle source +- `multiple-three-concurrent-writes` — moves 3 tables with sustained DML on all three +- `atomic-multi-table-cutover` — moves 2 tables while committing cross-table + transactions up to cutover; asserts the atomic multi-table RENAME leaves no + orphaned rows across the pair +- `resume-panic-on-row-copy`, `resume-panic-before-drain-complete`, + `resume-panic-before-on-success-hook` — crash mid-run via a failpoint, then + `--resume` to completion + +Run a single scenario by name, e.g.: +```bash +localtests/move-tables-test.sh multiple-three +``` \ No newline at end of file diff --git a/script/move-tables/insert-source-primary-loop b/script/move-tables/insert-source-primary-loop new file mode 100755 index 000000000..eebf43e73 --- /dev/null +++ b/script/move-tables/insert-source-primary-loop @@ -0,0 +1,117 @@ +#!/usr/bin/env bash +set -uo pipefail + +# Continuously insert new rows into whichever of the canonical move-tables +# fixtures currently exist on the source primary (gh_ost_test, gh_ost_test_other, +# gh_ost_test_third), so a move-tables run sees live DML on every migrated table. +# +# No arguments are required. Existing tables are detected at startup; the loop +# only writes to the ones that are present, so it works for 1-, 2-, or 3-table +# runs without changes. When a cutover renames the tables away, the next insert +# fails and the loop stops cleanly (this is expected). +# +# Usage: +# script/move-tables/insert-source-primary-loop [start_id] [sleep_seconds] [rows_per_batch] +# Example: +# script/move-tables/insert-source-primary-loop +# Fast example: +# script/move-tables/insert-source-primary-loop 100000 0 50 + +start_i="${1:-100000}" +delay="${2:-0.2}" +rows_per_batch="${3:-1}" +i="$start_i" +# Match the database created/seeded by script/move-tables/setup. +DATABASE="${DATABASE:-test}" + +GH_OST_ROOT="$(git rev-parse --show-toplevel)" +SCRIPT_PATH="${GH_OST_ROOT}/script/move-tables" + +# The canonical superset of move-tables fixtures. +ALL_TABLES=(gh_ost_test gh_ost_test_other gh_ost_test_third) + +table_exists() { + local table="$1" + local count + count="$(${SCRIPT_PATH}/mysql-source-primary -N -s -D "$DATABASE" -e \ + "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema='${DATABASE}' AND table_name='${table}'" 2>/dev/null || echo 0)" + [[ "$count" == "1" ]] +} + +# Detect which fixtures exist so we only write to tables that are present. +active_tables=() +for t in "${ALL_TABLES[@]}"; do + if table_exists "$t"; then + active_tables+=("$t") + fi +done + +if [[ ${#active_tables[@]} -eq 0 ]]; then + echo "No move-tables fixtures found in database '${DATABASE}'. Did you run script/move-tables/setup?" + exit 1 +fi + +echo "Starting continuous inserts on source primary. Press Ctrl+C to stop." +echo "start_id=$start_i sleep_seconds=$delay rows_per_batch=$rows_per_batch database=$DATABASE" +echo "inserting into: ${active_tables[*]}" + +trap 'echo; echo "Stopped."; exit 0' INT TERM + +while true; do + ts="$(date +%s)" + declare -A values=() + batch_start="$i" + + for ((n = 0; n < rows_per_batch; n++)); do + current_i=$((i + n)) + for t in "${active_tables[@]}"; do + case "$t" in + gh_ost_test) + # id is AUTO_INCREMENT, so we supply column1..column6 only. + row="($current_i, $((current_i % 65535)), $((current_i % 16777215)), $((current_i % 255)), $ts, $((ts + 1)))" + ;; + gh_ost_test_other) + # uid is an explicit (non-auto) PK and name is UNIQUE; derive both from + # current_i to stay collision-free with the seed data. + row="($current_i, 'row_$current_i', $((current_i % 100000)).50, FROM_UNIXTIME($ts))" + ;; + gh_ost_test_third) + # code is a varchar PK; updated_at uses its column default. + row="('code_$current_i', 'label_$current_i', $((current_i % 1000)).5)" + ;; + esac + if [[ -n "${values[$t]:-}" ]]; then + values[$t]+=" , " + fi + values[$t]+="$row" + done + done + + query="" + for t in "${active_tables[@]}"; do + case "$t" in + gh_ost_test) + query+="INSERT INTO gh_ost_test (column1, column2, column3, column4, column5, column6) VALUES ${values[$t]};"$'\n' + ;; + gh_ost_test_other) + query+="INSERT INTO gh_ost_test_other (uid, name, amount, created_at) VALUES ${values[$t]};"$'\n' + ;; + gh_ost_test_third) + query+="INSERT INTO gh_ost_test_third (code, label, score) VALUES ${values[$t]};"$'\n' + ;; + esac + done + + if ! ${SCRIPT_PATH}/mysql-source-primary -D "$DATABASE" -e "$query"; then + echo + echo "Insert failed (the tables were likely renamed at cutover). Stopping." + exit 0 + fi + + i=$((i + rows_per_batch)) + echo "inserted rows: id=${batch_start}..$((i - 1)) ts=$ts (${active_tables[*]})" + + if [[ "$delay" != "0" && "$delay" != "0.0" ]]; then + sleep "$delay" + fi +done diff --git a/script/move-tables/mysql-source-primary b/script/move-tables/mysql-source-primary new file mode 100755 index 000000000..9643924d7 --- /dev/null +++ b/script/move-tables/mysql-source-primary @@ -0,0 +1,6 @@ +#!/bin/bash +# +# This executes a command on the mysql-source-primary docker container created +# from localtests/docker-compose-move-tables.yml. + +MYSQL_PWD=opensesame mysql -uroot -h0.0.0.0 -P3307 "$@" \ No newline at end of file diff --git a/script/move-tables/mysql-source-replica b/script/move-tables/mysql-source-replica new file mode 100755 index 000000000..f42a698d4 --- /dev/null +++ b/script/move-tables/mysql-source-replica @@ -0,0 +1,6 @@ +#!/bin/bash +# +# This executes a command on the mysql-source-replica docker container created +# from localtests/docker-compose-move-tables.yml. + +MYSQL_PWD=opensesame mysql -uroot -h0.0.0.0 -P3308 "$@" \ No newline at end of file diff --git a/script/move-tables/mysql-target-primary b/script/move-tables/mysql-target-primary new file mode 100755 index 000000000..9a229231c --- /dev/null +++ b/script/move-tables/mysql-target-primary @@ -0,0 +1,6 @@ +#!/bin/bash +# +# This executes a command on the mysql-target-primary docker container created +# from localtests/docker-compose-move-tables.yml. + +MYSQL_PWD=opensesame mysql -uroot -h0.0.0.0 -P3309 "$@" \ No newline at end of file diff --git a/script/move-tables/mysql-target-replica b/script/move-tables/mysql-target-replica new file mode 100755 index 000000000..ae43d4779 --- /dev/null +++ b/script/move-tables/mysql-target-replica @@ -0,0 +1,6 @@ +#!/bin/bash +# +# This executes a command on the mysql-target-replica docker container created +# from localtests/docker-compose-move-tables.yml. + +MYSQL_PWD=opensesame mysql -uroot -h0.0.0.0 -P3310 "$@" \ No newline at end of file diff --git a/script/move-tables/reset b/script/move-tables/reset new file mode 100755 index 000000000..b5835a145 --- /dev/null +++ b/script/move-tables/reset @@ -0,0 +1,45 @@ +#!/bin/bash + +set -euo pipefail + +GH_OST_ROOT=$(git rev-parse --show-toplevel) +SCRIPT_PATH="${GH_OST_ROOT}/script/move-tables" +# Match the database that `setup` creates and seeds (override with GH_OST_TEST_DB). +DATABASE_NAME="${GH_OST_TEST_DB:-test}" + +# The canonical superset of tables seeded by setup (and listed in +# localtests/move-tables/multiple-three/tables.txt). The manual harness always +# sets up / cleans up all of them, regardless of how many you actually move, +# so reset works the same no matter which scenario you just ran. +TABLES=(gh_ost_test gh_ost_test_other gh_ost_test_third) + +# Reset source table state regardless of whether a cutover renamed the originals +# to their `_
_del` rollback handles. +source_drop="" +for t in "${TABLES[@]}"; do + source_drop+="_${t}_del, ${t}, " +done +source_drop="${source_drop%, }" +${SCRIPT_PATH}/mysql-source-primary -D "${DATABASE_NAME}" -e "DROP TABLE IF EXISTS ${source_drop};" + +# Recreate and seed source table data, same fixture as setup uses. +${SCRIPT_PATH}/mysql-source-primary -D "${DATABASE_NAME}" < "${GH_OST_ROOT}/localtests/move-tables/multiple-three/create.sql" + +# Drop the moved tables on the target cluster. +target_drop="" +for t in "${TABLES[@]}"; do + target_drop+="${t}, " +done +target_drop="${target_drop%, }" +${SCRIPT_PATH}/mysql-target-primary -D "${DATABASE_NAME}" -e "DROP TABLE IF EXISTS ${target_drop};" + +# The checkpoint table is named from the run token (_gho__ghk), which +# depends on the exact set of moved tables, so we can't name it statically. Drop +# any checkpoint tables that exist for this database. +checkpoint_tables=$(${SCRIPT_PATH}/mysql-target-primary -N -B -D "${DATABASE_NAME}" -e \ + "SELECT GROUP_CONCAT(CONCAT('\`', table_name, '\`')) FROM information_schema.tables WHERE table_schema='${DATABASE_NAME}' AND table_name LIKE '\\_gho\\_%\\_ghk';") +if [[ -n "${checkpoint_tables}" && "${checkpoint_tables}" != "NULL" ]]; then + ${SCRIPT_PATH}/mysql-target-primary -D "${DATABASE_NAME}" -e "DROP TABLE IF EXISTS ${checkpoint_tables};" +fi + +echo "Reset source and target tables (${TABLES[*]}) in ${DATABASE_NAME}" \ No newline at end of file diff --git a/script/move-tables/setup b/script/move-tables/setup new file mode 100755 index 000000000..df7644c3c --- /dev/null +++ b/script/move-tables/setup @@ -0,0 +1,162 @@ +#!/bin/bash + +# This script starts four MySQL docker containers in two primary-replica clusters +# which can be used for running tablemove replica tests. +# Set the environment var TEST_MYSQL_IMAGE to change the docker image. +# +# Usage: +# setup start the containers + +set -e + +GH_OST_ROOT=$(git rev-parse --show-toplevel) +SCRIPT_PATH="${GH_OST_ROOT}/script/move-tables" +DATABASE_NAME="test" + +poll_mysql() { + CTR=0 + cmd="exec-mysql-$1" + while ! $cmd -e "select 1;" >/dev/null 2>&1; do + sleep 1 + CTR=$((CTR + 1)) + if [ $CTR -gt 30 ]; then + echo " ❌ MySQL $1 failed to start" + return 1 + fi + done + echo " ✔ MySQL $1 OK" + return 0 +} + +exec-mysql-source-primary() { + if [[ $TEST_MYSQL_IMAGE =~ "mysql:8.4" ]]; then + ${SCRIPT_PATH}/mysql-source-primary --ssl-mode=required "$@" + else + ${SCRIPT_PATH}/mysql-source-primary "$@" + fi +} + +exec-mysql-source-replica() { + if [[ $TEST_MYSQL_IMAGE =~ "mysql:8.4" ]]; then + ${SCRIPT_PATH}/mysql-source-replica --ssl-mode=required "$@" + else + ${SCRIPT_PATH}/mysql-source-replica "$@" + fi +} + +exec-mysql-target-primary() { + if [[ $TEST_MYSQL_IMAGE =~ "mysql:8.4" ]]; then + ${SCRIPT_PATH}/mysql-target-primary --ssl-mode=required "$@" + else + ${SCRIPT_PATH}/mysql-target-primary "$@" + fi +} + +exec-mysql-target-replica() { + if [[ $TEST_MYSQL_IMAGE =~ "mysql:8.4" ]]; then + ${SCRIPT_PATH}/mysql-target-replica --ssl-mode=required "$@" + else + ${SCRIPT_PATH}/mysql-target-replica "$@" + fi +} + +# gh-ost runs on the host, outside the docker network, but source replication +# advertises the primary by its compose service name (mysql-source-primary:3307), +# which the host cannot resolve. Map the service names to loopback so gh-ost's +# auto-detected source primary resolves; published ports are 1:1 so the port +# already matches. Idempotent (guarded by a marker) and best-effort: a failure +# here is non-fatal, since operators can add the entries manually or pass +# --assume-master-host=mysql-source-primary:3307. +ensure_host_aliases() { + local marker="# gh-ost move-tables local aliases" + local entry="127.0.0.1 mysql-source-primary mysql-source-replica mysql-target-primary mysql-target-replica $marker" + if grep -qF "$marker" /etc/hosts 2>/dev/null; then + return 0 + fi + echo -n "Adding move-tables host aliases to /etc/hosts (may prompt for sudo)..." + if printf '%s\n' "$entry" | sudo tee -a /etc/hosts >/dev/null 2>&1; then + echo "OK" + else + echo "SKIPPED" + echo " Could not write /etc/hosts. gh-ost auto-detects the source primary as" + echo " 'mysql-source-primary:3307', which the host must resolve. Either add this line" + echo " to /etc/hosts manually:" + echo " $entry" + echo " or run gh-ost with --assume-master-host=mysql-source-primary:3307" + fi +} + +setup() { + [ -z "$TEST_MYSQL_IMAGE" ] && TEST_MYSQL_IMAGE="mysql:8.0.41" + + ensure_host_aliases + + echo "Starting MySQL $TEST_MYSQL_IMAGE containers (2 clusters)..." + compose_file="$GH_OST_ROOT/localtests/docker-compose-move-tables.yml" + MYSQL_SHA2_RSA_KEYS_FLAG="" + MYSQL_PASSWORD_HASHING_ALGORITHM="mysql_native_password" + MYSQL_NATIVE_PASSWORD_FLAG="" + if [[ $TEST_MYSQL_IMAGE =~ "mysql:8.4" ]]; then + MYSQL_PASSWORD_HASHING_ALGORITHM="caching_sha2_password" + MYSQL_SHA2_RSA_KEYS_FLAG="--caching-sha2-password-auto-generate-rsa-keys=ON" + MYSQL_NATIVE_PASSWORD_FLAG="$MYSQL_SHA2_RSA_KEYS_FLAG" + fi + (TEST_MYSQL_IMAGE="$TEST_MYSQL_IMAGE" MYSQL_NATIVE_PASSWORD_FLAG="$MYSQL_NATIVE_PASSWORD_FLAG" envsubst <"$compose_file") >"$compose_file.tmp" + + docker compose -f "$compose_file.tmp" up -d --wait + + echo "Waiting for MySQL..." + poll_mysql "source-primary" || exit 1 + poll_mysql "source-replica" || exit 1 + poll_mysql "target-primary" || exit 1 + poll_mysql "target-replica" || exit 1 + + # Setup replication for source cluster, not idempotent + echo -n "Setting up replication for source cluster..." + exec-mysql-source-primary -e "create user if not exists 'repl'@'%' identified with $MYSQL_PASSWORD_HASHING_ALGORITHM by 'repl';" + exec-mysql-source-primary -e "grant replication slave on *.* to 'repl'@'%'; flush privileges;" + exec-mysql-source-primary -e "create user if not exists 'gh-ost'@'%' identified with $MYSQL_PASSWORD_HASHING_ALGORITHM by 'gh-ost';" + exec-mysql-source-primary -e "grant all on *.* to 'gh-ost'@'%';" + + sleep 1 + if [[ $TEST_MYSQL_IMAGE =~ "mysql:8.4" ]]; then + exec-mysql-source-replica -e "change replication source to source_host='mysql-source-primary', source_port=3307, source_user='repl', source_password='repl', source_auto_position=1, source_ssl=1;" + exec-mysql-source-replica -e "start replica;" + else + exec-mysql-source-replica -e "change master to master_host='mysql-source-primary', master_port=3307, master_user='repl', master_password='repl', master_auto_position=1;" + exec-mysql-source-replica -e "start slave;" + fi + # Keep replicas non-writable so local move-tables runs that point --host at + # a replica fail at cutover (RENAME) instead of mutating only the replica. + exec-mysql-source-replica -e "set global read_only=ON; set global super_read_only=ON;" + echo "OK" + + # Setup replication for target cluster + echo -n "Setting up replication for target cluster..." + exec-mysql-target-primary -e "create user if not exists 'repl'@'%' identified with $MYSQL_PASSWORD_HASHING_ALGORITHM by 'repl';" + exec-mysql-target-primary -e "grant replication slave on *.* to 'repl'@'%'; flush privileges;" + exec-mysql-target-primary -e "create user if not exists 'gh-ost'@'%' identified with $MYSQL_PASSWORD_HASHING_ALGORITHM by 'gh-ost';" + exec-mysql-target-primary -e "grant all on *.* to 'gh-ost'@'%';" + + sleep 1 + if [[ $TEST_MYSQL_IMAGE =~ "mysql:8.4" ]]; then + exec-mysql-target-replica -e "change replication source to source_host='mysql-target-primary', source_port=3309, source_user='repl', source_password='repl', source_auto_position=1, source_ssl=1;" + exec-mysql-target-replica -e "start replica;" + else + exec-mysql-target-replica -e "change master to master_host='mysql-target-primary', master_port=3309, master_user='repl', master_password='repl', master_auto_position=1;" + exec-mysql-target-replica -e "start slave;" + fi + exec-mysql-target-replica -e "set global read_only=ON; set global super_read_only=ON;" + echo "OK" + + echo -n "Initializing '$DATABASE_NAME' database into each cluster..." + exec-mysql-source-primary -e "CREATE DATABASE IF NOT EXISTS $DATABASE_NAME;" + exec-mysql-target-primary -e "CREATE DATABASE IF NOT EXISTS $DATABASE_NAME;" + echo "OK" + + echo -n "Seeding data in source cluster..." + exec-mysql-source-primary -D $DATABASE_NAME < "$GH_OST_ROOT/localtests/move-tables/multiple-three/create.sql" + echo "OK" +} + +setup \ No newline at end of file diff --git a/script/move-tables/teardown b/script/move-tables/teardown new file mode 100755 index 000000000..b2ed313ef --- /dev/null +++ b/script/move-tables/teardown @@ -0,0 +1,24 @@ +#!/bin/bash + +# This script removes the four MySQL docker containers in two primary-replica clusters +# expected to be provisioned by the setup script. +# +# Usage: +# teardown remove the containers + +set -e + +GH_OST_ROOT=$(git rev-parse --show-toplevel) +if [[ ":$PATH:" != *":$GH_OST_ROOT:"* ]]; then + export PATH="${PATH}:${GH_OST_ROOT}/script" +fi + +echo "Stopping containers..." +docker stop mysql-source-replica mysql-source-primary mysql-target-replica mysql-target-primary 2>/dev/null || true + +echo "Removing containers..." +docker rm -f mysql-source-replica mysql-source-primary mysql-target-replica mysql-target-primary 2>/dev/null || true + +echo "Cleaning up Docker resources..." +docker system prune -f +docker volume prune -f \ No newline at end of file diff --git a/vendor/github.com/pingcap/failpoint/.codecov.yml b/vendor/github.com/pingcap/failpoint/.codecov.yml new file mode 100644 index 000000000..402988545 --- /dev/null +++ b/vendor/github.com/pingcap/failpoint/.codecov.yml @@ -0,0 +1,39 @@ +codecov: + notify: + require_ci_to_pass: yes + +coverage: + precision: 4 + round: down + range: "65...90" + + status: + project: + default: + threshold: 20 #Allow the coverage to drop by threshold%, and posting a success status. + patch: + default: + target: 0% # trial operation + changes: no + +parsers: + gcov: + branch_detection: + conditional: yes + loop: yes + method: no + macro: no + +comment: + layout: "header, diff" + behavior: default + require_changes: no + +ignore: + - "LICENSES" + - "*_test.go" + - "marker.go" # This file only contains empty function stub + - "failpoint-ctl" # Ignore the `failpoint-ctl` command line tool + - ".git" + - "*.yml" + - "*.md" diff --git a/vendor/github.com/pingcap/failpoint/.gitignore b/vendor/github.com/pingcap/failpoint/.gitignore new file mode 100644 index 000000000..b1e5133d6 --- /dev/null +++ b/vendor/github.com/pingcap/failpoint/.gitignore @@ -0,0 +1,27 @@ +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, build with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +bin +coverage.out +.idea/ +*.iml +*.swp +*.txt +*.log +tags +profile.coverprofile +overalls.coverprofile +explain_test +*.fail.go +vendor +.DS_Store diff --git a/vendor/github.com/pingcap/failpoint/CONTRIBUTING.md b/vendor/github.com/pingcap/failpoint/CONTRIBUTING.md new file mode 100644 index 000000000..cfefb0ed5 --- /dev/null +++ b/vendor/github.com/pingcap/failpoint/CONTRIBUTING.md @@ -0,0 +1,94 @@ +# How to contribute + +This document outlines some of the conventions on development workflow, commit +message formatting, contact points and other resources to make it easier to get +your contribution accepted. + +## Getting started + +- Fork the repository on GitHub. +- Read the README.md for build instructions. +- Play with the project, submit bugs, submit patches! + +## Building Failpoint + +Developing Failpoint requires: + +* [Go 1.13](http://golang.org/doc/code.html) +* An internet connection to download the dependencies + +Simply run `make` to build the program. + +```sh +make +``` + +### Running tests + +This project contains unit tests and integration tests with coverage collection. +See [tests/README.md](./tests/README.md) for how to execute and add tests. + +### Updating dependencies + +Failpoint manages dependencies using [Go module](https://github.com/golang/go/wiki/Modules). +To add or update a dependency, either + +* Use the `go mod edit` command to change the dependency, or +* Edit `go.mod` and then run `make update` to update the checksum. + +## Contribution flow + +This is a rough outline of what a contributor's workflow looks like: + +- Create a topic branch from where you want to base your work. This is usually `master`. +- Make commits of logical units and add test case if the change fixes a bug or adds new functionality. +- Run tests and make sure all the tests are passed. +- Make sure your commit messages are in the proper format (see below). +- Push your changes to a topic branch in your fork of the repository. +- Submit a pull request. +- Your PR must receive LGTMs from two maintainers. + +Thanks for your contributions! + +### Code style + +The coding style suggested by the Golang community is used in `failpoint`. +See the [style doc](https://github.com/golang/go/wiki/CodeReviewComments) for details. + +Please follow this style to makeg `failpoint` easy to review, maintain and develop. + +### Format of the Commit Message + +We follow a rough convention for commit messages that is designed to answer two +questions: what changed and why. The subject line should feature the what and +the body of the commit should describe the why. + +``` +restore: add comment for variable declaration + +Improve documentation. +``` + +The format can be described more formally as follows: + +``` +: + + + +