From ef02c862e38dbbd5e4b6fbea9527e798b077bade Mon Sep 17 00:00:00 2001 From: Daniel Joos Date: Wed, 10 Jun 2026 12:48:56 +0000 Subject: [PATCH 01/23] move-tables: add CLI parameters Add move-table command parsing and validate the table list input. Initialize migration context from the CLI configuration. Refs: #1702 --- go/base/context.go | 15 ++++++ go/cmd/gh-ost/main.go | 60 ++++++++++++++++++++++-- go/logic/migrator.go | 103 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 174 insertions(+), 4 deletions(-) diff --git a/go/base/context.go b/go/base/context.go index b2ec8ff1e..6a545baf6 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -280,6 +280,16 @@ type MigrationContext struct { SkipMetadataLockCheck bool IsOpenMetadataLockInstruments bool + // move tables: + MoveTables struct { + TableNames []string // List of table names to be moved. + TargetHost string // Target hostname for the move. This must be a primary/writable host. + TargetPort int // Target MySQL port for the move. + TargetUser string // Target username for the move. If not specified, it will default to the source user. + TargetPass string // Target password for the move. If not specified, it will default to the source password. + TargetDatabase string // Target database name for the move. If not specified, it will default to the source database name. + } + Log Logger } @@ -1038,6 +1048,11 @@ func (mctx *MigrationContext) CancelContext() { } } +// IsMoveTablesMode returns true if gh-ost should be used for moving tables instead of running a schema migration. +func (mctx *MigrationContext) IsMoveTablesMode() bool { + return len(mctx.MoveTables.TableNames) > 0 +} + // SendWithContext attempts to send a value to a channel, but returns early // if the context is cancelled. This prevents goroutine deadlocks when the // channel receiver has exited due to an error. diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index cd1f5993f..904240b44 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -12,6 +12,8 @@ import ( "os" "os/signal" "regexp" + "slices" + "strings" "syscall" "time" @@ -185,8 +187,16 @@ func main() { version := flag.Bool("version", false, "Print version & exit") checkFlag := flag.Bool("check-flag", false, "Check if another flag exists/supported. This allows for cross-version scripting. Exits with 0 when all additional provided flags exist, nonzero otherwise. You must provide (dummy) values for flags that require a value. Example: gh-ost --check-flag --cut-over-lock-timeout-seconds --nice-ratio 0") flag.StringVar(&migrationContext.ForceTmpTableName, "force-table-names", "", "table name prefix to be used on the temporary tables") - flag.CommandLine.SetOutput(os.Stdout) + // move tables flags + moveTables := flag.String("move-tables", "", "Comma delimited list of tables to move. e.g. 'table1,table2,table3'. This is a special mode that allows you to move tables between database clusters. This mode is mutually exclusive with --alter, --table, --test-on-replica, --migrate-on-replica and --revert.") + flag.StringVar(&migrationContext.MoveTables.TargetHost, "target-host", "", "Target MySQL hostname for --move-tables mode. Must be specified if --move-tables is specified.") + flag.IntVar(&migrationContext.MoveTables.TargetPort, "target-port", 3306, "Target MySQL port for --move-tables mode. Defaults to 3306.") + flag.StringVar(&migrationContext.MoveTables.TargetUser, "target-user", "", "Target MySQL username for --move-tables mode. If not provided, uses the same user as the source connection") + flag.StringVar(&migrationContext.MoveTables.TargetPass, "target-password", "", "Target MySQL password for --move-tables mode. If not provided, uses the same password as the source connection") + flag.StringVar(&migrationContext.MoveTables.TargetDatabase, "target-database", "", "Target MySQL database name for --move-tables mode. If not provided, uses the same database name as the source connection") + + flag.CommandLine.SetOutput(os.Stdout) flag.Parse() if *checkFlag { @@ -230,8 +240,8 @@ func main() { migrationContext.SetConnectionCharset(*charset) - if migrationContext.AlterStatement == "" && !migrationContext.Revert { - log.Fatal("--alter must be provided and statement must not be empty") + if migrationContext.AlterStatement == "" && !migrationContext.Revert && *moveTables == "" { + log.Fatal("--alter must be provided and statement must not be empty, or --revert must be used, or --move-tables must be used") } parser := sql.NewParserFromAlterStatement(migrationContext.AlterStatement) migrationContext.AlterStatementOptions = parser.GetAlterStatementOptions() @@ -271,7 +281,7 @@ func main() { migrationContext.Log.Fatale(err) } - if migrationContext.OriginalTableName == "" { + if migrationContext.OriginalTableName == "" && *moveTables == "" { if parser.HasExplicitTable() { migrationContext.OriginalTableName = parser.GetExplicitTable() } else { @@ -341,6 +351,46 @@ func main() { migrationContext.Log.Warning("--exact-rowcount with --panic-on-warnings: row counts cannot be exact due to warning detection") } + if *moveTables != "" { + if migrationContext.AlterStatement != "" { + log.Fatal("--move-tables is mutually exclusive with --alter") + } + if migrationContext.OriginalTableName != "" { + log.Fatal("--move-tables is mutually exclusive with --table") + } + if migrationContext.TestOnReplica { + log.Fatal("--move-tables is mutually exclusive with --test-on-replica") + } + if migrationContext.MigrateOnReplica { + log.Fatal("--move-tables is mutually exclusive with --migrate-on-replica") + } + if migrationContext.Revert { + log.Fatal("--move-tables is mutually exclusive with --revert") + } + if migrationContext.MoveTables.TargetHost == "" { + log.Fatal("--target-host must be specified when using --move-tables") + } + migrationContext.MoveTables.TableNames = strings.Split(*moveTables, ",") + for i := range migrationContext.MoveTables.TableNames { + migrationContext.MoveTables.TableNames[i] = strings.TrimSpace(migrationContext.MoveTables.TableNames[i]) + } + migrationContext.MoveTables.TableNames = slices.DeleteFunc(migrationContext.MoveTables.TableNames, func(s string) bool { return s == "" }) + if len(migrationContext.MoveTables.TableNames) > 1 { + // Future version will support moving multiple tables at the same time. + // For now, we only support moving a single table at a time. + log.Fatal("--move-tables currently supports only a single table") + } + if migrationContext.MoveTables.TargetUser == "" { + migrationContext.MoveTables.TargetUser = migrationContext.CliUser + } + if migrationContext.MoveTables.TargetPass == "" { + migrationContext.MoveTables.TargetPass = migrationContext.CliPassword + } + if migrationContext.MoveTables.TargetDatabase == "" { + migrationContext.MoveTables.TargetDatabase = migrationContext.DatabaseName + } + } + switch *cutOver { case "atomic", "default", "": migrationContext.CutOverType = base.CutOverAtomic @@ -411,6 +461,8 @@ func main() { var err error if migrationContext.Revert { err = migrator.Revert() + } else if migrationContext.IsMoveTablesMode() { + err = migrator.MoveTables() } else { err = migrator.Migrate() } diff --git a/go/logic/migrator.go b/go/logic/migrator.go index f2f6b3f20..974c6698b 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -798,6 +798,109 @@ func (mgtr *Migrator) Revert() error { return nil } +func (mgtr *Migrator) MoveTables() (err error) { + mgtr.migrationContext.Log.Infof("Moving tables %v from %s to %s (%s)", + mgtr.migrationContext.MoveTables.TableNames, + sql.EscapeName(mgtr.migrationContext.DatabaseName), + sql.EscapeName(mgtr.migrationContext.MoveTables.TargetDatabase), mgtr.migrationContext.MoveTables.TargetHost) + mgtr.migrationContext.StartTime = time.Now() + + // Ensure context is cancelled on exit (cleanup) + defer mgtr.migrationContext.CancelContext() + + if mgtr.migrationContext.Hostname, err = os.Hostname(); err != nil { + return err + } + + go mgtr.listenOnPanicAbort() + + // Run on-startup hook: + if err := mgtr.hooksExecutor.OnStartup(); err != nil { + return err + } + + // After this point, we'll need to teardown anything that's been started + // so we don't leave things hanging around + defer mgtr.teardown() + + if err := mgtr.initiateInspector(); err != nil { + return err + } + if err := mgtr.checkAbort(); err != nil { + return err + } + if err := mgtr.initiateApplier(); err != nil { + return err + } + if err := mgtr.checkAbort(); err != nil { + return err + } + + // Validation complete! Run on-validated hook. + if err := mgtr.hooksExecutor.OnValidated(); err != nil { + return err + } + + if err := mgtr.initiateServer(); err != nil { + return err + } + defer mgtr.server.RemoveSocketFile() + + if err := mgtr.countTableRows(); err != nil { + return err + } + if err := mgtr.addDMLEventsListener(); err != nil { + return err + } + if err := mgtr.applier.ReadMigrationRangeValues(); err != nil { + return err + } + + mgtr.initiateThrottler() + + // Run on-before-row-copy hook + if err := mgtr.hooksExecutor.OnBeforeRowCopy(); err != nil { + return err + } + go func() { + if err := mgtr.executeWriteFuncs(); err != nil { + // Send error to PanicAbort to trigger abort + _ = base.SendWithContext(mgtr.migrationContext.GetContext(), mgtr.migrationContext.PanicAbort, err) + } + }() + go mgtr.iterateChunks() + mgtr.migrationContext.MarkRowCopyStartTime() + go mgtr.initiateStatus() + + mgtr.migrationContext.Log.Debugf("Operating until row copy is complete") + mgtr.consumeRowCopyComplete() + mgtr.migrationContext.Log.Infof("Row copy complete") + // Check if row copy was aborted due to error + if err := mgtr.checkAbort(); err != nil { + return err + } + if err := mgtr.hooksExecutor.OnRowCopyComplete(); err != nil { + return err + } + + //TODO: cutover here + + if err := mgtr.finalCleanup(); err != nil { + return nil + } + if err := mgtr.hooksExecutor.OnSuccess(false); err != nil { + return err + } + mgtr.migrationContext.Log.Infof("Done moving tables %v from %s to %s (%s)", + mgtr.migrationContext.MoveTables.TableNames, sql.EscapeName(mgtr.migrationContext.DatabaseName), + sql.EscapeName(mgtr.migrationContext.MoveTables.TargetDatabase), mgtr.migrationContext.MoveTables.TargetHost) + // Final check for abort before declaring success + if err := mgtr.checkAbort(); err != nil { + return err + } + return nil +} + // ExecOnFailureHook executes the onFailure hook, and this method is provided as the only external // hook access point func (mgtr *Migrator) ExecOnFailureHook() (err error) { From b224d59c9064bb0cbf4de4fa04125822f12a5558 Mon Sep 17 00:00:00 2001 From: Daniel Joos Date: Thu, 11 Jun 2026 08:28:14 +0000 Subject: [PATCH 02/23] move-tables: add copy query and applier support Build range-bounded copy queries and prepare target-table DML queries. Apply copy batches through the applier and invoke them from the migrator. Refs: #1703 --- go/base/context.go | 19 ++- go/logic/applier.go | 198 +++++++++++++++++++++++++++-- go/logic/applier_test.go | 226 ++++++++++++++++++++++++++++++++- go/logic/migrator.go | 7 +- go/logic/test_utils_test.go | 5 + go/sql/builder.go | 141 +++++++++++++++++++++ go/sql/builder_test.go | 240 ++++++++++++++++++++++++++++++++++++ 7 files changed, 821 insertions(+), 15 deletions(-) diff --git a/go/base/context.go b/go/base/context.go index 6a545baf6..aebf4a0db 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -282,12 +282,13 @@ type MigrationContext struct { // move tables: MoveTables struct { - TableNames []string // List of table names to be moved. - TargetHost string // Target hostname for the move. This must be a primary/writable host. - TargetPort int // Target MySQL port for the move. - TargetUser string // Target username for the move. If not specified, it will default to the source user. - TargetPass string // Target password for the move. If not specified, it will default to the source password. - TargetDatabase string // Target database name for the move. If not specified, it will default to the source database name. + TableNames []string // List of table names to be moved. + TargetHost string // Target hostname for the move. This must be a primary/writable host. + TargetPort int // Target MySQL port for the move. + TargetUser string // Target username for the move. If not specified, it will default to the source user. + TargetPass string // Target password for the move. If not specified, it will default to the source password. + TargetDatabase string // Target database name for the move. If not specified, it will default to the source database name. + ConnectionConfig *mysql.ConnectionConfig } Log Logger @@ -362,6 +363,9 @@ func (mctx *MigrationContext) SetConnectionConfig(storageEngine string) error { } mctx.InspectorConnectionConfig.TransactionIsolation = transactionIsolation mctx.ApplierConnectionConfig.TransactionIsolation = transactionIsolation + if mctx.MoveTables.ConnectionConfig != nil { + mctx.MoveTables.ConnectionConfig.TransactionIsolation = transactionIsolation + } return nil } @@ -372,6 +376,9 @@ func (mctx *MigrationContext) SetConnectionCharset(charset string) { mctx.InspectorConnectionConfig.Charset = charset mctx.ApplierConnectionConfig.Charset = charset + if mctx.MoveTables.ConnectionConfig != nil { + mctx.MoveTables.ConnectionConfig.Charset = charset + } } func getSafeTableName(baseName string, suffix string) string { diff --git a/go/logic/applier.go b/go/logic/applier.go index 3f401c598..4e1a547f3 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -92,6 +92,12 @@ type Applier struct { migrationLockName string migrationLockStop chan struct{} migrationLockDone chan struct{} + + moveTablesTargetDB *gosql.DB + moveTablesConnectionConfig *mysql.ConnectionConfig + moveTablesCopySelectFirstQueryBuilder *sql.MoveTableCopySelectQueryBuilder + moveTablesCopySelectNextQueryBuilder *sql.MoveTableCopySelectQueryBuilder + moveTablesCopyInsertQueryBuilder *sql.MoveTableCopyInsertQueryBuilder } func NewApplier(migrationContext *base.MigrationContext) *Applier { @@ -100,6 +106,8 @@ func NewApplier(migrationContext *base.MigrationContext) *Applier { migrationContext: migrationContext, finishedMigrating: 0, name: "applier", + + moveTablesConnectionConfig: migrationContext.MoveTables.ConnectionConfig, } } @@ -150,6 +158,15 @@ func (apl *Applier) InitDBConnections() (err error) { if err := apl.readTableColumns(); err != nil { return err } + if apl.moveTablesConnectionConfig != nil { + moveTablesURI := apl.moveTablesConnectionConfig.GetDBUri(apl.migrationContext.MoveTables.TargetDatabase) + "&multiStatements=true" + if apl.moveTablesTargetDB, _, err = mysql.GetDB(apl.migrationContext.Uuid, moveTablesURI); err != nil { + return err + } + if _, err := base.ValidateConnection(apl.moveTablesTargetDB, apl.moveTablesConnectionConfig, apl.migrationContext, apl.name); err != nil { + return err + } + } apl.migrationContext.Log.Infof("Applier initiated on %+v, version %+v", apl.connectionConfig.ImpliedKey, apl.migrationContext.ApplierMySQLVersion) return nil } @@ -298,17 +315,24 @@ func (apl *Applier) releaseMigrationLock() { } func (apl *Applier) prepareQueries() (err error) { + targetDatabaseName := apl.migrationContext.DatabaseName + targetTableName := apl.migrationContext.GetGhostTableName() + if apl.migrationContext.IsMoveTablesMode() { + targetDatabaseName = apl.migrationContext.MoveTables.TargetDatabase + targetTableName = apl.migrationContext.OriginalTableName + } + if apl.dmlDeleteQueryBuilder, err = sql.NewDMLDeleteQueryBuilder( - apl.migrationContext.DatabaseName, - apl.migrationContext.GetGhostTableName(), + targetDatabaseName, + targetTableName, apl.migrationContext.OriginalTableColumns, &apl.migrationContext.UniqueKey.Columns, ); err != nil { return err } if apl.dmlInsertQueryBuilder, err = sql.NewDMLInsertQueryBuilder( - apl.migrationContext.DatabaseName, - apl.migrationContext.GetGhostTableName(), + targetDatabaseName, + targetTableName, apl.migrationContext.OriginalTableColumns, apl.migrationContext.SharedColumns, apl.migrationContext.MappedSharedColumns, @@ -316,8 +340,8 @@ func (apl *Applier) prepareQueries() (err error) { return err } if apl.dmlUpdateQueryBuilder, err = sql.NewDMLUpdateQueryBuilder( - apl.migrationContext.DatabaseName, - apl.migrationContext.GetGhostTableName(), + targetDatabaseName, + targetTableName, apl.migrationContext.OriginalTableColumns, apl.migrationContext.SharedColumns, apl.migrationContext.MappedSharedColumns, @@ -334,6 +358,35 @@ func (apl *Applier) prepareQueries() (err error) { return err } } + if apl.migrationContext.IsMoveTablesMode() { + if apl.moveTablesCopySelectFirstQueryBuilder, err = sql.NewMoveTableCopySelectQueryBuilder( + apl.migrationContext.DatabaseName, + apl.migrationContext.OriginalTableName, + apl.migrationContext.OriginalTableColumns, + apl.migrationContext.UniqueKey.Name, + &apl.migrationContext.UniqueKey.Columns, + true, // <-- include start range values for first select query + ); err != nil { + return err + } + if apl.moveTablesCopySelectNextQueryBuilder, err = sql.NewMoveTableCopySelectQueryBuilder( + apl.migrationContext.DatabaseName, + apl.migrationContext.OriginalTableName, + apl.migrationContext.OriginalTableColumns, + apl.migrationContext.UniqueKey.Name, + &apl.migrationContext.UniqueKey.Columns, + false, + ); err != nil { + return err + } + if apl.moveTablesCopyInsertQueryBuilder, err = sql.NewMoveTableCopyInsertQueryBuilder( + targetDatabaseName, + targetTableName, + apl.migrationContext.OriginalTableColumns, + ); err != nil { + return err + } + } return nil } @@ -1244,6 +1297,130 @@ func (apl *Applier) ApplyIterationInsertQuery() (chunkSize int64, rowsAffected i return chunkSize, rowsAffected, duration, nil } +// ApplyIterationMoveTableCopyQueries issues a SELECT query on the original table and an INSERT query on the target table, +// copying a chunk of rows. It is used when `--move-tables` is specified, instead of ApplyIterationInsertQuery. +func (apl *Applier) ApplyIterationMoveTableCopyQueries() (chunkSize int64, rowsAffected int64, duration time.Duration, err error) { + startTime := time.Now() + chunkSize = atomic.LoadInt64(&apl.migrationContext.ChunkSize) + + // First, select data from the source database: + rows, err := func() ([]*sql.ColumnValues, error) { + var qb *sql.MoveTableCopySelectQueryBuilder + if apl.migrationContext.GetIteration() == 0 { + qb = apl.moveTablesCopySelectFirstQueryBuilder + } else { + qb = apl.moveTablesCopySelectNextQueryBuilder + } + query, explodedArgs, err := qb.BuildQuery( + apl.migrationContext.MigrationIterationRangeMinValues.AbstractValues(), + apl.migrationContext.MigrationIterationRangeMaxValues.AbstractValues(), + ) + if err != nil { + return nil, err + } + sqlRows, err := apl.db.Query(query, explodedArgs...) + if err != nil { + return nil, err + } + defer sqlRows.Close() + chunkRows := make([]*sql.ColumnValues, 0, chunkSize) + for sqlRows.Next() { + row := sql.NewColumnValues(apl.migrationContext.SharedColumns.Len()) + err := sqlRows.Scan(row.ValuesPointers...) + if err != nil { + return nil, err + } + chunkRows = append(chunkRows, row) + } + if rowsErr := sqlRows.Err(); rowsErr != nil { + return nil, rowsErr + } + return chunkRows, nil + }() + if err != nil { + return chunkSize, rowsAffected, duration, err + } + + // no need to INSERT if there are no rows to copy: + if len(rows) == 0 { + duration = time.Since(startTime) + return chunkSize, 0, duration, nil + } + + // Then, insert data into the destination database: + sqlResult, err := func() (gosql.Result, error) { + query, explodedArgs, err := apl.moveTablesCopyInsertQueryBuilder.BuildQuery(rows) + if err != nil { + return nil, err + } + tx, err := apl.moveTablesTargetDB.Begin() + if err != nil { + return nil, err + } + defer tx.Rollback() + + sessionQuery := fmt.Sprintf(`SET SESSION time_zone = '%s', %s`, + apl.migrationContext.ApplierTimeZone, + apl.generateSqlModeQuery()) + if _, err := tx.Exec(sessionQuery); err != nil { + return nil, err + } + + sqlResult, err := tx.Exec(query, explodedArgs...) + if err != nil { + return nil, err + } + + if apl.migrationContext.PanicOnWarnings { + rows, err := tx.Query("SHOW WARNINGS") + if err != nil { + return nil, err + } + defer rows.Close() + if err = rows.Err(); err != nil { + return nil, err + } + migrationKeyRegex, err := apl.compileMigrationKeyWarningRegex() + if err != nil { + return nil, err + } + var sqlWarnings []string + for rows.Next() { + var level, message string + var code int + if err := rows.Scan(&level, &code, &message); err != nil { + apl.migrationContext.Log.Warningf("Failed to read SHOW WARNINGS row") + continue + } + if strings.Contains(message, "Duplicate entry") && migrationKeyRegex.MatchString(message) { + continue + } + sqlWarnings = append(sqlWarnings, fmt.Sprintf("%s: %s (%d)", level, message, code)) + } + apl.migrationContext.MigrationLastInsertSQLWarnings = sqlWarnings + } + + if err := tx.Commit(); err != nil { + return nil, err + } + return sqlResult, nil + }() + if err != nil { + return chunkSize, rowsAffected, duration, err + } + rowsAffected, _ = sqlResult.RowsAffected() + duration = time.Since(startTime) + apl.migrationContext.Log.Debugf( + "Issued SELECT+INSERT on range: [%s]..[%s]; iteration: %d; chunk-size: %d", + apl.migrationContext.MigrationIterationRangeMinValues, + apl.migrationContext.MigrationIterationRangeMaxValues, + apl.migrationContext.GetIteration(), + chunkSize, + ) + + return chunkSize, rowsAffected, duration, nil +} + // LockOriginalTable places a write lock on the original table func (apl *Applier) LockOriginalTable() error { query := fmt.Sprintf(`lock /* gh-ost */ tables %s.%s write`, @@ -1863,7 +2040,11 @@ func (apl *Applier) ApplyDMLEventQueries(dmlEvents [](*binlog.BinlogDMLEvent)) e ctx := context.Background() err := func() error { - conn, err := apl.db.Conn(ctx) + db := apl.db + if apl.migrationContext.IsMoveTablesMode() { + db = apl.moveTablesTargetDB + } + conn, err := db.Conn(ctx) if err != nil { return err } @@ -1972,6 +2153,9 @@ func (apl *Applier) Teardown() { apl.releaseMigrationLock() apl.db.Close() apl.singletonDB.Close() + if apl.moveTablesTargetDB != nil { + apl.moveTablesTargetDB.Close() + } atomic.StoreInt64(&apl.finishedMigrating, 1) } diff --git a/go/logic/applier_test.go b/go/logic/applier_test.go index f1fa28bc8..0c779b781 100644 --- a/go/logic/applier_test.go +++ b/go/logic/applier_test.go @@ -9,6 +9,7 @@ import ( "context" gosql "database/sql" "errors" + "net" "strings" "testing" "time" @@ -338,6 +339,7 @@ type ApplierTestSuite struct { mysqlContainer testcontainers.Container db *gosql.DB + otherDB *gosql.DB } func (suite *ApplierTestSuite) SetupSuite() { @@ -358,12 +360,30 @@ func (suite *ApplierTestSuite) SetupSuite() { db, err := gosql.Open("mysql", dsn) suite.Require().NoError(err) - suite.db = db + + containerHost, err := mysqlContainer.Host(ctx) + suite.Require().NoError(err) + containerPort, err := mysqlContainer.MappedPort(ctx, "3306/tcp") + suite.Require().NoError(err) + + // Second database & connection for move-tables tests: + _, err = suite.db.ExecContext(ctx, fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s", testMysqlDatabaseOther)) + suite.Require().NoError(err) + otherConf := drivermysql.NewConfig() + otherConf.DBName = testMysqlDatabaseOther + otherConf.User = testMysqlUser + otherConf.Passwd = testMysqlPass + otherConf.Net = "tcp" + otherConf.Addr = net.JoinHostPort(containerHost, containerPort.Port()) + otherDB, err := gosql.Open("mysql", otherConf.FormatDSN()) + suite.Require().NoError(err) + suite.otherDB = otherDB } func (suite *ApplierTestSuite) TearDownSuite() { suite.Assert().NoError(suite.db.Close()) + suite.Assert().NoError(suite.otherDB.Close()) suite.Assert().NoError(testcontainers.TerminateContainer(suite.mysqlContainer)) } @@ -380,6 +400,8 @@ func (suite *ApplierTestSuite) TearDownTest() { suite.Require().NoError(err) _, err = suite.db.ExecContext(ctx, "DROP TABLE IF EXISTS "+getTestGhostTableName()) suite.Require().NoError(err) + _, err = suite.otherDB.ExecContext(ctx, "DROP TABLE IF EXISTS "+getTestOtherTableName()) + suite.Require().NoError(err) } func (suite *ApplierTestSuite) TestInitDBConnections() { @@ -1728,6 +1750,208 @@ func (suite *ApplierTestSuite) TestMultipleDMLEventsInBatch() { // Critically: id=2 (bob@example.com) is NOT present, proving event #3 was rolled back } +func (suite *ApplierTestSuite) TestApplyDMLEventQueriesMoveTablesMode() { + ctx := context.Background() + var err error + + _, err = suite.db.ExecContext(ctx, fmt.Sprintf("CREATE TABLE %s (id INT, item_id INT);", getTestTableName())) + suite.Require().NoError(err) + _, err = suite.otherDB.ExecContext(ctx, fmt.Sprintf("CREATE TABLE %s (id INT, item_id INT);", getTestOtherTableName())) + suite.Require().NoError(err) + + connectionConfig, err := getTestConnectionConfig(ctx, suite.mysqlContainer) + suite.Require().NoError(err) + + migrationContext := newTestMigrationContext() + migrationContext.ApplierConnectionConfig = connectionConfig + migrationContext.MoveTables.ConnectionConfig = connectionConfig + migrationContext.SetConnectionConfig("innodb") + + migrationContext.OriginalTableColumns = sql.NewColumnList([]string{"id", "item_id"}) + migrationContext.SharedColumns = sql.NewColumnList([]string{"id", "item_id"}) + migrationContext.MappedSharedColumns = sql.NewColumnList([]string{"id", "item_id"}) + migrationContext.UniqueKey = &sql.UniqueKey{ + Name: "primary_key", + Columns: *sql.NewColumnList([]string{"id"}), + } + migrationContext.MoveTables.TableNames = []string{testMysqlTableName} + migrationContext.MoveTables.TargetDatabase = testMysqlDatabaseOther + + applier := NewApplier(migrationContext) + suite.Require().NoError(applier.prepareQueries()) + defer applier.Teardown() + + err = applier.InitDBConnections() + suite.Require().NoError(err) + + dmlEvents := []*binlog.BinlogDMLEvent{ + { + DatabaseName: testMysqlDatabase, + TableName: testMysqlTableName, + DML: binlog.InsertDML, + NewColumnValues: sql.ToColumnValues([]interface{}{123456, 42}), + }, + } + err = applier.ApplyDMLEventQueries(dmlEvents) + suite.Require().NoError(err) + + // Check that the row was inserted into the ghost table via moveTablesTargetDB + rows, err := suite.otherDB.Query("SELECT * FROM " + getTestOtherTableName()) + suite.Require().NoError(err) + defer rows.Close() + + var count, id, item_id int + for rows.Next() { + err = rows.Scan(&id, &item_id) + suite.Require().NoError(err) + count += 1 + } + suite.Require().NoError(rows.Err()) + + suite.Require().Equal(1, count) + suite.Require().Equal(123456, id) + suite.Require().Equal(42, item_id) + + suite.Require().Equal(int64(1), migrationContext.TotalDMLEventsApplied) + suite.Require().Equal(int64(0), migrationContext.RowsDeltaEstimate) +} + +func (suite *ApplierTestSuite) TestApplyIterationMoveTableCopyQueries() { + ctx := context.Background() + var err error + + _, err = suite.db.ExecContext(ctx, fmt.Sprintf("CREATE TABLE %s (id INT NOT NULL, name VARCHAR(50), created_at DATETIME NOT NULL, PRIMARY KEY(id));", getTestTableName())) + suite.Require().NoError(err) + _, err = suite.otherDB.ExecContext(ctx, fmt.Sprintf("CREATE TABLE %s (id INT NOT NULL, name VARCHAR(50), created_at DATETIME NOT NULL, PRIMARY KEY(id));", getTestOtherTableName())) + suite.Require().NoError(err) + _, err = suite.db.ExecContext(ctx, fmt.Sprintf("INSERT INTO %s (id, name, created_at) VALUES (1, 'alice', '2024-01-15 10:30:00'), (2, 'bob', '2024-06-20 14:45:00'), (3, 'carol', '2025-12-31 23:59:59');", getTestTableName())) + suite.Require().NoError(err) + + connectionConfig, err := getTestConnectionConfig(ctx, suite.mysqlContainer) + suite.Require().NoError(err) + + migrationContext := newTestMigrationContext() + migrationContext.ApplierConnectionConfig = connectionConfig + migrationContext.MoveTables.ConnectionConfig = connectionConfig + migrationContext.SetConnectionConfig("innodb") + migrationContext.OriginalTableColumns = sql.NewColumnList([]string{"id", "name", "created_at"}) + migrationContext.SharedColumns = sql.NewColumnList([]string{"id", "name", "created_at"}) + migrationContext.MappedSharedColumns = sql.NewColumnList([]string{"id", "name", "created_at"}) + migrationContext.UniqueKey = &sql.UniqueKey{ + Name: "PRIMARY", + Columns: *sql.NewColumnList([]string{"id"}), + } + migrationContext.MoveTables.TableNames = []string{testMysqlTableName} + migrationContext.MoveTables.TargetDatabase = testMysqlDatabaseOther + + applier := NewApplier(migrationContext) + applier.prepareQueries() + defer applier.Teardown() + + err = applier.InitDBConnections() + suite.Require().NoError(err) + + err = applier.CreateChangelogTable() + suite.Require().NoError(err) + + err = applier.ReadMigrationRangeValues() + suite.Require().NoError(err) + + migrationContext.SetNextIterationRangeMinValues() + hasFurtherRange, err := applier.CalculateNextIterationRangeEndValues() + suite.Require().NoError(err) + suite.Require().True(hasFurtherRange) + + chunkSize, rowsAffected, duration, err := applier.ApplyIterationMoveTableCopyQueries() + suite.Require().NoError(err) + suite.Require().Equal(int64(3), rowsAffected) + suite.Require().Equal(int64(1000), chunkSize) + suite.Require().Greater(duration, time.Duration(0)) + + // Verify rows were copied to the other table + rows, err := suite.otherDB.QueryContext(ctx, "SELECT id, name, created_at FROM "+getTestOtherTableName()+" ORDER BY id") + suite.Require().NoError(err) + defer rows.Close() + + type row struct { + id int + name string + createdAt string + } + var results []row + for rows.Next() { + var r row + err = rows.Scan(&r.id, &r.name, &r.createdAt) + suite.Require().NoError(err) + results = append(results, r) + } + suite.Require().NoError(rows.Err()) + + suite.Require().Len(results, 3) + suite.Require().Equal(1, results[0].id) + suite.Require().Equal("alice", results[0].name) + suite.Require().Equal("2024-01-15 10:30:00", results[0].createdAt) + suite.Require().Equal(2, results[1].id) + suite.Require().Equal("bob", results[1].name) + suite.Require().Equal("2024-06-20 14:45:00", results[1].createdAt) + suite.Require().Equal(3, results[2].id) + suite.Require().Equal("carol", results[2].name) + suite.Require().Equal("2025-12-31 23:59:59", results[2].createdAt) +} + +func (suite *ApplierTestSuite) TestApplyIterationMoveTableCopyQueriesNoRows() { + ctx := context.Background() + var err error + + _, err = suite.db.ExecContext(ctx, fmt.Sprintf("CREATE TABLE %s (id INT NOT NULL, name VARCHAR(50), created_at DATETIME NOT NULL, PRIMARY KEY(id));", getTestTableName())) + suite.Require().NoError(err) + _, err = suite.otherDB.ExecContext(ctx, fmt.Sprintf("CREATE TABLE %s (id INT NOT NULL, name VARCHAR(50), created_at DATETIME NOT NULL, PRIMARY KEY(id));", getTestOtherTableName())) + suite.Require().NoError(err) + _, err = suite.db.ExecContext(ctx, fmt.Sprintf("INSERT INTO %s (id, name, created_at) VALUES (1, 'alice', '2024-01-15 10:30:00'), (2, 'bob', '2024-06-20 14:45:00'), (3, 'carol', '2025-12-31 23:59:59');", getTestTableName())) + suite.Require().NoError(err) + + connectionConfig, err := getTestConnectionConfig(ctx, suite.mysqlContainer) + suite.Require().NoError(err) + + migrationContext := newTestMigrationContext() + migrationContext.ApplierConnectionConfig = connectionConfig + migrationContext.MoveTables.ConnectionConfig = connectionConfig + migrationContext.SetConnectionConfig("innodb") + migrationContext.OriginalTableColumns = sql.NewColumnList([]string{"id", "name", "created_at"}) + migrationContext.SharedColumns = sql.NewColumnList([]string{"id", "name", "created_at"}) + migrationContext.MappedSharedColumns = sql.NewColumnList([]string{"id", "name", "created_at"}) + migrationContext.UniqueKey = &sql.UniqueKey{ + Name: "PRIMARY", + Columns: *sql.NewColumnList([]string{"id"}), + } + migrationContext.MoveTables.TableNames = []string{testMysqlTableName} + migrationContext.MoveTables.TargetDatabase = testMysqlDatabaseOther + + applier := NewApplier(migrationContext) + applier.prepareQueries() + defer applier.Teardown() + + err = applier.InitDBConnections() + suite.Require().NoError(err) + + // Point the iteration range at a key range that contains no rows so the + // SELECT returns an empty result set and the INSERT is skipped. + migrationContext.MigrationIterationRangeMinValues = sql.ToColumnValues([]interface{}{100}) + migrationContext.MigrationIterationRangeMaxValues = sql.ToColumnValues([]interface{}{200}) + + chunkSize, rowsAffected, duration, err := applier.ApplyIterationMoveTableCopyQueries() + suite.Require().NoError(err) + suite.Require().Equal(int64(0), rowsAffected) + suite.Require().Equal(int64(1000), chunkSize) + suite.Require().Greater(duration, time.Duration(0)) + + // Verify no rows were copied to the target table. + var count int + err = suite.otherDB.QueryRowContext(ctx, "SELECT COUNT(*) FROM "+getTestOtherTableName()).Scan(&count) + suite.Require().NoError(err) + suite.Require().Equal(0, count) +} + func TestApplier(t *testing.T) { if testing.Short() { t.Skip("skipping applier test suite in short mode") diff --git a/go/logic/migrator.go b/go/logic/migrator.go index 974c6698b..ae0a1fbb8 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -1811,7 +1811,12 @@ func (mgtr *Migrator) iterateChunks() error { // _ghost_ table, which no longer exists. So, bothering error messages and all, but no damage. return nil } - _, rowsAffected, _, err := mgtr.applier.ApplyIterationInsertQuery() + var rowsAffected int64 + if mgtr.migrationContext.IsMoveTablesMode() { + _, rowsAffected, _, err = mgtr.applier.ApplyIterationMoveTableCopyQueries() + } else { + _, rowsAffected, _, err = mgtr.applier.ApplyIterationInsertQuery() + } if err != nil { return err // wrapping call will retry } diff --git a/go/logic/test_utils_test.go b/go/logic/test_utils_test.go index 6012d4556..b334830ec 100644 --- a/go/logic/test_utils_test.go +++ b/go/logic/test_utils_test.go @@ -17,6 +17,7 @@ var ( testMysqlUser = "root" testMysqlPass = "root-password" testMysqlDatabase = "test" + testMysqlDatabaseOther = "test_other" testMysqlTableName = "testing" ) @@ -36,6 +37,10 @@ func getTestOldTableName() string { return fmt.Sprintf("`%s`.`_%s_del`", testMysqlDatabase, testMysqlTableName) } +func getTestOtherTableName() string { + return fmt.Sprintf("`%s`.`%s`", testMysqlDatabaseOther, testMysqlTableName) +} + func getTestConnectionConfig(ctx context.Context, container testcontainers.Container) (*mysql.ConnectionConfig, error) { host, err := container.Host(ctx) if err != nil { diff --git a/go/sql/builder.go b/go/sql/builder.go index 7d0864601..1c3c612fa 100644 --- a/go/sql/builder.go +++ b/go/sql/builder.go @@ -7,6 +7,7 @@ package sql import ( "fmt" + "slices" "strconv" "strings" ) @@ -425,6 +426,146 @@ func BuildRangeInsertPreparedQuery(databaseName, originalTableName, ghostTableNa return BuildRangeInsertQuery(databaseName, originalTableName, ghostTableName, sharedColumns, mappedSharedColumns, uniqueKey, uniqueKeyColumns, rangeStartValues, rangeEndValues, rangeStartArgs, rangeEndArgs, includeRangeStartValues, transactionalTable, noWait) } +type MoveTableCopySelectQueryBuilder struct { + preparedStatement string + argsMapping []int + argsCount int +} + +func NewMoveTableCopySelectQueryBuilder(sourceDatabaseName, sourceTableName string, columns *ColumnList, uniqueKey string, uniqueKeyColumns *ColumnList, includeRangeStartValues bool) (*MoveTableCopySelectQueryBuilder, error) { + sourceDatabaseName = EscapeName(sourceDatabaseName) + sourceTableName = EscapeName(sourceTableName) + columnNames := columns.Names() + for i := range columnNames { + columnNames[i] = EscapeName(columnNames[i]) + } + sharedColumnsListing := strings.Join(columnNames, ", ") + uniqueKey = EscapeName(uniqueKey) + var minRangeComparisonSign = GreaterThanComparisonSign + if includeRangeStartValues { + minRangeComparisonSign = GreaterThanOrEqualsComparisonSign + } + rangeStartValues := buildColumnsPreparedValues(uniqueKeyColumns) + rangeEndValues := buildColumnsPreparedValues(uniqueKeyColumns) + dummyArgs := make([]any, len(uniqueKeyColumns.Columns())) + for i := range dummyArgs { + dummyArgs[i] = i + } + var argsMapping []int + + rangeStartComparison, rangeExplodedArgs, err := BuildRangeComparison(uniqueKeyColumns.Names(), rangeStartValues, dummyArgs, minRangeComparisonSign) + if err != nil { + return nil, err + } + for _, a := range rangeExplodedArgs { + idx := slices.Index(dummyArgs, a) + if idx == -1 { + return nil, fmt.Errorf("failed to build args mapping, missing argument pointer %v", a) + } + argsMapping = append(argsMapping, idx) + } + + rangeEndComparison, rangeExplodedArgs, err := BuildRangeComparison(uniqueKeyColumns.Names(), rangeEndValues, dummyArgs, LessThanOrEqualsComparisonSign) + if err != nil { + return nil, err + } + for _, a := range rangeExplodedArgs { + idx := slices.Index(dummyArgs, a) + if idx == -1 { + return nil, fmt.Errorf("failed to build args mapping, missing argument pointer %v", a) + } + argsMapping = append(argsMapping, idx+len(dummyArgs)) + } + + stmt := fmt.Sprintf(` + select /* gh-ost %s.%s */ %s + from + %s.%s + force index (%s) + where + (%s and %s) + `, + sourceDatabaseName, sourceTableName, sharedColumnsListing, + sourceDatabaseName, sourceTableName, + uniqueKey, + rangeStartComparison, rangeEndComparison, + ) + return &MoveTableCopySelectQueryBuilder{ + preparedStatement: stmt, + argsMapping: argsMapping, + argsCount: len(dummyArgs) * 2, + }, nil +} + +func (b *MoveTableCopySelectQueryBuilder) BuildQuery(rangeStartArgs, rangeEndArgs []any) (string, []any, error) { + if len(rangeStartArgs)+len(rangeEndArgs) != b.argsCount { + return "", nil, fmt.Errorf("got %d args but expected %d", len(rangeStartArgs)+len(rangeEndArgs), b.argsCount) + } + if len(rangeStartArgs) != len(rangeEndArgs) { + return "", nil, fmt.Errorf("mismatched number of start and end args: %d != %d", len(rangeStartArgs), len(rangeEndArgs)) + } + explodedArgs := make([]any, 0, len(b.argsMapping)) + for _, idx := range b.argsMapping { + if idx < len(rangeStartArgs) { + explodedArgs = append(explodedArgs, rangeStartArgs[idx]) + } else { + explodedArgs = append(explodedArgs, rangeEndArgs[idx-len(rangeStartArgs)]) + } + } + return b.preparedStatement, explodedArgs, nil +} + +type MoveTableCopyInsertQueryBuilder struct { + preparedStatement string + valueListPlaceholder string + valueListSize int +} + +func NewMoveTableCopyInsertQueryBuilder(targetDatabaseName, targetTableName string, columns *ColumnList) (*MoveTableCopyInsertQueryBuilder, error) { + targetDatabaseName = EscapeName(targetDatabaseName) + targetTableName = EscapeName(targetTableName) + columnsNames := columns.Names() + for i := range columnsNames { + columnsNames[i] = EscapeName(columnsNames[i]) + } + sharedColumnsListing := strings.Join(columnsNames, ", ") + valueListPlaceholder := "(" + strings.Join(buildColumnsPreparedValues(columns), ", ") + ")" + valueListSize := len(columnsNames) + stmt := fmt.Sprintf(` + insert /* gh-ost %s.%s */ ignore + into + %s.%s + (%s) + values + `, + targetDatabaseName, targetTableName, + targetDatabaseName, targetTableName, + sharedColumnsListing, + ) + return &MoveTableCopyInsertQueryBuilder{ + preparedStatement: stmt, + valueListPlaceholder: valueListPlaceholder, + valueListSize: valueListSize, + }, nil +} + +func (b *MoveTableCopyInsertQueryBuilder) BuildQuery(values []*ColumnValues) (string, []any, error) { + var explodedArgs []any + var builder strings.Builder + builder.WriteString(b.preparedStatement) + for i, value := range values { + if len(value.AbstractValues()) != b.valueListSize { + return "", nil, fmt.Errorf("got %d column values but expected %d", len(value.AbstractValues()), b.valueListSize) + } + if i > 0 { + builder.WriteString(",\n") + } + builder.WriteString(b.valueListPlaceholder) + explodedArgs = append(explodedArgs, value.AbstractValues()...) + } + return builder.String(), explodedArgs, nil +} + func BuildUniqueKeyRangeEndPreparedQueryViaOffset(databaseName, tableName string, uniqueKeyColumns *ColumnList, rangeStartArgs, rangeEndArgs []interface{}, chunkSize int64, includeRangeStartValues bool, hint string) (result string, explodedArgs []interface{}, err error) { if uniqueKeyColumns.Len() == 0 { return "", explodedArgs, fmt.Errorf("got 0 columns in BuildUniqueKeyRangeEndPreparedQuery") diff --git a/go/sql/builder_test.go b/go/sql/builder_test.go index be7075927..0fcf31441 100644 --- a/go/sql/builder_test.go +++ b/go/sql/builder_test.go @@ -1102,6 +1102,246 @@ func TestBuildDMLUpdateQuerySignedUnsigned(t *testing.T) { } } +func TestMoveTableCopySelectQueryBuilder(t *testing.T) { + t.Run("single column unique key", func(t *testing.T) { + sharedColumns := NewColumnList([]string{"id", "name", "position"}) + uniqueKeyColumns := NewColumnList([]string{"id"}) + + builder, err := NewMoveTableCopySelectQueryBuilder("mydb", "tbl", sharedColumns, "PRIMARY", uniqueKeyColumns, true) + require.NoError(t, err) + + query, args, err := builder.BuildQuery([]any{3}, []any{103}) + require.NoError(t, err) + + expected := ` + select /* gh-ost mydb.tbl */ id, name, position + from + mydb.tbl + force index (PRIMARY) + where + (((id > ?) or ((id = ?))) and ((id < ?) or ((id = ?)))) + ` + require.Equal(t, normalizeQuery(expected), normalizeQuery(query)) + require.Equal(t, []any{3, 3, 103, 103}, args) + }) + + t.Run("single column unique key without range start", func(t *testing.T) { + sharedColumns := NewColumnList([]string{"id", "name", "position"}) + uniqueKeyColumns := NewColumnList([]string{"id"}) + + builder, err := NewMoveTableCopySelectQueryBuilder("mydb", "tbl", sharedColumns, "PRIMARY", uniqueKeyColumns, false) + require.NoError(t, err) + + query, args, err := builder.BuildQuery([]any{3}, []any{103}) + require.NoError(t, err) + + expected := ` + select /* gh-ost mydb.tbl */ id, name, position + from + mydb.tbl + force index (PRIMARY) + where + (((id > ?)) and ((id < ?) or ((id = ?)))) + ` + require.Equal(t, normalizeQuery(expected), normalizeQuery(query)) + require.Equal(t, []any{3, 103, 103}, args) + }) + + t.Run("compound unique key", func(t *testing.T) { + sharedColumns := NewColumnList([]string{"id", "name", "position"}) + uniqueKeyColumns := NewColumnList([]string{"name", "position"}) + + builder, err := NewMoveTableCopySelectQueryBuilder("mydb", "tbl", sharedColumns, "name_position_uidx", uniqueKeyColumns, true) + require.NoError(t, err) + + query, args, err := builder.BuildQuery([]any{3, 17}, []any{103, 117}) + require.NoError(t, err) + + expected := ` + select /* gh-ost mydb.tbl */ id, name, position + from + mydb.tbl + force index (name_position_uidx) + where + (((name > ?) or (((name = ?)) AND (position > ?)) or ((name = ?) and (position = ?))) + and ((name < ?) or (((name = ?)) AND (position < ?)) or ((name = ?) and (position = ?)))) + ` + require.Equal(t, normalizeQuery(expected), normalizeQuery(query)) + require.Equal(t, []any{3, 3, 17, 3, 17, 103, 103, 117, 103, 117}, args) + }) + + t.Run("reuses prepared statement across calls", func(t *testing.T) { + sharedColumns := NewColumnList([]string{"id", "name"}) + uniqueKeyColumns := NewColumnList([]string{"id"}) + + builder, err := NewMoveTableCopySelectQueryBuilder("mydb", "tbl", sharedColumns, "PRIMARY", uniqueKeyColumns, true) + require.NoError(t, err) + + query1, args1, err := builder.BuildQuery([]any{1}, []any{10}) + require.NoError(t, err) + query2, args2, err := builder.BuildQuery([]any{11}, []any{20}) + require.NoError(t, err) + + require.Equal(t, query1, query2) + require.Equal(t, []any{1, 1, 10, 10}, args1) + require.Equal(t, []any{11, 11, 20, 20}, args2) + }) + + t.Run("wrong args count", func(t *testing.T) { + sharedColumns := NewColumnList([]string{"id", "name"}) + uniqueKeyColumns := NewColumnList([]string{"id"}) + + builder, err := NewMoveTableCopySelectQueryBuilder("mydb", "tbl", sharedColumns, "PRIMARY", uniqueKeyColumns, true) + require.NoError(t, err) + + _, _, err = builder.BuildQuery([]any{1, 2}, []any{10}) + require.Error(t, err) + }) + + t.Run("mismatched start and end args count", func(t *testing.T) { + sharedColumns := NewColumnList([]string{"id", "name", "position"}) + uniqueKeyColumns := NewColumnList([]string{"name", "position"}) + + builder, err := NewMoveTableCopySelectQueryBuilder("mydb", "tbl", sharedColumns, "name_position_uidx", uniqueKeyColumns, true) + require.NoError(t, err) + + // Total args count matches argsCount (4), but start and end counts differ. + _, _, err = builder.BuildQuery([]any{1, 2, 3}, []any{10}) + require.Error(t, err) + require.Contains(t, err.Error(), "mismatched number of start and end args") + }) +} + +func BenchmarkMoveTableCopySelectQueryBuilderBuildQuery(b *testing.B) { + sharedColumns := NewColumnList([]string{"id", "name", "position"}) + uniqueKeyColumns := NewColumnList([]string{"name", "position"}) + + builder, err := NewMoveTableCopySelectQueryBuilder("mydb", "tbl", sharedColumns, "name_position_uidx", uniqueKeyColumns, true) + if err != nil { + b.Fatal(err) + } + + rangeStartArgs := []any{3, 17} + rangeEndArgs := []any{103, 117} + + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, _, err := builder.BuildQuery(rangeStartArgs, rangeEndArgs) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMoveTableCopyInsertQueryBuilder(t *testing.T) { + t.Run("single row", func(t *testing.T) { + sharedColumns := NewColumnList([]string{"id", "name", "position"}) + + builder, err := NewMoveTableCopyInsertQueryBuilder("mydb", "ghost", sharedColumns) + require.NoError(t, err) + + values := []*ColumnValues{ + ToColumnValues([]interface{}{1, "alice", 10}), + } + query, args, err := builder.BuildQuery(values) + require.NoError(t, err) + + expected := ` + insert /* gh-ost mydb.ghost */ ignore + into + mydb.ghost + (id, name, position) + values + (?, ?, ?) + ` + require.Equal(t, normalizeQuery(expected), normalizeQuery(query)) + require.Equal(t, []any{1, "alice", 10}, args) + }) + + t.Run("multiple rows", func(t *testing.T) { + sharedColumns := NewColumnList([]string{"id", "name", "position"}) + + builder, err := NewMoveTableCopyInsertQueryBuilder("mydb", "ghost", sharedColumns) + require.NoError(t, err) + + values := []*ColumnValues{ + ToColumnValues([]interface{}{1, "alice", 10}), + ToColumnValues([]interface{}{2, "bob", 20}), + ToColumnValues([]interface{}{3, "carol", 30}), + } + query, args, err := builder.BuildQuery(values) + require.NoError(t, err) + + expected := ` + insert /* gh-ost mydb.ghost */ ignore + into + mydb.ghost + (id, name, position) + values + (?, ?, ?), + (?, ?, ?), + (?, ?, ?) + ` + require.Equal(t, normalizeQuery(expected), normalizeQuery(query)) + require.Equal(t, []any{1, "alice", 10, 2, "bob", 20, 3, "carol", 30}, args) + }) + + t.Run("wrong column count", func(t *testing.T) { + sharedColumns := NewColumnList([]string{"id", "name", "position"}) + + builder, err := NewMoveTableCopyInsertQueryBuilder("mydb", "ghost", sharedColumns) + require.NoError(t, err) + + values := []*ColumnValues{ + ToColumnValues([]interface{}{1, "alice"}), + } + _, _, err = builder.BuildQuery(values) + require.Error(t, err) + }) + + t.Run("reuses prepared statement", func(t *testing.T) { + sharedColumns := NewColumnList([]string{"id", "name"}) + + builder, err := NewMoveTableCopyInsertQueryBuilder("mydb", "ghost", sharedColumns) + require.NoError(t, err) + + values1 := []*ColumnValues{ToColumnValues([]interface{}{1, "a"})} + values2 := []*ColumnValues{ToColumnValues([]interface{}{2, "b"})} + + query1, args1, err := builder.BuildQuery(values1) + require.NoError(t, err) + query2, args2, err := builder.BuildQuery(values2) + require.NoError(t, err) + + require.Equal(t, query1, query2) + require.Equal(t, []any{1, "a"}, args1) + require.Equal(t, []any{2, "b"}, args2) + }) +} + +func BenchmarkMoveTableCopyInsertQueryBuilderBuildQuery(b *testing.B) { + sharedColumns := NewColumnList([]string{"id", "name", "position"}) + + builder, err := NewMoveTableCopyInsertQueryBuilder("mydb", "ghost", sharedColumns) + if err != nil { + b.Fatal(err) + } + + values := []*ColumnValues{ + ToColumnValues([]interface{}{1, "alice", 10}), + ToColumnValues([]interface{}{2, "bob", 20}), + ToColumnValues([]interface{}{3, "carol", 30}), + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, _, err := builder.BuildQuery(values) + if err != nil { + b.Fatal(err) + } + } +} + func TestCheckpointQueryBuilder(t *testing.T) { databaseName := "mydb" tableName := "_tbl_ghk" From 42590a5abc8f4e6cbc2f8da6e688457c88ec6dd1 Mon Sep 17 00:00:00 2001 From: Zach Sierakowski Date: Thu, 11 Jun 2026 14:38:15 +0000 Subject: [PATCH 03/23] move-tables: implement target-table migration flow Enable end-to-end target-table copying, route DML events to the target, and harden move-table initialization and table selection. Refs: #1705 Co-authored-by: Chris Kirkland Co-authored-by: Daniel Joos Co-authored-by: womoruyi --- go/base/context.go | 36 +++- go/base/context_test.go | 56 ++++++ go/cmd/gh-ost/main.go | 15 +- go/logic/applier.go | 221 ++++++++++++++++------ go/logic/applier_test.go | 139 +++++++++++++- go/logic/inspect.go | 103 +++++----- go/logic/migrator.go | 182 ++++++++++++------ go/logic/migrator_test.go | 4 +- localtests/docker-compose-move-tables.yml | 57 ++++++ localtests/move-tables/create.sql | 34 ++++ script/move-tables/README.md | 180 ++++++++++++++++++ script/move-tables/mysql-source-primary | 6 + script/move-tables/mysql-source-replica | 6 + script/move-tables/mysql-target-primary | 6 + script/move-tables/mysql-target-replica | 6 + script/move-tables/setup | 130 +++++++++++++ script/move-tables/teardown | 20 ++ 17 files changed, 1025 insertions(+), 176 deletions(-) create mode 100644 localtests/docker-compose-move-tables.yml create mode 100644 localtests/move-tables/create.sql create mode 100644 script/move-tables/README.md create mode 100755 script/move-tables/mysql-source-primary create mode 100755 script/move-tables/mysql-source-replica create mode 100755 script/move-tables/mysql-target-primary create mode 100755 script/move-tables/mysql-target-replica create mode 100755 script/move-tables/setup create mode 100755 script/move-tables/teardown diff --git a/go/base/context.go b/go/base/context.go index aebf4a0db..48eeded64 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -404,6 +404,24 @@ func (mctx *MigrationContext) GetGhostTableName() string { } } +// GetTargetTableName generates the name of the target table, based on original table name and +// the migration context (i.e. move-tables mode). +func (mctx *MigrationContext) GetTargetTableName() string { + if mctx.IsMoveTablesMode() { + return mctx.MoveTables.TableNames[0] + } + return mctx.GetGhostTableName() +} + +// GetTargetDatabaseName fetches the name of the target database, which defaults to the original +// database name unless we're in move-tables mode. +func (mctx *MigrationContext) GetTargetDatabaseName() string { + if mctx.IsMoveTablesMode() { + return mctx.MoveTables.TargetDatabase + } + return mctx.DatabaseName +} + // GetOldTableName generates the name of the "old" table, into which the original table is renamed. func (mctx *MigrationContext) GetOldTableName() string { var tableName string @@ -945,11 +963,27 @@ func (mctx *MigrationContext) ApplyCredentials() { // Override mctx.InspectorConnectionConfig.Password = mctx.CliPassword } + + if mctx.IsMoveTablesMode() { + // Derive the applier config from the inspector config, but point it at + // the target host and override credentials from the target CLI args. + mctx.MoveTables.ConnectionConfig = mctx.InspectorConnectionConfig.DuplicateCredentials(mysql.InstanceKey{ + Hostname: mctx.MoveTables.TargetHost, + Port: mctx.MoveTables.TargetPort, + }) + mctx.MoveTables.ConnectionConfig.User = mctx.MoveTables.TargetUser + mctx.MoveTables.ConnectionConfig.Password = mctx.MoveTables.TargetPass + } } func (mctx *MigrationContext) SetupTLS() error { if mctx.UseTLS { - return mctx.InspectorConnectionConfig.UseTLS(mctx.TLSCACertificate, mctx.TLSCertificate, mctx.TLSKey, mctx.TLSAllowInsecure) + if err := mctx.InspectorConnectionConfig.UseTLS(mctx.TLSCACertificate, mctx.TLSCertificate, mctx.TLSKey, mctx.TLSAllowInsecure); err != nil { + return err + } + if mctx.IsMoveTablesMode() && mctx.MoveTables.ConnectionConfig != nil { + return mctx.MoveTables.ConnectionConfig.UseTLS(mctx.TLSCACertificate, mctx.TLSCertificate, mctx.TLSKey, mctx.TLSAllowInsecure) + } } return nil } diff --git a/go/base/context_test.go b/go/base/context_test.go index a9f62150d..ffbc174a4 100644 --- a/go/base/context_test.go +++ b/go/base/context_test.go @@ -216,6 +216,62 @@ func TestReadConfigFile(t *testing.T) { } } +func TestApplyCredentialsMoveTablesDerivesConnectionConfig(t *testing.T) { + ctx := NewMigrationContext() + ctx.MoveTables.TableNames = []string{"some_table"} + ctx.MoveTables.TargetHost = "target-host" + ctx.MoveTables.TargetPort = 3307 + ctx.MoveTables.TargetUser = "target-user" + ctx.MoveTables.TargetPass = "target-pass" + + ctx.InspectorConnectionConfig.Key.Hostname = "source-host" + ctx.InspectorConnectionConfig.Key.Port = 3306 + ctx.InspectorConnectionConfig.User = "source-user" + ctx.InspectorConnectionConfig.Password = "source-pass" + ctx.InspectorConnectionConfig.Timeout = 12.5 + ctx.InspectorConnectionConfig.TransactionIsolation = "REPEATABLE-READ" + ctx.InspectorConnectionConfig.Charset = "utf8mb4" + + ctx.ApplyCredentials() + + got := ctx.MoveTables.ConnectionConfig + require.NotNil(t, got) + require.Equal(t, "target-host", got.Key.Hostname) + require.Equal(t, 3307, got.Key.Port) + require.Equal(t, "target-user", got.User) + require.Equal(t, "target-pass", got.Password) + require.Equal(t, 12.5, got.Timeout) + require.Equal(t, "REPEATABLE-READ", got.TransactionIsolation) + require.Equal(t, "utf8mb4", got.Charset) + require.NotNil(t, got.ImpliedKey) + require.Equal(t, "target-host", got.ImpliedKey.Hostname) + require.Equal(t, 3307, got.ImpliedKey.Port) +} + +func TestSetupTLSAppliesToMoveTablesConfig(t *testing.T) { + ctx := NewMigrationContext() + ctx.UseTLS = true + ctx.TLSAllowInsecure = true + ctx.MoveTables.TableNames = []string{"some_table"} + ctx.MoveTables.TargetHost = "target-host" + ctx.MoveTables.TargetPort = 3307 + ctx.MoveTables.TargetUser = "target-user" + ctx.MoveTables.TargetPass = "target-pass" + + ctx.InspectorConnectionConfig.Key.Hostname = "source-host" + ctx.InspectorConnectionConfig.Key.Port = 3306 + ctx.InspectorConnectionConfig.User = "source-user" + ctx.InspectorConnectionConfig.Password = "source-pass" + + ctx.ApplyCredentials() + require.NoError(t, ctx.SetupTLS()) + + require.NotNil(t, ctx.InspectorConnectionConfig.TLSConfig()) + require.NotNil(t, ctx.MoveTables.ConnectionConfig.TLSConfig()) + require.Equal(t, "source-host", ctx.InspectorConnectionConfig.TLSConfig().ServerName) + require.Equal(t, "target-host", ctx.MoveTables.ConnectionConfig.TLSConfig().ServerName) +} + func TestSetAbortError_StoresFirstError(t *testing.T) { ctx := NewMigrationContext() diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index 904240b44..73e4f1b76 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -20,6 +20,7 @@ import ( "github.com/github/gh-ost/go/base" "github.com/github/gh-ost/go/logic" "github.com/github/gh-ost/go/metrics" + "github.com/github/gh-ost/go/mysql" "github.com/github/gh-ost/go/sql" _ "github.com/go-sql-driver/mysql" "github.com/openark/golib/log" @@ -234,12 +235,6 @@ func main() { migrationContext.Log.SetLevel(log.ERROR) } - if err := migrationContext.SetConnectionConfig(*storageEngine); err != nil { - migrationContext.Log.Fatale(err) - } - - migrationContext.SetConnectionCharset(*charset) - if migrationContext.AlterStatement == "" && !migrationContext.Revert && *moveTables == "" { log.Fatal("--alter must be provided and statement must not be empty, or --revert must be used, or --move-tables must be used") } @@ -380,6 +375,7 @@ func main() { // For now, we only support moving a single table at a time. log.Fatal("--move-tables currently supports only a single table") } + if migrationContext.MoveTables.TargetUser == "" { migrationContext.MoveTables.TargetUser = migrationContext.CliUser } @@ -389,8 +385,15 @@ func main() { if migrationContext.MoveTables.TargetDatabase == "" { migrationContext.MoveTables.TargetDatabase = migrationContext.DatabaseName } + migrationContext.MoveTables.ConnectionConfig = mysql.NewConnectionConfig() } + if err := migrationContext.SetConnectionConfig(*storageEngine); err != nil { + migrationContext.Log.Fatale(err) + } + + migrationContext.SetConnectionCharset(*charset) + switch *cutOver { case "atomic", "default", "": migrationContext.CutOverType = base.CutOverAtomic diff --git a/go/logic/applier.go b/go/logic/applier.go index 4e1a547f3..bc355549e 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -116,7 +116,7 @@ func NewApplier(migrationContext *base.MigrationContext) *Applier { // hence the optional table name prefix. Metacharacters in table/index names are escaped to avoid // regex syntax errors. func (apl *Applier) compileMigrationKeyWarningRegex() (*regexp.Regexp, error) { - escapedTable := regexp.QuoteMeta(apl.migrationContext.GetGhostTableName()) + escapedTable := regexp.QuoteMeta(apl.migrationContext.GetTargetTableName()) escapedKey := regexp.QuoteMeta(apl.migrationContext.UniqueKey.NameInGhostTable) migrationUniqueKeyPattern := fmt.Sprintf(`for key '(%s\.)?%s'`, escapedTable, escapedKey) migrationKeyRegex, err := regexp.Compile(migrationUniqueKeyPattern) @@ -127,7 +127,7 @@ func (apl *Applier) compileMigrationKeyWarningRegex() (*regexp.Regexp, error) { } func (apl *Applier) InitDBConnections() (err error) { - applierUri := apl.connectionConfig.GetDBUri(apl.migrationContext.DatabaseName) + applierUri := apl.connectionConfig.GetDBUri(apl.migrationContext.GetTargetDatabaseName()) uriWithMulti := fmt.Sprintf("%s&multiStatements=true", applierUri) if apl.db, _, err = mysql.GetDB(apl.migrationContext.Uuid, uriWithMulti); err != nil { return err @@ -155,11 +155,23 @@ func (apl *Applier) InitDBConnections() (err error) { apl.connectionConfig.ImpliedKey = impliedKey } } - if err := apl.readTableColumns(); err != nil { - return err + if !apl.migrationContext.IsMoveTablesMode() { + // read target table columns from applier + if err := apl.readTableColumns(); err != nil { + return err + } + } + if apl.moveTablesConnectionConfig != nil { + moveTablesURI := apl.moveTablesConnectionConfig.GetDBUri(apl.migrationContext.GetTargetDatabaseName()) + "&multiStatements=true" + if apl.moveTablesTargetDB, _, err = mysql.GetDB(apl.migrationContext.Uuid, moveTablesURI); err != nil { + return err + } + if _, err := base.ValidateConnection(apl.moveTablesTargetDB, apl.moveTablesConnectionConfig, apl.migrationContext, apl.name); err != nil { + return err + } } if apl.moveTablesConnectionConfig != nil { - moveTablesURI := apl.moveTablesConnectionConfig.GetDBUri(apl.migrationContext.MoveTables.TargetDatabase) + "&multiStatements=true" + moveTablesURI := apl.moveTablesConnectionConfig.GetDBUri(apl.migrationContext.GetTargetDatabaseName()) + "&multiStatements=true" if apl.moveTablesTargetDB, _, err = mysql.GetDB(apl.migrationContext.Uuid, moveTablesURI); err != nil { return err } @@ -186,7 +198,7 @@ func buildMigrationLockName(db, table string) string { // preventing two gh-ost processes from migrating the same table concurrently // on the same MySQL server. func (apl *Applier) AcquireMigrationLock(ctx context.Context) error { - lockName := buildMigrationLockName(apl.migrationContext.DatabaseName, apl.migrationContext.OriginalTableName) + lockName := buildMigrationLockName(apl.migrationContext.DatabaseName, apl.originalTableName()) // Use a dedicated *sql.DB so the pinned connection does not consume a // slot in apl.db's small pool (mysql.MaxDBPoolConnections). @@ -224,10 +236,10 @@ func (apl *Applier) AcquireMigrationLock(ctx context.Context) error { lockDB.Close() if holderID.Valid { return fmt.Errorf("another gh-ost process is already migrating `%s`.`%s`: migration lock %s held by connection id %d", - apl.migrationContext.DatabaseName, apl.migrationContext.OriginalTableName, lockName, holderID.Int64) + apl.migrationContext.DatabaseName, apl.originalTableName(), lockName, holderID.Int64) } return fmt.Errorf("another gh-ost process is already migrating `%s`.`%s`: migration lock %s is held", - apl.migrationContext.DatabaseName, apl.migrationContext.OriginalTableName, lockName) + apl.migrationContext.DatabaseName, apl.originalTableName(), lockName) } apl.migrationLockConn = conn @@ -315,12 +327,8 @@ func (apl *Applier) releaseMigrationLock() { } func (apl *Applier) prepareQueries() (err error) { - targetDatabaseName := apl.migrationContext.DatabaseName - targetTableName := apl.migrationContext.GetGhostTableName() - if apl.migrationContext.IsMoveTablesMode() { - targetDatabaseName = apl.migrationContext.MoveTables.TargetDatabase - targetTableName = apl.migrationContext.OriginalTableName - } + targetDatabaseName := apl.migrationContext.GetTargetDatabaseName() + targetTableName := apl.migrationContext.GetTargetTableName() if apl.dmlDeleteQueryBuilder, err = sql.NewDMLDeleteQueryBuilder( targetDatabaseName, @@ -361,7 +369,7 @@ func (apl *Applier) prepareQueries() (err error) { if apl.migrationContext.IsMoveTablesMode() { if apl.moveTablesCopySelectFirstQueryBuilder, err = sql.NewMoveTableCopySelectQueryBuilder( apl.migrationContext.DatabaseName, - apl.migrationContext.OriginalTableName, + apl.originalTableName(), apl.migrationContext.OriginalTableColumns, apl.migrationContext.UniqueKey.Name, &apl.migrationContext.UniqueKey.Columns, @@ -371,7 +379,7 @@ func (apl *Applier) prepareQueries() (err error) { } if apl.moveTablesCopySelectNextQueryBuilder, err = sql.NewMoveTableCopySelectQueryBuilder( apl.migrationContext.DatabaseName, - apl.migrationContext.OriginalTableName, + apl.originalTableName(), apl.migrationContext.OriginalTableColumns, apl.migrationContext.UniqueKey.Name, &apl.migrationContext.UniqueKey.Columns, @@ -426,7 +434,7 @@ func (apl *Applier) generateSqlModeQuery() string { func (apl *Applier) generateInstantDDLQuery() string { return fmt.Sprintf(`ALTER /* gh-ost */ TABLE %s.%s %s, ALGORITHM=INSTANT`, sql.EscapeName(apl.migrationContext.DatabaseName), - sql.EscapeName(apl.migrationContext.OriginalTableName), + sql.EscapeName(apl.originalTableName()), apl.migrationContext.AlterStatementOptions, ) } @@ -434,7 +442,7 @@ func (apl *Applier) generateInstantDDLQuery() string { // readTableColumns reads table columns on applier func (apl *Applier) readTableColumns() (err error) { apl.migrationContext.Log.Infof("Examining table structure on applier") - apl.migrationContext.OriginalTableColumnsOnApplier, _, err = mysql.GetTableColumns(apl.db, apl.migrationContext.DatabaseName, apl.migrationContext.OriginalTableName) + apl.migrationContext.OriginalTableColumnsOnApplier, _, err = mysql.GetTableColumns(apl.db, apl.migrationContext.DatabaseName, apl.originalTableName()) if err != nil { return err } @@ -457,6 +465,13 @@ func (apl *Applier) tableExists(tableName string) (tableFound bool) { return (m != nil) } +func (apl *Applier) originalTableName() string { + if apl.migrationContext.IsMoveTablesMode() { + return apl.migrationContext.MoveTables.TableNames[0] + } + return apl.migrationContext.OriginalTableName +} + // ValidateOrDropExistingTables verifies ghost and changelog tables do not exist, // or attempts to drop them if instructed to. func (apl *Applier) ValidateOrDropExistingTables() error { @@ -538,17 +553,19 @@ func retryOnLockWaitTimeout(operation func() error, maxRetries int64, logger bas return err } -// CreateGhostTable creates the ghost table on the applier host -func (apl *Applier) CreateGhostTable() error { +// createTargetTable creates the table on the applier host to which the applier will +// apply changes. +func (apl *Applier) createTargetTable(targetTableName string) error { + targetDatabase := apl.migrationContext.GetTargetDatabaseName() query := fmt.Sprintf(`create /* gh-ost */ table %s.%s like %s.%s`, + sql.EscapeName(targetDatabase), + sql.EscapeName(targetTableName), sql.EscapeName(apl.migrationContext.DatabaseName), - sql.EscapeName(apl.migrationContext.GetGhostTableName()), - sql.EscapeName(apl.migrationContext.DatabaseName), - sql.EscapeName(apl.migrationContext.OriginalTableName), + sql.EscapeName(apl.originalTableName()), ) - apl.migrationContext.Log.Infof("Creating ghost table %s.%s", - sql.EscapeName(apl.migrationContext.DatabaseName), - sql.EscapeName(apl.migrationContext.GetGhostTableName()), + apl.migrationContext.Log.Infof("Creating target table %s.%s", + sql.EscapeName(targetDatabase), + sql.EscapeName(targetTableName), ) err := func() error { @@ -567,7 +584,7 @@ func (apl *Applier) CreateGhostTable() error { if _, err := tx.Exec(query); err != nil { return err } - apl.migrationContext.Log.Infof("Ghost table created") + apl.migrationContext.Log.Infof("Target table created") if err := tx.Commit(); err != nil { // Neither SET SESSION nor ALTER are really transactional, so strictly speaking // there's no need to commit; but let's do this the legit way anyway. @@ -651,6 +668,59 @@ func (apl *Applier) AnalyzeGhostTable() error { return nil } +// createTargetTableFromStatement creates the table on the applier host to which the applier will +// apply changes. +func (apl *Applier) createTargetTableFromStatement(targetTableName, createStatement string) error { + targetDatabase := apl.migrationContext.GetTargetDatabaseName() + apl.migrationContext.Log.Infof("Creating target table %s.%s", + sql.EscapeName(targetDatabase), + sql.EscapeName(targetTableName), + ) + + err := func() error { + tx, err := apl.db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + + sessionQuery := fmt.Sprintf(`SET SESSION time_zone = '%s'`, apl.migrationContext.ApplierTimeZone) + sessionQuery = fmt.Sprintf("%s, %s", sessionQuery, apl.generateSqlModeQuery()) + + if _, err := tx.Exec(sessionQuery); err != nil { + return err + } + if _, err := tx.Exec(createStatement); err != nil { + return err + } + apl.migrationContext.Log.Infof("Target table created") + if err := tx.Commit(); err != nil { + // Neither SET SESSION nor ALTER are really transactional, so strictly speaking + // there's no need to commit; but let's do this the legit way anyway. + return err + } + return nil + }() + + return err +} + +// CreateGhostTable creates the ghost table on the applier host +func (apl *Applier) CreateGhostTable() error { + if apl.migrationContext.IsMoveTablesMode() { + return errors.New("CreateGhostTable is not available in MoveTables mode") + } + return apl.createTargetTable(apl.migrationContext.GetGhostTableName()) +} + +// CreateTargetTable creates the target table on the target host (for move-tables) +func (apl *Applier) CreateTargetTable(createStatement string) error { + if !apl.migrationContext.IsMoveTablesMode() { + return errors.New("CreateTargetTable is only available in MoveTables mode") + } + return apl.createTargetTableFromStatement(apl.originalTableName(), createStatement) +} + // AlterGhost applies `alter` statement on ghost table func (apl *Applier) AlterGhost() error { query := fmt.Sprintf(`alter /* gh-ost */ table %s.%s %s`, @@ -857,7 +927,7 @@ func (apl *Applier) createTriggers(tableName string) error { sql.EscapeName(tableName), trigger.Statement, ) - apl.migrationContext.Log.Infof("Createing trigger %s on %s.%s", + apl.migrationContext.Log.Infof("Creating trigger %s on %s.%s", sql.EscapeName(triggerName), sql.EscapeName(apl.migrationContext.DatabaseName), sql.EscapeName(tableName), @@ -873,8 +943,10 @@ func (apl *Applier) createTriggers(tableName string) error { // CreateTriggers creates the original triggers on applier host func (apl *Applier) CreateTriggersOnGhost() error { - err := apl.createTriggers(apl.migrationContext.GetGhostTableName()) - return err + if err := apl.createTriggers(apl.migrationContext.GetGhostTableName()); err != nil { + return fmt.Errorf("error creating triggers on ghost table: %w", err) + } + return nil } // DropChangelogTable drops the changelog table on the applier host @@ -894,12 +966,22 @@ func (apl *Applier) DropOldTable() error { // DropGhostTable drops the ghost table on the applier host func (apl *Applier) DropGhostTable() error { - return apl.dropTable(apl.migrationContext.GetGhostTableName()) + if err := apl.dropTable(apl.migrationContext.GetGhostTableName()); err != nil { + return fmt.Errorf("error dropping ghost table: %w", err) + } + return nil } // WriteChangelog writes a value to the changelog table. -// It returns the hint as given, for convenience +// It returns the hint (or an empty string in move-tables mode), for convenience func (apl *Applier) WriteChangelog(hint, value string) (string, error) { + // In move-tables mode, there is no changelog table (§1.2). All changelog + // writes are no-ops. This is a single chokepoint rather than per-caller + // guards to prevent drift when new callers are added. + if apl.migrationContext.IsMoveTablesMode() { + return "", nil + } + explicitId := 0 switch hint { case "heartbeat": @@ -992,6 +1074,11 @@ func (apl *Applier) ReadLastCheckpoint() (*Checkpoint, error) { // InitiateHeartbeat creates a heartbeat cycle, writing to the changelog table. // Apl is done asynchronously func (apl *Applier) InitiateHeartbeat() { + // In move-tables mode, there is no heartbeat table (§1.2). + if apl.migrationContext.IsMoveTablesMode() { + return + } + var numSuccessiveFailures int64 injectHeartbeat := func() error { if atomic.LoadInt64(&apl.migrationContext.HibernateUntil) > 0 { @@ -1060,8 +1147,7 @@ func (apl *Applier) ExecuteThrottleQuery() (int64, error) { // readMigrationMinValues returns the minimum values to be iterated on rowcopy func (apl *Applier) readMigrationMinValues(tx *gosql.Tx, uniqueKey *sql.UniqueKey) error { - apl.migrationContext.Log.Debugf("Reading migration range according to key: %s", uniqueKey.Name) - query, err := sql.BuildUniqueKeyMinValuesPreparedQuery(apl.migrationContext.DatabaseName, apl.migrationContext.OriginalTableName, uniqueKey) + query, err := sql.BuildUniqueKeyMinValuesPreparedQuery(apl.migrationContext.DatabaseName, apl.originalTableName(), uniqueKey) if err != nil { return err } @@ -1086,7 +1172,7 @@ func (apl *Applier) readMigrationMinValues(tx *gosql.Tx, uniqueKey *sql.UniqueKe // readMigrationMaxValues returns the maximum values to be iterated on rowcopy func (apl *Applier) readMigrationMaxValues(tx *gosql.Tx, uniqueKey *sql.UniqueKey) error { apl.migrationContext.Log.Debugf("Reading migration range according to key: %s", uniqueKey.Name) - query, err := sql.BuildUniqueKeyMaxValuesPreparedQuery(apl.migrationContext.DatabaseName, apl.migrationContext.OriginalTableName, uniqueKey) + query, err := sql.BuildUniqueKeyMaxValuesPreparedQuery(apl.migrationContext.DatabaseName, apl.originalTableName(), uniqueKey) if err != nil { return err } @@ -1125,12 +1211,17 @@ Detail description of the lost data in mysql two-phase commit issue by @Fanduzi: will not be run. When the changelog writes successfully, the ReadMigrationRangeValues will read the newly inserted data, thus Avoiding data loss due to the above problem. */ -func (apl *Applier) ReadMigrationRangeValues() error { +func (apl *Applier) ReadMigrationRangeValues(db *gosql.DB) error { if _, err := apl.WriteChangelogState(string(ReadMigrationRangeValues)); err != nil { return err } - tx, err := apl.db.Begin() + if db == nil { + // default to reading from applier database + db = apl.db + } + + tx, err := db.Begin() if err != nil { return err } @@ -1150,7 +1241,7 @@ func (apl *Applier) ReadMigrationRangeValues() error { // which will be used for copying the next chunk of rows. Ir returns "false" if there is // no further chunk to work through, i.e. we're past the last chunk and are done with // iterating the range (and thus done with copying row chunks) -func (apl *Applier) CalculateNextIterationRangeEndValues() (hasFurtherRange bool, err error) { +func (apl *Applier) CalculateNextIterationRangeEndValues(db *gosql.DB) (hasFurtherRange bool, err error) { for i := 0; i < 2; i++ { buildFunc := sql.BuildUniqueKeyRangeEndPreparedQueryViaOffset if i == 1 { @@ -1158,7 +1249,7 @@ func (apl *Applier) CalculateNextIterationRangeEndValues() (hasFurtherRange bool } query, explodedArgs, err := buildFunc( apl.migrationContext.DatabaseName, - apl.migrationContext.OriginalTableName, + apl.originalTableName(), &apl.migrationContext.UniqueKey.Columns, apl.migrationContext.MigrationIterationRangeMinValues.AbstractValues(), apl.migrationContext.MigrationRangeMaxValues.AbstractValues(), @@ -1171,7 +1262,12 @@ func (apl *Applier) CalculateNextIterationRangeEndValues() (hasFurtherRange bool } queryStartTime := time.Now() - rows, err := apl.db.Query(query, explodedArgs...) + if db == nil { + // default to applier database if not provided + db = apl.db + } + + rows, err := db.Query(query, explodedArgs...) if err != nil { metrics.RecordQueryDuration(apl.migrationContext.Metrics, "source", "range_select", time.Since(queryStartTime), err) return hasFurtherRange, err @@ -1208,7 +1304,7 @@ func (apl *Applier) ApplyIterationInsertQuery() (chunkSize int64, rowsAffected i query, explodedArgs, err := sql.BuildRangeInsertPreparedQuery( apl.migrationContext.DatabaseName, - apl.migrationContext.OriginalTableName, + apl.originalTableName(), apl.migrationContext.GetGhostTableName(), apl.migrationContext.SharedColumns.Names(), apl.migrationContext.MappedSharedColumns.Names(), @@ -1299,13 +1395,20 @@ func (apl *Applier) ApplyIterationInsertQuery() (chunkSize int64, rowsAffected i // ApplyIterationMoveTableCopyQueries issues a SELECT query on the original table and an INSERT query on the target table, // copying a chunk of rows. It is used when `--move-tables` is specified, instead of ApplyIterationInsertQuery. -func (apl *Applier) ApplyIterationMoveTableCopyQueries() (chunkSize int64, rowsAffected int64, duration time.Duration, err error) { +func (apl *Applier) ApplyIterationMoveTableCopyQueries(sourceDB *gosql.DB) (chunkSize int64, rowsAffected int64, duration time.Duration, err error) { startTime := time.Now() chunkSize = atomic.LoadInt64(&apl.migrationContext.ChunkSize) + if sourceDB == nil { + return chunkSize, rowsAffected, duration, errors.New("source DB is required for move-tables copy") + } // First, select data from the source database: rows, err := func() ([]*sql.ColumnValues, error) { var qb *sql.MoveTableCopySelectQueryBuilder + apl.migrationContext.Log.Debugf("Building SELECT query for move-tables; first: %v; rest: %v", + apl.moveTablesCopySelectFirstQueryBuilder, + apl.moveTablesCopySelectNextQueryBuilder) + if apl.migrationContext.GetIteration() == 0 { qb = apl.moveTablesCopySelectFirstQueryBuilder } else { @@ -1318,7 +1421,7 @@ func (apl *Applier) ApplyIterationMoveTableCopyQueries() (chunkSize int64, rowsA if err != nil { return nil, err } - sqlRows, err := apl.db.Query(query, explodedArgs...) + sqlRows, err := sourceDB.Query(query, explodedArgs...) if err != nil { return nil, err } @@ -1425,11 +1528,11 @@ func (apl *Applier) ApplyIterationMoveTableCopyQueries() (chunkSize int64, rowsA func (apl *Applier) LockOriginalTable() error { query := fmt.Sprintf(`lock /* gh-ost */ tables %s.%s write`, sql.EscapeName(apl.migrationContext.DatabaseName), - sql.EscapeName(apl.migrationContext.OriginalTableName), + sql.EscapeName(apl.originalTableName()), ) apl.migrationContext.Log.Infof("Locking %s.%s", sql.EscapeName(apl.migrationContext.DatabaseName), - sql.EscapeName(apl.migrationContext.OriginalTableName), + sql.EscapeName(apl.originalTableName()), ) apl.migrationContext.LockTablesStartTime = time.Now() if _, err := sqlutils.ExecNoPrepare(apl.singletonDB, query); err != nil { @@ -1457,7 +1560,7 @@ func (apl *Applier) UnlockTables() error { func (apl *Applier) SwapTablesQuickAndBumpy() error { query := fmt.Sprintf(`alter /* gh-ost */ table %s.%s rename %s`, sql.EscapeName(apl.migrationContext.DatabaseName), - sql.EscapeName(apl.migrationContext.OriginalTableName), + sql.EscapeName(apl.originalTableName()), sql.EscapeName(apl.migrationContext.GetOldTableName()), ) apl.migrationContext.Log.Infof("Renaming original table") @@ -1468,9 +1571,9 @@ func (apl *Applier) SwapTablesQuickAndBumpy() error { query = fmt.Sprintf(`alter /* gh-ost */ table %s.%s rename %s`, sql.EscapeName(apl.migrationContext.DatabaseName), sql.EscapeName(apl.migrationContext.GetGhostTableName()), - sql.EscapeName(apl.migrationContext.OriginalTableName), + sql.EscapeName(apl.originalTableName()), ) - apl.migrationContext.Log.Infof("Renaming ghost table") + apl.migrationContext.Log.Infof("Renaming target table") if _, err := sqlutils.ExecNoPrepare(apl.db, query); err != nil { return err } @@ -1487,13 +1590,13 @@ func (apl *Applier) RenameTablesRollback() (renameError error) { // We prefer the single, atomic operation: query := fmt.Sprintf(`rename /* gh-ost */ table %s.%s to %s.%s, %s.%s to %s.%s`, sql.EscapeName(apl.migrationContext.DatabaseName), - sql.EscapeName(apl.migrationContext.OriginalTableName), + sql.EscapeName(apl.originalTableName()), sql.EscapeName(apl.migrationContext.DatabaseName), sql.EscapeName(apl.migrationContext.GetGhostTableName()), sql.EscapeName(apl.migrationContext.DatabaseName), sql.EscapeName(apl.migrationContext.GetOldTableName()), sql.EscapeName(apl.migrationContext.DatabaseName), - sql.EscapeName(apl.migrationContext.OriginalTableName), + sql.EscapeName(apl.originalTableName()), ) apl.migrationContext.Log.Infof("Renaming back both tables") if _, err := sqlutils.ExecNoPrepare(apl.db, query); err == nil { @@ -1502,7 +1605,7 @@ func (apl *Applier) RenameTablesRollback() (renameError error) { // But, if for some reason the above was impossible to do, we rename one by one. query = fmt.Sprintf(`rename /* gh-ost */ table %s.%s to %s.%s`, sql.EscapeName(apl.migrationContext.DatabaseName), - sql.EscapeName(apl.migrationContext.OriginalTableName), + sql.EscapeName(apl.originalTableName()), sql.EscapeName(apl.migrationContext.DatabaseName), sql.EscapeName(apl.migrationContext.GetGhostTableName()), ) @@ -1514,7 +1617,7 @@ func (apl *Applier) RenameTablesRollback() (renameError error) { sql.EscapeName(apl.migrationContext.DatabaseName), sql.EscapeName(apl.migrationContext.GetOldTableName()), sql.EscapeName(apl.migrationContext.DatabaseName), - sql.EscapeName(apl.migrationContext.OriginalTableName), + sql.EscapeName(apl.originalTableName()), ) apl.migrationContext.Log.Infof("Renaming back to original table") if _, err := sqlutils.ExecNoPrepare(apl.db, query); err != nil { @@ -1763,13 +1866,13 @@ func (apl *Applier) AtomicCutOverMagicLock(sessionIdChan chan int64, tableLocked query = fmt.Sprintf(`lock /* gh-ost */ tables %s.%s write, %s.%s write`, sql.EscapeName(apl.migrationContext.DatabaseName), - sql.EscapeName(apl.migrationContext.OriginalTableName), + sql.EscapeName(apl.originalTableName()), sql.EscapeName(apl.migrationContext.DatabaseName), sql.EscapeName(apl.migrationContext.GetOldTableName()), ) apl.migrationContext.Log.Infof("Locking %s.%s, %s.%s", sql.EscapeName(apl.migrationContext.DatabaseName), - sql.EscapeName(apl.migrationContext.OriginalTableName), + sql.EscapeName(apl.originalTableName()), sql.EscapeName(apl.migrationContext.DatabaseName), sql.EscapeName(apl.migrationContext.GetOldTableName()), ) @@ -1819,7 +1922,7 @@ func (apl *Applier) AtomicCutOverMagicLock(sessionIdChan chan int64, tableLocked // Tables still locked apl.migrationContext.Log.Infof("Releasing lock from %s.%s, %s.%s", sql.EscapeName(apl.migrationContext.DatabaseName), - sql.EscapeName(apl.migrationContext.OriginalTableName), + sql.EscapeName(apl.originalTableName()), sql.EscapeName(apl.migrationContext.DatabaseName), sql.EscapeName(apl.migrationContext.GetOldTableName()), ) @@ -1858,13 +1961,13 @@ func (apl *Applier) AtomicCutoverRename(sessionIdChan chan int64, tablesRenamed query = fmt.Sprintf(`rename /* gh-ost */ table %s.%s to %s.%s, %s.%s to %s.%s`, sql.EscapeName(apl.migrationContext.DatabaseName), - sql.EscapeName(apl.migrationContext.OriginalTableName), + sql.EscapeName(apl.originalTableName()), sql.EscapeName(apl.migrationContext.DatabaseName), sql.EscapeName(apl.migrationContext.GetOldTableName()), sql.EscapeName(apl.migrationContext.DatabaseName), sql.EscapeName(apl.migrationContext.GetGhostTableName()), sql.EscapeName(apl.migrationContext.DatabaseName), - sql.EscapeName(apl.migrationContext.OriginalTableName), + sql.EscapeName(apl.originalTableName()), ) apl.migrationContext.Log.Infof("Issuing and expecting this to block: %s", query) if _, err := tx.Exec(query); err != nil { @@ -2172,12 +2275,12 @@ func (apl *Applier) ExpectMetadataLock(sessionId int64) error { err := sqlutils.QueryRowsMap(apl.db, query, func(m sqlutils.RowMap) error { found = true return nil - }, apl.migrationContext.DatabaseName, apl.migrationContext.OriginalTableName, sessionId) + }, apl.migrationContext.DatabaseName, apl.originalTableName(), sessionId) if err != nil { return err } if !found { - err = fmt.Errorf("cannot find PENDING metadata lock on original table: `%s`.`%s`", apl.migrationContext.DatabaseName, apl.migrationContext.OriginalTableName) + err = fmt.Errorf("cannot find PENDING metadata lock on original table: `%s`.`%s`", apl.migrationContext.DatabaseName, apl.originalTableName()) return apl.migrationContext.Log.Errore(err) } return nil diff --git a/go/logic/applier_test.go b/go/logic/applier_test.go index 0c779b781..f3d620ca3 100644 --- a/go/logic/applier_test.go +++ b/go/logic/applier_test.go @@ -402,6 +402,8 @@ func (suite *ApplierTestSuite) TearDownTest() { suite.Require().NoError(err) _, err = suite.otherDB.ExecContext(ctx, "DROP TABLE IF EXISTS "+getTestOtherTableName()) suite.Require().NoError(err) + _, err = suite.db.ExecContext(ctx, fmt.Sprintf("DROP TABLE IF EXISTS `%s`.`_%s_ghc`", testMysqlDatabase, testMysqlTableName)) + suite.Require().NoError(err) } func (suite *ApplierTestSuite) TestInitDBConnections() { @@ -433,6 +435,41 @@ func (suite *ApplierTestSuite) TestInitDBConnections() { suite.Require().Equal(sql.NewColumnList([]string{"id", "item_id"}), migrationContext.OriginalTableColumnsOnApplier) } +func (suite *ApplierTestSuite) TestInitiateApplierMoveTablesMode_NoGhostOrChangelogTable() { + ctx := context.Background() + + var err error + _, err = suite.db.ExecContext(ctx, fmt.Sprintf("CREATE TABLE %s (id INT, item_id INT);", getTestTableName())) + suite.Require().NoError(err) + + connectionConfig, err := getTestConnectionConfig(ctx, suite.mysqlContainer) + suite.Require().NoError(err) + + migrationContext := newTestMigrationContext() + migrationContext.MoveTables.TableNames = []string{testMysqlTableName} + migrationContext.MoveTables.TargetDatabase = testMysqlDatabaseOther + migrationContext.ApplierConnectionConfig = connectionConfig + migrationContext.MoveTables.ConnectionConfig = connectionConfig + migrationContext.SetConnectionConfig("innodb") + + applier := NewApplier(migrationContext) + defer applier.Teardown() + + migrationContext.OriginalTableColumns = sql.NewColumnList([]string{"id", "item_id"}) + + err = applier.InitDBConnections() + suite.Require().NoError(err) + + // #8206 [Task] [1.2] Skip ghost/changelog tables, heartbeat in gh-ost move-tables mode + // In move-tables mode, no ghost or changelog table should exist. + // InitDBConnections() should succeed without them. + suite.Require().False(applier.tableExists("_testing_gho"), "ghost table should not exist in move-tables mode") + suite.Require().False(applier.tableExists("_testing_ghc"), "changelog table should not exist in move-tables mode") + + // In move-tables mode, OriginalTableColumnsOnApplier is unused and intentionally never populated + suite.Require().Nil(migrationContext.OriginalTableColumnsOnApplier) +} + func (suite *ApplierTestSuite) TestApplyDMLEventQueries() { ctx := context.Background() @@ -498,6 +535,53 @@ func (suite *ApplierTestSuite) TestApplyDMLEventQueries() { suite.Require().Equal(int64(0), migrationContext.RowsDeltaEstimate) } +// finalCleanup() requires a fully wired migrator to call directly. +// This test verifies the IsMoveTablesMode() predicate that gates the early return. +// Full behavioral coverage relies on the suite: no ghost/changelog tables are +// created (Test #1), and WriteChangelog is a no-op (Test #2). +func (suite *ApplierTestSuite) TestFinalCleanupMoveTablesMode_SkipsDrops() { + migrationContext := newTestMigrationContext() + migrationContext.MoveTables.TableNames = []string{testMysqlTableName} + migrationContext.MoveTables.TargetDatabase = testMysqlDatabaseOther + + suite.Require().True(migrationContext.IsMoveTablesMode()) +} + +// initiateStreaming() requires a binlog-capable MySQL connection to call directly. +// This test verifies IsMoveTablesMode() and that GetChangelogTableName() returns +// a derivable name. A new streamer always starts with zero listeners; the real +// proof that no changelog listener is registered comes from the full run not +// failing on a nonexistent _ghc table. +func (suite *ApplierTestSuite) TestInitiateStreamingMoveTablesMode_NoChangelogListener() { + migrationContext := newTestMigrationContext() + migrationContext.MoveTables.TableNames = []string{testMysqlTableName} + migrationContext.MoveTables.TargetDatabase = testMysqlDatabaseOther + + suite.Require().True(migrationContext.IsMoveTablesMode()) + + changelogTableName := migrationContext.GetChangelogTableName() + suite.Require().NotEmpty(changelogTableName, "changelog table name should be derivable") + + streamer := NewEventsStreamer(migrationContext) + suite.Require().Empty(streamer.listeners, "new streamer should have no listeners") +} + +// initiateApplier() requires a full migrator to call directly. +// This test verifies the IsMoveTablesMode() predicate that gates InitiateHeartbeat(). +// Even if heartbeat ran, TestWriteChangelogNoOpInMoveTablesMode proves WriteChangelog +// is a no-op, so no SQL would execute against a nonexistent changelog table. +// +// A stronger test would instrument InitiateHeartbeat() (e.g., via a callback or +// channel) to assert the goroutine is never started. That requires test-infrastructure +// changes to the Applier and is beyond #8206's scope. +func (suite *ApplierTestSuite) TestNoHeartbeatInMoveTablesMode() { + migrationContext := newTestMigrationContext() + migrationContext.MoveTables.TableNames = []string{testMysqlTableName} + migrationContext.MoveTables.TargetDatabase = testMysqlDatabaseOther + + suite.Require().True(migrationContext.IsMoveTablesMode()) +} + func (suite *ApplierTestSuite) TestValidateOrDropExistingTables() { ctx := context.Background() @@ -596,6 +680,43 @@ func (suite *ApplierTestSuite) TestValidateOrDropExistingTablesWithGhostTableExi suite.Require().Equal(gosql.ErrNoRows, err) } +func (suite *ApplierTestSuite) TestWriteChangelogNoOpInMoveTablesMode() { + ctx := context.Background() + + var err error + _, err = suite.db.ExecContext(ctx, fmt.Sprintf("CREATE TABLE %s (id INT, item_id INT);", getTestTableName())) + suite.Require().NoError(err) + + connectionConfig, err := getTestConnectionConfig(ctx, suite.mysqlContainer) + suite.Require().NoError(err) + + migrationContext := newTestMigrationContext() + migrationContext.MoveTables.TableNames = []string{testMysqlTableName} + migrationContext.MoveTables.TargetDatabase = testMysqlDatabaseOther + migrationContext.ApplierConnectionConfig = connectionConfig + migrationContext.MoveTables.ConnectionConfig = connectionConfig + migrationContext.SetConnectionConfig("innodb") + migrationContext.OriginalTableColumns = sql.NewColumnList([]string{"id", "item_id"}) + + applier := NewApplier(migrationContext) + defer applier.Teardown() + + err = applier.InitDBConnections() + suite.Require().NoError(err) + + // #8206 [Task] [1.2] Skip ghost/changelog tables, heartbeat in gh-ost move-tables mode + // WriteChangelog should be a no-op in move-tables mode. + // No changelog table exists, so if it tried to execute, it would fail. + hint, err := applier.WriteChangelog("heartbeat", "2026-06-05T00:00:00Z") + suite.Require().NoError(err) + suite.Require().Empty(hint) + + // Also verify state writes are no-ops + hint, err = applier.WriteChangelogState("Migrated") + suite.Require().NoError(err) + suite.Require().Equal("", hint) +} + func (suite *ApplierTestSuite) TestAcquireMigrationLockSucceedsWhenFree() { ctx := context.Background() @@ -810,11 +931,11 @@ func (suite *ApplierTestSuite) TestPanicOnWarningsInApplyIterationInsertQuerySuc err = applier.CreateChangelogTable() suite.Require().NoError(err) - err = applier.ReadMigrationRangeValues() + err = applier.ReadMigrationRangeValues(nil) suite.Require().NoError(err) migrationContext.SetNextIterationRangeMinValues() - hasFurtherRange, err := applier.CalculateNextIterationRangeEndValues() + hasFurtherRange, err := applier.CalculateNextIterationRangeEndValues(nil) suite.Require().NoError(err) suite.Require().True(hasFurtherRange) @@ -887,14 +1008,14 @@ func (suite *ApplierTestSuite) TestPanicOnWarningsInApplyIterationInsertQueryFai err = applier.CreateChangelogTable() suite.Require().NoError(err) - err = applier.ReadMigrationRangeValues() + err = applier.ReadMigrationRangeValues(nil) suite.Require().NoError(err) err = applier.AlterGhost() suite.Require().NoError(err) migrationContext.SetNextIterationRangeMinValues() - hasFurtherRange, err := applier.CalculateNextIterationRangeEndValues() + hasFurtherRange, err := applier.CalculateNextIterationRangeEndValues(nil) suite.Require().NoError(err) suite.Require().True(hasFurtherRange) @@ -960,7 +1081,7 @@ func (suite *ApplierTestSuite) TestWriteCheckpoint() { err = applier.prepareQueries() suite.Require().NoError(err) - err = applier.ReadMigrationRangeValues() + err = applier.ReadMigrationRangeValues(nil) suite.Require().NoError(err) // checkpoint table is empty @@ -1854,15 +1975,15 @@ func (suite *ApplierTestSuite) TestApplyIterationMoveTableCopyQueries() { err = applier.CreateChangelogTable() suite.Require().NoError(err) - err = applier.ReadMigrationRangeValues() + err = applier.ReadMigrationRangeValues(nil) suite.Require().NoError(err) migrationContext.SetNextIterationRangeMinValues() - hasFurtherRange, err := applier.CalculateNextIterationRangeEndValues() + hasFurtherRange, err := applier.CalculateNextIterationRangeEndValues(nil) suite.Require().NoError(err) suite.Require().True(hasFurtherRange) - chunkSize, rowsAffected, duration, err := applier.ApplyIterationMoveTableCopyQueries() + chunkSize, rowsAffected, duration, err := applier.ApplyIterationMoveTableCopyQueries(applier.db) suite.Require().NoError(err) suite.Require().Equal(int64(3), rowsAffected) suite.Require().Equal(int64(1000), chunkSize) @@ -1939,7 +2060,7 @@ func (suite *ApplierTestSuite) TestApplyIterationMoveTableCopyQueriesNoRows() { migrationContext.MigrationIterationRangeMinValues = sql.ToColumnValues([]interface{}{100}) migrationContext.MigrationIterationRangeMaxValues = sql.ToColumnValues([]interface{}{200}) - chunkSize, rowsAffected, duration, err := applier.ApplyIterationMoveTableCopyQueries() + chunkSize, rowsAffected, duration, err := applier.ApplyIterationMoveTableCopyQueries(applier.db) suite.Require().NoError(err) suite.Require().Equal(int64(0), rowsAffected) suite.Require().Equal(int64(1000), chunkSize) diff --git a/go/logic/inspect.go b/go/logic/inspect.go index 05d6b67b5..d4654e831 100644 --- a/go/logic/inspect.go +++ b/go/logic/inspect.go @@ -91,13 +91,13 @@ func (isp *Inspector) ValidateOriginalTable() (err error) { return err } if err := isp.validateTableForeignKeys(isp.migrationContext.DiscardForeignKeys); err != nil { - return err + return fmt.Errorf("failed to validate table foreign keys: %w", err) } if err := isp.validateTableTriggers(); err != nil { - return err + return fmt.Errorf("failed to validate table triggers: %w", err) } if err := isp.estimateTableRowsViaExplain(); err != nil { - return err + return fmt.Errorf("failed to estimate table rows: %w", err) } return nil } @@ -119,17 +119,24 @@ func (isp *Inspector) InspectTableColumnsAndUniqueKeys(tableName string) (column } func (isp *Inspector) InspectOriginalTable() (err error) { - isp.migrationContext.OriginalTableColumns, isp.migrationContext.OriginalTableVirtualColumns, isp.migrationContext.OriginalTableUniqueKeys, err = isp.InspectTableColumnsAndUniqueKeys(isp.migrationContext.OriginalTableName) + isp.migrationContext.OriginalTableColumns, isp.migrationContext.OriginalTableVirtualColumns, isp.migrationContext.OriginalTableUniqueKeys, err = isp.InspectTableColumnsAndUniqueKeys(isp.originalTableName()) if err != nil { return err } - isp.migrationContext.OriginalTableAutoIncrement, err = isp.getAutoIncrementValue(isp.migrationContext.OriginalTableName) + isp.migrationContext.OriginalTableAutoIncrement, err = isp.getAutoIncrementValue(isp.originalTableName()) if err != nil { return err } return nil } +func (isp *Inspector) originalTableName() string { + if isp.migrationContext.IsMoveTablesMode() { + return isp.migrationContext.MoveTables.TableNames[0] + } + return isp.migrationContext.OriginalTableName +} + // inspectOriginalAndGhostTables compares original and ghost tables to see whether the migration // makes sense and is valid. It extracts the list of shared columns and the chosen migration unique key func (isp *Inspector) inspectOriginalAndGhostTables() (err error) { @@ -144,30 +151,7 @@ func (isp *Inspector) inspectOriginalAndGhostTables() (err error) { return err } sharedUniqueKeys := isp.getSharedUniqueKeys(isp.migrationContext.OriginalTableUniqueKeys, isp.migrationContext.GhostTableUniqueKeys) - for i, sharedUniqueKey := range sharedUniqueKeys { - isp.applyColumnTypes(isp.migrationContext.DatabaseName, isp.migrationContext.OriginalTableName, &sharedUniqueKey.Columns) - uniqueKeyIsValid := true - for _, column := range sharedUniqueKey.Columns.Columns() { - switch column.Type { - case sql.FloatColumnType: - { - isp.migrationContext.Log.Warningf("Will not use %+v as shared key due to FLOAT data type", sharedUniqueKey.Name) - uniqueKeyIsValid = false - } - case sql.JSONColumnType: - { - // Noteworthy that at this time MySQL does not allow JSON indexing anyhow, but this code - // will remain in place to potentially handle the future case where JSON is supported in indexes. - isp.migrationContext.Log.Warningf("Will not use %+v as shared key due to JSON data type", sharedUniqueKey.Name) - uniqueKeyIsValid = false - } - } - } - if uniqueKeyIsValid { - isp.migrationContext.UniqueKey = sharedUniqueKeys[i] - break - } - } + isp.migrationContext.UniqueKey = isp.selectUniqueKey(sharedUniqueKeys) if isp.migrationContext.UniqueKey == nil { return fmt.Errorf("no shared unique key can be found after ALTER! Bailing out") } @@ -187,7 +171,7 @@ func (isp *Inspector) inspectOriginalAndGhostTables() (err error) { // This additional step looks at which columns are unsigned. We could have merged this within // the `getTableColumns()` function, but it's a later patch and introduces some complexity; I feel // comfortable in doing this as a separate step. - isp.applyColumnTypes(isp.migrationContext.DatabaseName, isp.migrationContext.OriginalTableName, isp.migrationContext.OriginalTableColumns, isp.migrationContext.SharedColumns, &isp.migrationContext.UniqueKey.Columns) + isp.applyColumnTypes(isp.migrationContext.DatabaseName, isp.originalTableName(), isp.migrationContext.OriginalTableColumns, isp.migrationContext.SharedColumns, &isp.migrationContext.UniqueKey.Columns) isp.applyColumnTypes(isp.migrationContext.DatabaseName, isp.migrationContext.GetGhostTableName(), isp.migrationContext.GhostTableColumns, isp.migrationContext.MappedSharedColumns) for i := range isp.migrationContext.SharedColumns.Columns() { @@ -218,6 +202,33 @@ func (isp *Inspector) inspectOriginalAndGhostTables() (err error) { return nil } +func (isp *Inspector) selectUniqueKey(candidateKeys []*sql.UniqueKey) *sql.UniqueKey { + for i, candidateKey := range candidateKeys { + isp.applyColumnTypes(isp.migrationContext.DatabaseName, isp.originalTableName(), &candidateKey.Columns) + uniqueKeyIsValid := true + for _, column := range candidateKey.Columns.Columns() { + switch column.Type { + case sql.FloatColumnType: + { + isp.migrationContext.Log.Warningf("Will not use %+v as unique key due to FLOAT data type", candidateKey.Name) + uniqueKeyIsValid = false + } + case sql.JSONColumnType: + { + // Noteworthy that at this time MySQL does not allow JSON indexing anyhow, but this code + // will remain in place to potentially handle the future case where JSON is supported in indexes. + isp.migrationContext.Log.Warningf("Will not use %+v as unique key due to JSON data type", candidateKey.Name) + uniqueKeyIsValid = false + } + } + } + if uniqueKeyIsValid { + return candidateKeys[i] + } + } + return nil +} + // validateConnection issues a simple can-connect to MySQL func (isp *Inspector) validateConnection() error { version, err := base.ValidateConnection(isp.db, isp.connectionConfig, isp.migrationContext, isp.name) @@ -480,7 +491,7 @@ func (isp *Inspector) validateLogSlaveUpdates() error { // validateTable makes sure the table we need to operate on actually exists func (isp *Inspector) validateTable() error { - query := fmt.Sprintf(`show /* gh-ost */ table status from %s like '%s'`, sql.EscapeName(isp.migrationContext.DatabaseName), isp.migrationContext.OriginalTableName) + query := fmt.Sprintf(`show /* gh-ost */ table status from %s like '%s'`, sql.EscapeName(isp.migrationContext.DatabaseName), isp.originalTableName()) tableFound := false err := sqlutils.QueryRowsMap(isp.db, query, func(rowMap sqlutils.RowMap) error { @@ -488,7 +499,7 @@ func (isp *Inspector) validateTable() error { isp.migrationContext.RowsEstimate = rowMap.GetInt64("Rows") isp.migrationContext.UsedRowsEstimateMethod = base.TableStatusRowsEstimate if rowMap.GetString("Comment") == "VIEW" { - return fmt.Errorf("%s.%s is a VIEW, not a real table. Bailing out", sql.EscapeName(isp.migrationContext.DatabaseName), sql.EscapeName(isp.migrationContext.OriginalTableName)) + return fmt.Errorf("%s.%s is a VIEW, not a real table. Bailing out", sql.EscapeName(isp.migrationContext.DatabaseName), sql.EscapeName(isp.originalTableName())) } tableFound = true @@ -498,7 +509,7 @@ func (isp *Inspector) validateTable() error { return err } if !tableFound { - return isp.migrationContext.Log.Errorf("cannot find table %s.%s!", sql.EscapeName(isp.migrationContext.DatabaseName), sql.EscapeName(isp.migrationContext.OriginalTableName)) + return isp.migrationContext.Log.Errorf("cannot find table %s.%s!", sql.EscapeName(isp.migrationContext.DatabaseName), sql.EscapeName(isp.originalTableName())) } isp.migrationContext.Log.Infof("Table found. Engine=%s", isp.migrationContext.TableEngine) isp.migrationContext.Log.Debugf("Estimated number of rows via STATUS: %d", isp.migrationContext.RowsEstimate) @@ -532,26 +543,26 @@ func (isp *Inspector) validateTableForeignKeys(allowChildForeignKeys bool) error return nil }, isp.migrationContext.DatabaseName, - isp.migrationContext.OriginalTableName, + isp.originalTableName(), isp.migrationContext.DatabaseName, - isp.migrationContext.OriginalTableName, + isp.originalTableName(), isp.migrationContext.DatabaseName, - isp.migrationContext.OriginalTableName, + isp.originalTableName(), isp.migrationContext.DatabaseName, - isp.migrationContext.OriginalTableName, + isp.originalTableName(), ) if err != nil { return err } if numParentForeignKeys > 0 { - return isp.migrationContext.Log.Errorf("found %d parent-side foreign keys on %s.%s. Parent-side foreign keys are not supported. Bailing out", numParentForeignKeys, sql.EscapeName(isp.migrationContext.DatabaseName), sql.EscapeName(isp.migrationContext.OriginalTableName)) + return isp.migrationContext.Log.Errorf("found %d parent-side foreign keys on %s.%s. Parent-side foreign keys are not supported. Bailing out", numParentForeignKeys, sql.EscapeName(isp.migrationContext.DatabaseName), sql.EscapeName(isp.originalTableName())) } if numChildForeignKeys > 0 { if allowChildForeignKeys { isp.migrationContext.Log.Debugf("Foreign keys found and will be dropped, as per given --discard-foreign-keys flag") return nil } - return isp.migrationContext.Log.Errorf("found %d child-side foreign keys on %s.%s. Child-side foreign keys are not supported. Bailing out", numChildForeignKeys, sql.EscapeName(isp.migrationContext.DatabaseName), sql.EscapeName(isp.migrationContext.OriginalTableName)) + return isp.migrationContext.Log.Errorf("found %d child-side foreign keys on %s.%s. Child-side foreign keys are not supported. Bailing out", numChildForeignKeys, sql.EscapeName(isp.migrationContext.DatabaseName), sql.EscapeName(isp.originalTableName())) } isp.migrationContext.Log.Debugf("Validated no foreign keys exist on table") return nil @@ -573,15 +584,15 @@ func (isp *Inspector) validateTableTriggers() error { return nil }, isp.migrationContext.DatabaseName, - isp.migrationContext.OriginalTableName, + isp.originalTableName(), ) if err != nil { return err } if numTriggers > 0 { if isp.migrationContext.IncludeTriggers { - isp.migrationContext.Log.Infof("Found %d triggers on %s.%s.", numTriggers, sql.EscapeName(isp.migrationContext.DatabaseName), sql.EscapeName(isp.migrationContext.OriginalTableName)) - isp.migrationContext.Triggers, err = mysql.GetTriggers(isp.db, isp.migrationContext.DatabaseName, isp.migrationContext.OriginalTableName) + isp.migrationContext.Log.Infof("Found %d triggers on %s.%s.", numTriggers, sql.EscapeName(isp.migrationContext.DatabaseName), sql.EscapeName(isp.originalTableName())) + isp.migrationContext.Triggers, err = mysql.GetTriggers(isp.db, isp.migrationContext.DatabaseName, isp.originalTableName()) if err != nil { return err } @@ -593,7 +604,7 @@ func (isp *Inspector) validateTableTriggers() error { } return nil } - return isp.migrationContext.Log.Errorf("found triggers on %s.%s. Tables with triggers are supported only when using \"include-triggers\" flag. Bailing out", sql.EscapeName(isp.migrationContext.DatabaseName), sql.EscapeName(isp.migrationContext.OriginalTableName)) + return isp.migrationContext.Log.Errorf("found triggers on %s.%s. Tables with triggers are supported only when using \"include-triggers\" flag. Bailing out", sql.EscapeName(isp.migrationContext.DatabaseName), sql.EscapeName(isp.originalTableName())) } isp.migrationContext.Log.Debugf("Validated no triggers exist on table") return nil @@ -646,7 +657,7 @@ func (isp *Inspector) validateGhostTriggersLength() error { // estimateTableRowsViaExplain estimates number of rows on original table func (isp *Inspector) estimateTableRowsViaExplain() error { - query := fmt.Sprintf(`explain select /* gh-ost */ * from %s.%s where 1=1`, sql.EscapeName(isp.migrationContext.DatabaseName), sql.EscapeName(isp.migrationContext.OriginalTableName)) + query := fmt.Sprintf(`explain select /* gh-ost */ * from %s.%s where 1=1`, sql.EscapeName(isp.migrationContext.DatabaseName), sql.EscapeName(isp.originalTableName())) outputFound := false err := sqlutils.QueryRowsMap(isp.db, query, func(rowMap sqlutils.RowMap) error { @@ -660,7 +671,7 @@ func (isp *Inspector) estimateTableRowsViaExplain() error { return err } if !outputFound { - return isp.migrationContext.Log.Errorf("cannot run EXPLAIN on %s.%s!", sql.EscapeName(isp.migrationContext.DatabaseName), sql.EscapeName(isp.migrationContext.OriginalTableName)) + return isp.migrationContext.Log.Errorf("cannot run EXPLAIN on %s.%s!", sql.EscapeName(isp.migrationContext.DatabaseName), sql.EscapeName(isp.originalTableName())) } isp.migrationContext.Log.Infof("Estimated number of rows via EXPLAIN: %d", isp.migrationContext.RowsEstimate) return nil @@ -684,7 +695,7 @@ func (isp *Inspector) CountTableRows(ctx context.Context) error { return err } - query := fmt.Sprintf(`select /* gh-ost */ count(*) as count_rows from %s.%s`, sql.EscapeName(isp.migrationContext.DatabaseName), sql.EscapeName(isp.migrationContext.OriginalTableName)) + query := fmt.Sprintf(`select /* gh-ost */ count(*) as count_rows from %s.%s`, sql.EscapeName(isp.migrationContext.DatabaseName), sql.EscapeName(isp.originalTableName())) var rowsEstimate int64 queryStartTime := time.Now() if err := conn.QueryRowContext(ctx, query).Scan(&rowsEstimate); err != nil { diff --git a/go/logic/migrator.go b/go/logic/migrator.go index ae0a1fbb8..422ab1afa 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -503,7 +503,7 @@ func (mgtr *Migrator) Migrate() (err error) { defer mgtr.teardown() if err := mgtr.initiateInspector(); err != nil { - return err + return fmt.Errorf("failed to initiate inspector: %w", err) } if err := mgtr.checkAbort(); err != nil { return err @@ -612,7 +612,7 @@ func (mgtr *Migrator) Migrate() (err error) { if err := mgtr.addDMLEventsListener(); err != nil { return err } - if err := mgtr.applier.ReadMigrationRangeValues(); err != nil { + if err := mgtr.applier.ReadMigrationRangeValues(nil); err != nil { return err } @@ -798,13 +798,27 @@ func (mgtr *Migrator) Revert() error { return nil } +// prepareMoveTablesCopyState initializes state for row copy in move-tables mode. +// for move-tables functionality, the source and target tables are identical so we just need to grab any valid UNIQUE key constraint. +func (mgtr *Migrator) prepareMoveTablesCopyState() { + mgtr.migrationContext.UniqueKey = mgtr.inspector.selectUniqueKey(mgtr.migrationContext.OriginalTableUniqueKeys) + + // In move-tables mode source and target schemas match, so shared columns are identical. + mgtr.migrationContext.SharedColumns = mgtr.migrationContext.OriginalTableColumns + mgtr.migrationContext.MappedSharedColumns = mgtr.migrationContext.OriginalTableColumns +} + func (mgtr *Migrator) MoveTables() (err error) { mgtr.migrationContext.Log.Infof("Moving tables %v from %s to %s (%s)", mgtr.migrationContext.MoveTables.TableNames, sql.EscapeName(mgtr.migrationContext.DatabaseName), - sql.EscapeName(mgtr.migrationContext.MoveTables.TargetDatabase), mgtr.migrationContext.MoveTables.TargetHost) + sql.EscapeName(mgtr.migrationContext.GetTargetDatabaseName()), mgtr.migrationContext.MoveTables.TargetHost) mgtr.migrationContext.StartTime = time.Now() + if mgtr.migrationContext.OriginalTableName == "" { + mgtr.migrationContext.OriginalTableName = mgtr.migrationContext.MoveTables.TableNames[0] + } + // Ensure context is cancelled on exit (cleanup) defer mgtr.migrationContext.CancelContext() @@ -832,10 +846,20 @@ func (mgtr *Migrator) MoveTables() (err error) { if err := mgtr.initiateApplier(); err != nil { return err } + if err := mgtr.initiateStreaming(); err != nil { + return err + } if err := mgtr.checkAbort(); err != nil { return err } + mgtr.prepareMoveTablesCopyState() + + // this function assumes that the unique key constraint has been set. + if err := mgtr.applier.prepareQueries(); err != nil { + return err + } + // Validation complete! Run on-validated hook. if err := mgtr.hooksExecutor.OnValidated(); err != nil { return err @@ -852,7 +876,7 @@ func (mgtr *Migrator) MoveTables() (err error) { if err := mgtr.addDMLEventsListener(); err != nil { return err } - if err := mgtr.applier.ReadMigrationRangeValues(); err != nil { + if err := mgtr.applier.ReadMigrationRangeValues(mgtr.inspector.db); err != nil { return err } @@ -893,7 +917,7 @@ func (mgtr *Migrator) MoveTables() (err error) { } mgtr.migrationContext.Log.Infof("Done moving tables %v from %s to %s (%s)", mgtr.migrationContext.MoveTables.TableNames, sql.EscapeName(mgtr.migrationContext.DatabaseName), - sql.EscapeName(mgtr.migrationContext.MoveTables.TargetDatabase), mgtr.migrationContext.MoveTables.TargetHost) + sql.EscapeName(mgtr.migrationContext.GetTargetDatabaseName()), mgtr.migrationContext.MoveTables.TargetHost) // Final check for abort before declaring success if err := mgtr.checkAbort(); err != nil { return err @@ -1295,14 +1319,16 @@ func (mgtr *Migrator) initiateInspector() (err error) { return err } if err := mgtr.inspector.ValidateOriginalTable(); err != nil { - return err + return fmt.Errorf("failed to validate original table: %w", err) } if err := mgtr.inspector.InspectOriginalTable(); err != nil { - return err + return fmt.Errorf("failed to inspect original table: %w", err) } // So far so good, table is accessible and valid. // Let's get master connection config - if mgtr.migrationContext.AssumeMasterHostname == "" { + if mgtr.migrationContext.IsMoveTablesMode() { + mgtr.migrationContext.ApplierConnectionConfig = mgtr.migrationContext.MoveTables.ConnectionConfig + } else if mgtr.migrationContext.AssumeMasterHostname == "" { // No forced master host; detect master if mgtr.migrationContext.ApplierConnectionConfig, err = mgtr.inspector.getMasterConnectionConfig(); err != nil { return err @@ -1334,7 +1360,9 @@ func (mgtr *Migrator) initiateInspector() (err error) { mgtr.migrationContext.Log.Infof("--test-on-replica or --migrate-on-replica given. Will not execute on master %+v but rather on replica %+v itself", *mgtr.migrationContext.ApplierConnectionConfig.ImpliedKey, *mgtr.migrationContext.InspectorConnectionConfig.ImpliedKey, ) - mgtr.migrationContext.ApplierConnectionConfig = mgtr.migrationContext.InspectorConnectionConfig.Duplicate() + if !mgtr.migrationContext.IsMoveTablesMode() { + mgtr.migrationContext.ApplierConnectionConfig = mgtr.migrationContext.InspectorConnectionConfig.Duplicate() + } if mgtr.migrationContext.GetThrottleControlReplicaKeys().Len() == 0 { mgtr.migrationContext.AddThrottleControlReplicaKey(mgtr.migrationContext.InspectorConnectionConfig.Key) } @@ -1403,11 +1431,11 @@ func (mgtr *Migrator) initiateStatus() { // migration, and as response to the "status" interactive command. func (mgtr *Migrator) printMigrationStatusHint(writers ...io.Writer) { w := io.MultiWriter(writers...) - fmt.Fprintf(w, "# Migrating %s.%s; Ghost table is %s.%s\n", + fmt.Fprintf(w, "# Migrating %s.%s; Target table is %s.%s\n", sql.EscapeName(mgtr.migrationContext.DatabaseName), sql.EscapeName(mgtr.migrationContext.OriginalTableName), - sql.EscapeName(mgtr.migrationContext.DatabaseName), - sql.EscapeName(mgtr.migrationContext.GetGhostTableName()), + sql.EscapeName(mgtr.migrationContext.GetTargetDatabaseName()), + sql.EscapeName(mgtr.migrationContext.GetTargetTableName()), ) fmt.Fprintf(w, "# Migrating %+v; inspecting %+v; executing on %+v\n", *mgtr.applier.connectionConfig.ImpliedKey, @@ -1635,14 +1663,19 @@ func (mgtr *Migrator) initiateStreaming() error { if err := mgtr.eventsStreamer.InitDBConnections(); err != nil { return err } - mgtr.eventsStreamer.AddListener( - false, - mgtr.migrationContext.DatabaseName, - mgtr.migrationContext.GetChangelogTableName(), - func(dmlEntry *binlog.BinlogEntry) error { - return mgtr.onChangelogEvent(dmlEntry) - }, - ) + + if mgtr.migrationContext.IsMoveTablesMode() { + mgtr.migrationContext.Log.Info("Skipping stream of the changelog table") + } else { + mgtr.eventsStreamer.AddListener( + false, + mgtr.migrationContext.DatabaseName, + mgtr.migrationContext.GetChangelogTableName(), + func(dmlEntry *binlog.BinlogEntry) error { + return mgtr.onChangelogEvent(dmlEntry) + }, + ) + } go func() { mgtr.migrationContext.Log.Debugf("Beginning streaming") @@ -1670,10 +1703,15 @@ func (mgtr *Migrator) initiateStreaming() error { // addDMLEventsListener begins listening for binlog events on the original table, // and creates & enqueues a write task per such event. func (mgtr *Migrator) addDMLEventsListener() error { + originalTableName := mgtr.migrationContext.OriginalTableName + if mgtr.migrationContext.IsMoveTablesMode() { + originalTableName = mgtr.migrationContext.MoveTables.TableNames[0] + } + err := mgtr.eventsStreamer.AddListener( false, mgtr.migrationContext.DatabaseName, - mgtr.migrationContext.OriginalTableName, + originalTableName, func(dmlEntry *binlog.BinlogEntry) error { // Use helper to prevent deadlock if buffer fills and executeWriteFuncs exits // This is critical because this callback blocks the event streamer @@ -1685,6 +1723,12 @@ func (mgtr *Migrator) addDMLEventsListener() error { // initiateThrottler kicks in the throttling collection and the throttling checks. func (mgtr *Migrator) initiateThrottler() { + if mgtr.migrationContext.IsMoveTablesMode() { + // TODO(chriskirkland): throttle against the target cluster + mgtr.migrationContext.Log.Info("Skipping throttling in move tables mode") + return + } + mgtr.throttler = NewThrottler(mgtr.migrationContext, mgtr.applier, mgtr.inspector, mgtr.appVersion) go mgtr.throttler.initiateThrottlerCollection(mgtr.firstThrottlingCollected) @@ -1704,39 +1748,51 @@ func (mgtr *Migrator) initiateApplier() error { if err := mgtr.applier.AcquireMigrationLock(mgtr.migrationContext.GetContext()); err != nil { return err } - if mgtr.migrationContext.Revert { - if err := mgtr.applier.CreateChangelogTable(); err != nil { - mgtr.migrationContext.Log.Errorf("unable to create changelog table, see further error details. Perhaps a previous migration failed without dropping the table? OR is there a running migration? Bailing out") - return err - } - } else if !mgtr.migrationContext.Resume { - if err := mgtr.applier.ValidateOrDropExistingTables(); err != nil { - return err - } - if err := mgtr.applier.CreateChangelogTable(); err != nil { - mgtr.migrationContext.Log.Errorf("unable to create changelog table, see further error details. Perhaps a previous migration failed without dropping the table? OR is there a running migration? Bailing out") - return err - } - if err := mgtr.applier.CreateGhostTable(); err != nil { - mgtr.migrationContext.Log.Errorf("unable to create ghost table, see further error details. Perhaps a previous migration failed without dropping the table? Bailing out") - return err + + if mgtr.migrationContext.IsMoveTablesMode() { + createTableStatement, err := mgtr.inspector.showCreateTable(mgtr.migrationContext.MoveTables.TableNames[0]) + if err != nil { + return fmt.Errorf("failed to fetch create table statement: %w", err) } - if err := mgtr.applier.AlterGhost(); err != nil { - mgtr.migrationContext.Log.Errorf("unable to ALTER ghost table, see further error details. Bailing out") + if err := mgtr.applier.CreateTargetTable(createTableStatement); err != nil { + mgtr.migrationContext.Log.Errorf("unable to create target table, see further error details. Perhaps a previous migration failed without dropping the table? Bailing out") return err } + } else { + if mgtr.migrationContext.Revert { + if err := mgtr.applier.CreateChangelogTable(); err != nil { + mgtr.migrationContext.Log.Errorf("unable to create changelog table, see further error details. Perhaps a previous migration failed without dropping the table? OR is there a running migration? Bailing out") + return err + } + } else if !mgtr.migrationContext.Resume { + if err := mgtr.applier.ValidateOrDropExistingTables(); err != nil { + return err + } + if err := mgtr.applier.CreateChangelogTable(); err != nil { + mgtr.migrationContext.Log.Errorf("unable to create changelog table, see further error details. Perhaps a previous migration failed without dropping the table? OR is there a running migration? Bailing out") + return err + } + if err := mgtr.applier.CreateGhostTable(); err != nil { + mgtr.migrationContext.Log.Errorf("unable to create ghost table, see further error details. Perhaps a previous migration failed without dropping the table? Bailing out") + return err + } + if err := mgtr.applier.AlterGhost(); err != nil { + mgtr.migrationContext.Log.Errorf("unable to ALTER ghost table, see further error details. Bailing out") + return err + } - if mgtr.migrationContext.OriginalTableAutoIncrement > 0 && !mgtr.parser.IsAutoIncrementDefined() { - // Original table has AUTO_INCREMENT value and the -alter statement does not indicate any override, - // so we should copy AUTO_INCREMENT value onto our ghost table. - if err := mgtr.applier.AlterGhostAutoIncrement(); err != nil { - mgtr.migrationContext.Log.Errorf("unable to ALTER ghost table AUTO_INCREMENT value, see further error details. Bailing out") + if mgtr.migrationContext.OriginalTableAutoIncrement > 0 && !mgtr.parser.IsAutoIncrementDefined() { + // Original table has AUTO_INCREMENT value and the -alter statement does not indicate any override, + // so we should copy AUTO_INCREMENT value onto our ghost table. + if err := mgtr.applier.AlterGhostAutoIncrement(); err != nil { + mgtr.migrationContext.Log.Errorf("unable to ALTER ghost table AUTO_INCREMENT value, see further error details. Bailing out") + return err + } + } + if _, err := mgtr.applier.WriteChangelogState(string(GhostTableMigrated)); err != nil { return err } } - if _, err := mgtr.applier.WriteChangelogState(string(GhostTableMigrated)); err != nil { - return err - } } // ensure performance_schema.metadata_locks is available. @@ -1750,7 +1806,9 @@ func (mgtr *Migrator) initiateApplier() error { mgtr.migrationContext.Log.Warning("proceeding without metadata lock check. There is a small chance of data loss if another session accesses the ghost table during cut-over. See https://github.com/github/gh-ost/pull/1536 for details") } - go mgtr.applier.InitiateHeartbeat() + if !mgtr.migrationContext.IsMoveTablesMode() { + go mgtr.applier.InitiateHeartbeat() + } return nil } @@ -1792,7 +1850,13 @@ func (mgtr *Migrator) iterateChunks() error { } // When hasFurtherRange is false, original table might be write locked and CalculateNextIterationRangeEndValues would hangs forever - hasFurtherRange, err := mgtr.applier.CalculateNextIterationRangeEndValues() + var hasFurtherRange bool + var err error + if mgtr.migrationContext.IsMoveTablesMode() { + hasFurtherRange, err = mgtr.applier.CalculateNextIterationRangeEndValues(mgtr.inspector.db) + } else { + hasFurtherRange, err = mgtr.applier.CalculateNextIterationRangeEndValues(nil) + } if err != nil { return err // wrapping call will retry } @@ -1813,13 +1877,14 @@ func (mgtr *Migrator) iterateChunks() error { } var rowsAffected int64 if mgtr.migrationContext.IsMoveTablesMode() { - _, rowsAffected, _, err = mgtr.applier.ApplyIterationMoveTableCopyQueries() + _, rowsAffected, _, err = mgtr.applier.ApplyIterationMoveTableCopyQueries(mgtr.inspector.db) } else { _, rowsAffected, _, err = mgtr.applier.ApplyIterationInsertQuery() } if err != nil { return err // wrapping call will retry } + mgtr.migrationContext.Log.Debugf("ApplyIterationInsertQuery affected %d rows", rowsAffected) if mgtr.migrationContext.PanicOnWarnings { if len(mgtr.migrationContext.MigrationLastInsertSQLWarnings) > 0 { @@ -2031,7 +2096,11 @@ func (mgtr *Migrator) executeWriteFuncs() error { return nil } - mgtr.throttler.throttle(nil) + if !mgtr.migrationContext.IsMoveTablesMode() { + // disable throttling in move-tables mode for now + // https://github.com/github/database-infrastructure/issues/8212 + mgtr.throttler.throttle(nil) + } // We give higher priority to event processing, then secondary priority to // rowcopy @@ -2115,14 +2184,21 @@ func (mgtr *Migrator) finalCleanup() error { if createTableStatement, err := mgtr.inspector.showCreateTable(mgtr.migrationContext.GetGhostTableName()); err == nil { mgtr.migrationContext.Log.Infof("New table structure follows") fmt.Println(createTableStatement) - } else { - mgtr.migrationContext.Log.Errore(err) + } else if !mgtr.migrationContext.IsMoveTablesMode() { + mgtr.migrationContext.Log.Errore(fmt.Errorf("error showing create table: %w", err)) } } if err := mgtr.eventsStreamer.Close(); err != nil { mgtr.migrationContext.Log.Errore(err) } + if mgtr.migrationContext.IsMoveTablesMode() { + // for move-tables mode, we're done at this point + // TODO(zacharysierakowski): when we add the checkpoint table in for 1.6, make sure we cleanup + // the checkpoint table here first before returning (looks like that's a few lines below changelog table cleanup) + return nil + } + if err := mgtr.retryOperation(mgtr.applier.DropChangelogTable); err != nil { return err } diff --git a/go/logic/migrator_test.go b/go/logic/migrator_test.go index ad068691c..ef170f1ee 100644 --- a/go/logic/migrator_test.go +++ b/go/logic/migrator_test.go @@ -932,7 +932,7 @@ func (suite *MigratorTestSuite) TestCopierIntPK() { suite.Require().NoError(migrator.initiateApplier()) defer migrator.applier.Teardown() suite.Require().NoError(migrator.applier.prepareQueries()) - suite.Require().NoError(migrator.applier.ReadMigrationRangeValues()) + suite.Require().NoError(migrator.applier.ReadMigrationRangeValues(nil)) go migrator.iterateChunks() go func() { @@ -1004,7 +1004,7 @@ func (suite *MigratorTestSuite) TestCopierCompositePK() { suite.Require().NoError(migrator.initiateApplier()) defer migrator.applier.Teardown() suite.Require().NoError(migrator.applier.prepareQueries()) - suite.Require().NoError(migrator.applier.ReadMigrationRangeValues()) + suite.Require().NoError(migrator.applier.ReadMigrationRangeValues(nil)) go migrator.iterateChunks() go func() { diff --git a/localtests/docker-compose-move-tables.yml b/localtests/docker-compose-move-tables.yml new file mode 100644 index 000000000..8ba8647ea --- /dev/null +++ b/localtests/docker-compose-move-tables.yml @@ -0,0 +1,57 @@ +services: + mysql-source-primary: + image: $TEST_MYSQL_IMAGE + container_name: mysql-source-primary + command: --server-id=1 --log-bin=mysql-bin --binlog-format=row --gtid-mode=ON --enforce-gtid-consistency=ON --character-set-server=utf8mb4 $MYSQL_NATIVE_PASSWORD_FLAG + environment: + MYSQL_ROOT_PASSWORD: opensesame + MYSQL_ROOT_HOST: '%' + MYSQL_DATABASE: test + MYSQL_TCP_PORT: 3307 + INIT_ROCKSDB: 1 # for percona-server + ports: + - '3307:3307' + expose: + - '3307' + mysql-source-replica: + image: $TEST_MYSQL_IMAGE + container_name: mysql-source-replica + command: --server-id=2 --log-bin=mysql-bin --binlog-format=row --gtid-mode=ON --enforce-gtid-consistency=ON --log-slave-updates=ON --character-set-server=utf8mb4 $MYSQL_NATIVE_PASSWORD_FLAG + environment: + MYSQL_ROOT_PASSWORD: opensesame + MYSQL_ROOT_HOST: '%' + MYSQL_DATABASE: test + MYSQL_TCP_PORT: 3308 + INIT_ROCKSDB: 1 # for percona-server + ports: + - '3308:3308' + expose: + - '3308' + mysql-target-primary: + image: $TEST_MYSQL_IMAGE + container_name: mysql-target-primary + command: --server-id=3 --log-bin=mysql-bin --binlog-format=row --gtid-mode=ON --enforce-gtid-consistency=ON --character-set-server=utf8mb4 $MYSQL_NATIVE_PASSWORD_FLAG + environment: + MYSQL_ROOT_PASSWORD: opensesame + MYSQL_ROOT_HOST: '%' + MYSQL_DATABASE: test + MYSQL_TCP_PORT: 3309 + INIT_ROCKSDB: 1 # for percona-server + ports: + - '3309:3309' + expose: + - '3309' + mysql-target-replica: + image: $TEST_MYSQL_IMAGE + container_name: mysql-target-replica + command: --server-id=4 --log-bin=mysql-bin --binlog-format=row --gtid-mode=ON --enforce-gtid-consistency=ON --log-slave-updates=ON --character-set-server=utf8mb4 $MYSQL_NATIVE_PASSWORD_FLAG + environment: + MYSQL_ROOT_PASSWORD: opensesame + MYSQL_ROOT_HOST: '%' + MYSQL_DATABASE: test + MYSQL_TCP_PORT: 3310 + INIT_ROCKSDB: 1 # for percona-server + ports: + - '3310:3310' + expose: + - '3310' \ No newline at end of file diff --git a/localtests/move-tables/create.sql b/localtests/move-tables/create.sql new file mode 100644 index 000000000..46e919003 --- /dev/null +++ b/localtests/move-tables/create.sql @@ -0,0 +1,34 @@ +drop table if exists gh_ost_test; +create table gh_ost_test ( + id bigint(20) NOT NULL AUTO_INCREMENT, + column1 int(11) NOT NULL, + column2 smallint(5) unsigned NOT NULL, + column3 mediumint(8) unsigned NOT NULL, + column4 tinyint(3) unsigned NOT NULL, + column5 int(11) NOT NULL, + column6 int(11) NOT NULL, + PRIMARY KEY (id), + KEY c12_ix (column1, column2) +) auto_increment=1; + +insert into gh_ost_test values + (NULL, 1001, 100, 500000, 10, 1700000001, 1700000002), + (NULL, 1002, 200, 600000, 20, 1700000003, 1700000004), + (NULL, 1003, 300, 700000, 30, 1700000005, 1700000006), + (NULL, 1004, 400, 800000, 40, 1700000007, 1700000008), + (NULL, 1005, 500, 900000, 50, 1700000009, 1700000010), + (NULL, 1006, 600, 1000000, 60, 1700000011, 1700000012), + (NULL, 1007, 700, 1100000, 70, 1700000013, 1700000014), + (NULL, 1008, 800, 1200000, 80, 1700000015, 1700000016), + (NULL, 1009, 900, 1300000, 90, 1700000017, 1700000018), + (NULL, 1010, 1000, 1400000, 100, 1700000019, 1700000020), + (NULL, 1011, 1100, 1500000, 110, 1700000021, 1700000022), + (NULL, 1012, 1200, 1600000, 120, 1700000023, 1700000024), + (NULL, 1013, 1300, 1700000, 130, 1700000025, 1700000026), + (NULL, 1014, 1400, 1800000, 140, 1700000027, 1700000028), + (NULL, 1015, 1500, 1900000, 150, 1700000029, 1700000030), + (NULL, 1016, 1600, 2000000, 160, 1700000031, 1700000032), + (NULL, 1017, 1700, 2100000, 170, 1700000033, 1700000034), + (NULL, 1018, 1800, 2200000, 180, 1700000035, 1700000036), + (NULL, 1019, 1900, 2300000, 190, 1700000037, 1700000038), + (NULL, 1020, 2000, 2400000, 200, 1700000039, 1700000040); \ No newline at end of file diff --git a/script/move-tables/README.md b/script/move-tables/README.md new file mode 100644 index 000000000..52d138d09 --- /dev/null +++ b/script/move-tables/README.md @@ -0,0 +1,180 @@ +### Setup + +Setup the multi-cluster topology and seed the data +```bash +script/move-tables/setup +``` + +Verify data is present in the source cluster. +```bash +script/move-tables/mysql-source-primary -D gh_ost_test_db -e "SELECT * FROM gh_ost_test;" +``` + +Verify the empty database is present in the target cluster. +```bash +script/move-tables/mysql-target-primary -D gh_ost_test_db -e "SHOW TABLES;" +``` + +### Testing `gh-ost` + +Checkout your branch of `github/gh-ost` and build the binaries: +```bash +script/build --cli +``` + +Run gh-ost to move tables: +```bash +./bin/gh-ost --move-tables=gh_ost_test --host=localhost --port=3308 --user root --password opensesame --database=gh_ost_test_db --target-host=localhost --target-port=3309 --target-user root --target-password opensesame --target-database=gh_ost_test_db --execute --verbose +``` + +### WIP + +Current state based on the current outer dev loop: + + +```bash + + +\u2718 \e[2m\u2388 (\u2205) gh-ost:(move-tables/1.2-skip-ghost-tables) +> rm /tmp/gh-ost.gh_ost_test_db..sock; ./script/build --cli && ./bin/gh-ost --move-tables=gh_ost_test --host=localhost --port=3308 --user root --password opensesame --database=gh_ost_test_db --target-host=localhost --target-port=3309 --target-user root --target-password opensesame --target-database=gh_ost_test_db --execute --verbose +rm: /tmp/gh-ost.gh_ost_test_db..sock: No such file or directory +go version go1.25.9 darwin/arm64 found in : Go Binary: /opt/homebrew/bin/go +++ '[' '!' -L .gopath/src/github.com/github/gh-ost ']' +++ export GOPATH=/Users/chriskirkland/git/src/github.com/github/gh-ost/.gopath:/Users/chriskirkland/git/src/github.com/github/gh-ost/.vendor +++ GOPATH=/Users/chriskirkland/git/src/github.com/github/gh-ost/.gopath:/Users/chriskirkland/git/src/github.com/github/gh-ost/.vendor ++ mkdir -p bin ++ bindir=/Users/chriskirkland/git/src/github.com/github/gh-ost/bin ++ scriptdir=/Users/chriskirkland/git/src/github.com/github/gh-ost/script +++ git rev-parse HEAD ++ version=0508dd782e1871de9dcaa51d3f59e5ba4cd92117 +++ git describe --tags --always --dirty ++ describe=v1.1.9-19-g0508dd78-dirty ++ export GOPATH=/Users/chriskirkland/git/src/github.com/github/gh-ost/.gopath ++ GOPATH=/Users/chriskirkland/git/src/github.com/github/gh-ost/.gopath ++ cd .gopath/src/github.com/github/gh-ost ++ go build -o /Users/chriskirkland/git/src/github.com/github/gh-ost/bin/gh-ost -ldflags '-X main.AppVersion=0508dd782e1871de9dcaa51d3f59e5ba4cd92117 -X main.BuildDescribe=v1.1.9-19-g0508dd78-dirty' ./go/cmd/gh-ost/main.go +2026-06-01 16:41:13 INFO starting gh-ost 0508dd782e1871de9dcaa51d3f59e5ba4cd92117 (git commit: unknown) +2026-06-01 16:41:13 INFO Moving tables [gh_ost_test] from `gh_ost_test_db` to `gh_ost_test_db` (localhost) +2026-06-01 16:41:13 INFO inspector connection validated on localhost:3308 +2026-06-01 16:41:13 INFO User has SUPER, REPLICATION SLAVE privileges, and has ALL privileges on `gh_ost_test_db`.* +2026-06-01 16:41:13 INFO binary logs validated on localhost:3308 +2026-06-01 16:41:13 INFO Restarting replication on localhost:3308 to make sure binlog settings apply to replication thread +2026-06-01 16:41:13 INFO Inspector initiated on 3e162abb4a14:3308, version 8.0.41 +2026-06-01 16:41:13 INFO Inspector validating original table +2026-06-01 16:41:13 INFO Table found. Engine=InnoDB +2026-06-01 16:41:13 INFO Estimated number of rows via EXPLAIN: 20 +2026-06-01 16:41:13 INFO Inspector validated original table +2026-06-01 16:41:13 INFO Inspector inspected original table +2026-06-01 16:41:13 INFO log_slave_updates validated on localhost:3308 +2026-06-01 16:41:13 INFO Inspector validated and initialized +2026-06-01 16:41:13 INFO applier connection validated on localhost:3309 +2026-06-01 16:41:13 INFO applier connection validated on localhost:3309 +2026-06-01 16:41:13 INFO will use time_zone='SYSTEM' on applier +2026-06-01 16:41:13 INFO applier connection validated on localhost:3309 +2026-06-01 16:41:13 INFO Applier initiated on 381ee87dc2c6:3309, version 8.0.41 +2026-06-01 16:41:13 INFO Fetching create table statement for `gh_ost_test_db.gh_ost_test` +2026-06-01 16:41:13 INFO Create table statement: CREATE TABLE `gh_ost_test` ( + `id` bigint NOT NULL AUTO_INCREMENT, + `column1` int NOT NULL, + `column2` smallint unsigned NOT NULL, + `column3` mediumint unsigned NOT NULL, + `column4` tinyint unsigned NOT NULL, + `column5` int NOT NULL, + `column6` int NOT NULL, + PRIMARY KEY (`id`), + KEY `c12_ix` (`column1`,`column2`) +) ENGINE=InnoDB AUTO_INCREMENT=21 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci +2026-06-01 16:41:13 INFO Creating target table `gh_ost_test_db`.`gh_ost_test` +2026-06-01 16:41:13 INFO Target table created +2026-06-01 16:41:13 INFO streamer connection validated on localhost:3308 +[2026/06/01 16:41:13] [info] binlogsyncer.go:191 create BinlogSyncer with config {ServerID:99999 Flavor:mysql Host:localhost Port:3308 User:root Password: Localhost: Charset: SemiSyncEnabled:false RawModeEnabled:false TLSConfig: ParseTime:false TimestampStringLocation:UTC UseDecimal:true RecvBufferSize:0 HeartbeatPeriod:0s ReadTimeout:0s MaxReconnectAttempts:0 DisableRetrySync:false VerifyChecksum:false DumpCommandFlag:0 Option: Logger:0x14000494a20 Dialer:0x100c8c0c0 RowsEventDecodeFunc: TableMapOptionalMetaDecodeFunc: DiscardGTIDSet:false EventCacheCount:10240 SynchronousEventHandler:} +2026-06-01 16:41:13 INFO Connecting binlog streamer at mysql-bin.000003:2987908 +[2026/06/01 16:41:13] [info] binlogsyncer.go:443 begin to sync binlog from position (mysql-bin.000003, 2987908) +[2026/06/01 16:41:13] [info] binlogsyncer.go:409 Connected to mysql 8.0.41 server +2026-06-01 16:41:13 INFO Skipping stream of the changelog table [] +[2026/06/01 16:41:13] [info] binlogsyncer.go:868 rotate to (mysql-bin.000003, 2987908) +2026-06-01 16:41:13 INFO rotate to next log from mysql-bin.000003:0 to mysql-bin.000003 +2026-06-01 16:41:13 INFO Listening on unix socket file: /tmp/gh-ost.gh_ost_test_db..sock +2026-06-01 16:41:13 INFO Adding listener for gh_ost_test_db.gh_ost_test +2026-06-01 16:41:13 INFO Reading migration range according to key: PRIMARY ( + select /* gh-ost `gh_ost_test_db`.`gh_ost_test` */ `id` + from + `gh_ost_test_db`.`gh_ost_test` + force index (PRIMARY) + order by + `id` asc + limit 1) +2026-06-01 16:41:13 INFO Migration min values: [1] +2026-06-01 16:41:13 INFO Migration max values: [20] +2026-06-01 16:41:13 INFO Skipping throttling in move tables mode [] +# Migrating `gh_ost_test_db`.`gh_ost_test`; Target table is `gh_ost_test_db`.`gh_ost_test` +# Migrating 381ee87dc2c6:3309; inspecting 3e162abb4a14:3308; executing on Chriss-MBP-2 +# Migration started at Mon Jun 01 16:41:13 -0600 2026 +# chunk-size: 1000; max-lag-millis: 1500ms; dml-batch-size: 10; max-load: ; critical-load: ; nice-ratio: 0.000000 +# throttle-additional-flag-file: /tmp/gh-ost.throttle +# Serving on unix socket: /tmp/gh-ost.gh_ost_test_db..sock +Copy: 0/20 0.0%; Applied: 0; Backlog: 0/1000; Time: 0s(total), 0s(copy); streamer: mysql-bin.000003:0; Lag: 0.00s, HeartbeatLag: 9223372036.85s, State: migrating; ETA: N/A +2026-06-01 16:41:13 INFO Copy: 0/20 0.0%; Applied: 0; Backlog: 0/1000; Time: 0s(total), 0s(copy); streamer: mysql-bin.000003:0; Lag: 0.00s, HeartbeatLag: 9223372036.85s, State: migrating; ETA: N/A [] +Copy: 0/20 0.0%; Applied: 0; Backlog: 0/1000; Time: 1s(total), 1s(copy); streamer: mysql-bin.000003:0; Lag: 0.00s, HeartbeatLag: 9223372036.85s, State: migrating; ETA: N/A +2026-06-01 16:41:14 INFO [execWriteFuncs] Processing row copy function [] +2026-06-01 16:41:14 INFO Copy: 0/20 0.0%; Applied: 0; Backlog: 0/1000; Time: 1s(total), 1s(copy); streamer: mysql-bin.000003:0; Lag: 0.00s, HeartbeatLag: 9223372036.85s, State: migrating; ETA: N/A [] +2026-06-01 16:41:14 INFO ApplyIterationInsertQuery affected 20 rows +2026-06-01 16:41:14 INFO [execWriteFuncs] Processing row copy function [] +2026-06-01 16:41:14 INFO [execWriteFuncs] Processing row copy function [] +2026-06-01 16:41:14 INFO Row copy complete +2026-06-01 16:41:14 INFO Writing changelog state: Migrated +[2026/06/01 16:41:14] [info] binlogsyncer.go:225 syncer is closing... +2026-06-01 16:41:14 INFO StreamEvents encountered unexpected error: Sync was closed +github.com/go-mysql-org/go-mysql/replication.init + :1 +runtime.doInit1 + /Users/chriskirkland/git/src/github.com/github/gh-ost/.gopath/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.9.darwin-arm64/src/runtime/proc.go:7670 +runtime.doInit + /Users/chriskirkland/git/src/github.com/github/gh-ost/.gopath/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.9.darwin-arm64/src/runtime/proc.go:7637 +runtime.main + /Users/chriskirkland/git/src/github.com/github/gh-ost/.gopath/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.9.darwin-arm64/src/runtime/proc.go:256 +runtime.goexit + /Users/chriskirkland/git/src/github.com/github/gh-ost/.gopath/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.9.darwin-arm64/src/runtime/asm_arm64.s:1268 +[2026/06/01 16:41:14] [info] binlogsyncer.go:988 kill last connection id 405 +[2026/06/01 16:41:14] [info] binlogsyncer.go:255 syncer is closed +2026-06-01 16:41:14 INFO Closed streamer connection. err= +2026-06-01 16:41:14 INFO Done moving tables [gh_ost_test] from `gh_ost_test_db` to `gh_ost_test_db` (localhost) +2026-06-01 16:41:14 INFO Removing socket file: /tmp/gh-ost.gh_ost_test_db..sock +2026-06-01 16:41:14 INFO Tearing down inspector +2026-06-01 16:41:14 INFO Tearing down applier +2026-06-01 16:41:14 INFO Tearing down streamer +# Done + +``` + +:tada: :tada: :tada: :tada: +```bash + +\u2714 \e[2m\u2388 (\u2205) gh-ost-tablemove-poc:(chriskirkland/move-tables) +> ./script/move-tables/mysql-target-primary -D "gh_ost_test_db" -e "SELECT * FROM gh_ost_test;" ++----+---------+---------+---------+---------+------------+------------+ +| id | column1 | column2 | column3 | column4 | column5 | column6 | ++----+---------+---------+---------+---------+------------+------------+ +| 1 | 1001 | 100 | 500000 | 10 | 1700000001 | 1700000002 | +| 2 | 1002 | 200 | 600000 | 20 | 1700000003 | 1700000004 | +| 3 | 1003 | 300 | 700000 | 30 | 1700000005 | 1700000006 | +| 4 | 1004 | 400 | 800000 | 40 | 1700000007 | 1700000008 | +| 5 | 1005 | 500 | 900000 | 50 | 1700000009 | 1700000010 | +| 6 | 1006 | 600 | 1000000 | 60 | 1700000011 | 1700000012 | +| 7 | 1007 | 700 | 1100000 | 70 | 1700000013 | 1700000014 | +| 8 | 1008 | 800 | 1200000 | 80 | 1700000015 | 1700000016 | +| 9 | 1009 | 900 | 1300000 | 90 | 1700000017 | 1700000018 | +| 10 | 1010 | 1000 | 1400000 | 100 | 1700000019 | 1700000020 | +| 11 | 1011 | 1100 | 1500000 | 110 | 1700000021 | 1700000022 | +| 12 | 1012 | 1200 | 1600000 | 120 | 1700000023 | 1700000024 | +| 13 | 1013 | 1300 | 1700000 | 130 | 1700000025 | 1700000026 | +| 14 | 1014 | 1400 | 1800000 | 140 | 1700000027 | 1700000028 | +| 15 | 1015 | 1500 | 1900000 | 150 | 1700000029 | 1700000030 | +| 16 | 1016 | 1600 | 2000000 | 160 | 1700000031 | 1700000032 | +| 17 | 1017 | 1700 | 2100000 | 170 | 1700000033 | 1700000034 | +| 18 | 1018 | 1800 | 2200000 | 180 | 1700000035 | 1700000036 | +| 19 | 1019 | 1900 | 2300000 | 190 | 1700000037 | 1700000038 | +| 20 | 1020 | 2000 | 2400000 | 200 | 1700000039 | 1700000040 | ++----+---------+---------+---------+---------+------------+------------+ + +``` \ No newline at end of file diff --git a/script/move-tables/mysql-source-primary b/script/move-tables/mysql-source-primary new file mode 100755 index 000000000..9643924d7 --- /dev/null +++ b/script/move-tables/mysql-source-primary @@ -0,0 +1,6 @@ +#!/bin/bash +# +# This executes a command on the mysql-source-primary docker container created +# from localtests/docker-compose-move-tables.yml. + +MYSQL_PWD=opensesame mysql -uroot -h0.0.0.0 -P3307 "$@" \ No newline at end of file diff --git a/script/move-tables/mysql-source-replica b/script/move-tables/mysql-source-replica new file mode 100755 index 000000000..f42a698d4 --- /dev/null +++ b/script/move-tables/mysql-source-replica @@ -0,0 +1,6 @@ +#!/bin/bash +# +# This executes a command on the mysql-source-replica docker container created +# from localtests/docker-compose-move-tables.yml. + +MYSQL_PWD=opensesame mysql -uroot -h0.0.0.0 -P3308 "$@" \ No newline at end of file diff --git a/script/move-tables/mysql-target-primary b/script/move-tables/mysql-target-primary new file mode 100755 index 000000000..9a229231c --- /dev/null +++ b/script/move-tables/mysql-target-primary @@ -0,0 +1,6 @@ +#!/bin/bash +# +# This executes a command on the mysql-target-primary docker container created +# from localtests/docker-compose-move-tables.yml. + +MYSQL_PWD=opensesame mysql -uroot -h0.0.0.0 -P3309 "$@" \ No newline at end of file diff --git a/script/move-tables/mysql-target-replica b/script/move-tables/mysql-target-replica new file mode 100755 index 000000000..ae43d4779 --- /dev/null +++ b/script/move-tables/mysql-target-replica @@ -0,0 +1,6 @@ +#!/bin/bash +# +# This executes a command on the mysql-target-replica docker container created +# from localtests/docker-compose-move-tables.yml. + +MYSQL_PWD=opensesame mysql -uroot -h0.0.0.0 -P3310 "$@" \ No newline at end of file diff --git a/script/move-tables/setup b/script/move-tables/setup new file mode 100755 index 000000000..05fd1ece9 --- /dev/null +++ b/script/move-tables/setup @@ -0,0 +1,130 @@ +#!/bin/bash + +# This script starts four MySQL docker containers in two primary-replica clusters +# which can be used for running tablemove replica tests. +# Set the environment var TEST_MYSQL_IMAGE to change the docker image. +# +# Usage: +# setup start the containers + +set -e + +GH_OST_ROOT=$(git rev-parse --show-toplevel) +SCRIPT_PATH="${GH_OST_ROOT}/script/move-tables" +DATABASE_NAME="gh_ost_test_db" + +poll_mysql() { + CTR=0 + cmd="exec-mysql-$1" + while ! $cmd -e "select 1;" >/dev/null 2>&1; do + sleep 1 + CTR=$((CTR + 1)) + if [ $CTR -gt 30 ]; then + echo " ❌ MySQL $1 failed to start" + return 1 + fi + done + echo " ✔ MySQL $1 OK" + return 0 +} + +exec-mysql-source-primary() { + if [[ $TEST_MYSQL_IMAGE =~ "mysql:8.4" ]]; then + ${SCRIPT_PATH}/mysql-source-primary --ssl-mode=required "$@" + else + ${SCRIPT_PATH}/mysql-source-primary "$@" + fi +} + +exec-mysql-source-replica() { + if [[ $TEST_MYSQL_IMAGE =~ "mysql:8.4" ]]; then + ${SCRIPT_PATH}/mysql-source-replica --ssl-mode=required "$@" + else + ${SCRIPT_PATH}/mysql-source-replica "$@" + fi +} + +exec-mysql-target-primary() { + if [[ $TEST_MYSQL_IMAGE =~ "mysql:8.4" ]]; then + ${SCRIPT_PATH}/mysql-target-primary --ssl-mode=required "$@" + else + ${SCRIPT_PATH}/mysql-target-primary "$@" + fi +} + +exec-mysql-target-replica() { + if [[ $TEST_MYSQL_IMAGE =~ "mysql:8.4" ]]; then + ${SCRIPT_PATH}/mysql-target-replica --ssl-mode=required "$@" + else + ${SCRIPT_PATH}/mysql-target-replica "$@" + fi +} + +setup() { + [ -z "$TEST_MYSQL_IMAGE" ] && TEST_MYSQL_IMAGE="mysql:8.0.41" + + echo "Starting MySQL $TEST_MYSQL_IMAGE containers (2 clusters)..." + compose_file="$GH_OST_ROOT/localtests/docker-compose-move-tables.yml" + MYSQL_SHA2_RSA_KEYS_FLAG="" + MYSQL_PASSWORD_HASHING_ALGORITHM="mysql_native_password" + MYSQL_NATIVE_PASSWORD_FLAG="" + if [[ $TEST_MYSQL_IMAGE =~ "mysql:8.4" ]]; then + MYSQL_PASSWORD_HASHING_ALGORITHM="caching_sha2_password" + MYSQL_SHA2_RSA_KEYS_FLAG="--caching-sha2-password-auto-generate-rsa-keys=ON" + MYSQL_NATIVE_PASSWORD_FLAG="$MYSQL_SHA2_RSA_KEYS_FLAG" + fi + (TEST_MYSQL_IMAGE="$TEST_MYSQL_IMAGE" MYSQL_NATIVE_PASSWORD_FLAG="$MYSQL_NATIVE_PASSWORD_FLAG" envsubst <"$compose_file") >"$compose_file.tmp" + + docker compose -f "$compose_file.tmp" up -d --wait + + echo "Waiting for MySQL..." + poll_mysql "source-primary" || exit 1 + poll_mysql "source-replica" || exit 1 + poll_mysql "target-primary" || exit 1 + poll_mysql "target-replica" || exit 1 + + # Setup replication for source cluster, not idempotent + echo -n "Setting up replication for source cluster..." + exec-mysql-source-primary -e "create user if not exists 'repl'@'%' identified with $MYSQL_PASSWORD_HASHING_ALGORITHM by 'repl';" + exec-mysql-source-primary -e "grant replication slave on *.* to 'repl'@'%'; flush privileges;" + exec-mysql-source-primary -e "create user if not exists 'gh-ost'@'%' identified with $MYSQL_PASSWORD_HASHING_ALGORITHM by 'gh-ost';" + exec-mysql-source-primary -e "grant all on *.* to 'gh-ost'@'%';" + + sleep 1 + if [[ $TEST_MYSQL_IMAGE =~ "mysql:8.4" ]]; then + exec-mysql-source-replica -e "change replication source to source_host='mysql-source-primary', source_port=3307, source_user='repl', source_password='repl', source_auto_position=1, source_ssl=1;" + exec-mysql-source-replica -e "start replica;" + else + exec-mysql-source-replica -e "change master to master_host='mysql-source-primary', master_port=3307, master_user='repl', master_password='repl', master_auto_position=1;" + exec-mysql-source-replica -e "start slave;" + fi + echo "OK" + + # Setup replication for target cluster + echo -n "Setting up replication for target cluster..." + exec-mysql-target-primary -e "create user if not exists 'repl'@'%' identified with $MYSQL_PASSWORD_HASHING_ALGORITHM by 'repl';" + exec-mysql-target-primary -e "grant replication slave on *.* to 'repl'@'%'; flush privileges;" + exec-mysql-target-primary -e "create user if not exists 'gh-ost'@'%' identified with $MYSQL_PASSWORD_HASHING_ALGORITHM by 'gh-ost';" + exec-mysql-target-primary -e "grant all on *.* to 'gh-ost'@'%';" + + sleep 1 + if [[ $TEST_MYSQL_IMAGE =~ "mysql:8.4" ]]; then + exec-mysql-target-replica -e "change replication source to source_host='mysql-target-primary', source_port=3309, source_user='repl', source_password='repl', source_auto_position=1, source_ssl=1;" + exec-mysql-target-replica -e "start replica;" + else + exec-mysql-target-replica -e "change master to master_host='mysql-target-primary', master_port=3309, master_user='repl', master_password='repl', master_auto_position=1;" + exec-mysql-target-replica -e "start slave;" + fi + echo "OK" + + echo -n "Initializing '$DATABASE_NAME' database into each cluster..." + exec-mysql-source-primary -e "CREATE DATABASE IF NOT EXISTS $DATABASE_NAME;" + exec-mysql-target-primary -e "CREATE DATABASE IF NOT EXISTS $DATABASE_NAME;" + echo "OK" + + echo -n "Seeding data in source cluster..." + exec-mysql-source-primary -D $DATABASE_NAME < "$GH_OST_ROOT/localtests/move-tables/create.sql" + echo "OK" +} + +setup \ No newline at end of file diff --git a/script/move-tables/teardown b/script/move-tables/teardown new file mode 100755 index 000000000..edddb4b59 --- /dev/null +++ b/script/move-tables/teardown @@ -0,0 +1,20 @@ +#!/bin/bash + +# This script removes the four MySQL docker containers in two primary-replica clusters +# expected to be provisioned by the setup script. +# +# Usage: +# teardown remove the containers + +set -e + +GH_OST_ROOT=$(git rev-parse --show-toplevel) +if [[ ":$PATH:" != *":$GH_OST_ROOT:"* ]]; then + export PATH="${PATH}:${GH_OST_ROOT}/script" +fi + +echo "Stopping containers..." +docker stop mysql-source-replica mysql-source-primary mysql-target-replica mysql-target-primary 2>/dev/null || true + +echo "Removing containers..." +docker rm -f mysql-source-replica mysql-source-primary mysql-target-replica mysql-target-primary 2>/dev/null || true \ No newline at end of file From f76e78b6e99697a9eea7741958e798ec17f3b103 Mon Sep 17 00:00:00 2001 From: womoruyi Date: Thu, 11 Jun 2026 21:06:18 -0400 Subject: [PATCH 04/23] move-tables: add cooperative cutover protocol Implement the T0-T6 cooperative cutover workflow and cover atomic target-table cutover behavior. Refs: #1704 Co-authored-by: Zach Sierakowski --- go/base/context.go | 8 +- go/base/context_test.go | 19 + go/cmd/gh-ost/main.go | 12 + go/logic/migrator.go | 181 ++++++++- go/logic/migrator_move_tables_cutover_test.go | 366 ++++++++++++++++++ script/move-tables/README.md | 157 +------- script/move-tables/insert-source-primary-loop | 30 ++ script/move-tables/setup | 4 + 8 files changed, 619 insertions(+), 158 deletions(-) create mode 100644 go/logic/migrator_move_tables_cutover_test.go create mode 100755 script/move-tables/insert-source-primary-loop diff --git a/go/base/context.go b/go/base/context.go index 48eeded64..54e157885 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -514,8 +514,12 @@ func (mctx *MigrationContext) SetCutOverLockTimeoutSeconds(timeoutSeconds int64) if timeoutSeconds < 1 { return fmt.Errorf("minimal timeout is 1sec. Timeout remains at %d", mctx.CutOverLockTimeoutSeconds) } - if timeoutSeconds > 10 { - return fmt.Errorf("maximal timeout is 10sec. Timeout remains at %d", mctx.CutOverLockTimeoutSeconds) + maxTimeout := int64(10) + if mctx.IsMoveTablesMode() { + maxTimeout = 60 + } + if timeoutSeconds > maxTimeout { + return fmt.Errorf("maximal timeout is %dsec. Timeout remains at %d", maxTimeout, mctx.CutOverLockTimeoutSeconds) } mctx.CutOverLockTimeoutSeconds = timeoutSeconds return nil diff --git a/go/base/context_test.go b/go/base/context_test.go index ffbc174a4..35b1b4304 100644 --- a/go/base/context_test.go +++ b/go/base/context_test.go @@ -326,3 +326,22 @@ func TestSetAbortError_ThreadSafe(t *testing.T) { t.Errorf("Stored error %v not in list of sent errors", got) } } + +func TestSetCutOverLockTimeoutSecondsRangeByMode(t *testing.T) { + { + ctx := NewMigrationContext() + require.NoError(t, ctx.SetCutOverLockTimeoutSeconds(10)) + err := ctx.SetCutOverLockTimeoutSeconds(11) + require.Error(t, err) + require.Contains(t, err.Error(), "maximal timeout is 10sec") + } + + { + ctx := NewMigrationContext() + ctx.MoveTables.TableNames = []string{"tbl"} + require.NoError(t, ctx.SetCutOverLockTimeoutSeconds(60)) + err := ctx.SetCutOverLockTimeoutSeconds(61) + require.Error(t, err) + require.Contains(t, err.Error(), "maximal timeout is 60sec") + } +} diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index 73e4f1b76..3d520f2e9 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -199,6 +199,12 @@ func main() { flag.CommandLine.SetOutput(os.Stdout) flag.Parse() + cutOverLockTimeoutUserSpecified := false + flag.Visit(func(f *flag.Flag) { + if f.Name == "cut-over-lock-timeout-seconds" { + cutOverLockTimeoutUserSpecified = true + } + }) if *checkFlag { return @@ -365,6 +371,9 @@ func main() { if migrationContext.MoveTables.TargetHost == "" { log.Fatal("--target-host must be specified when using --move-tables") } + if migrationContext.PostponeCutOverFlagFile == "" { + log.Fatal("--postpone-cut-over-flag-file must be specified when using --move-tables") + } migrationContext.MoveTables.TableNames = strings.Split(*moveTables, ",") for i := range migrationContext.MoveTables.TableNames { migrationContext.MoveTables.TableNames[i] = strings.TrimSpace(migrationContext.MoveTables.TableNames[i]) @@ -385,6 +394,9 @@ func main() { if migrationContext.MoveTables.TargetDatabase == "" { migrationContext.MoveTables.TargetDatabase = migrationContext.DatabaseName } + if !cutOverLockTimeoutUserSpecified { + *cutOverLockTimeoutSeconds = 60 + } migrationContext.MoveTables.ConnectionConfig = mysql.NewConnectionConfig() } diff --git a/go/logic/migrator.go b/go/logic/migrator.go index 422ab1afa..b0a4d05e1 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -28,6 +28,10 @@ var ( ErrMigrationNotAllowedOnMaster = errors.New("it seems like this migration attempt to run directly on master. Preferably it would be executed on a replica (this reduces load from the master). To proceed please provide --allow-on-master") RetrySleepFn = time.Sleep checkpointTimeout = 2 * time.Second + + // moveTablesCutOverDrainPollInterval is the per-iteration sleep in T3's + // drain poll. 100ms per move_table_mode.md §1.5. + moveTablesCutOverDrainPollInterval = 100 * time.Millisecond ) type ChangelogState string @@ -846,6 +850,15 @@ func (mgtr *Migrator) MoveTables() (err error) { if err := mgtr.initiateApplier(); err != nil { return err } + if err := mgtr.checkAbort(); err != nil { + return err + } + if err := mgtr.createFlagFiles(); err != nil { + return err + } + if err := mgtr.checkAbort(); err != nil { + return err + } if err := mgtr.initiateStreaming(); err != nil { return err } @@ -907,14 +920,13 @@ func (mgtr *Migrator) MoveTables() (err error) { return err } - //TODO: cutover here + if err := mgtr.moveTablesCutOver(); err != nil { + return err + } if err := mgtr.finalCleanup(); err != nil { return nil } - if err := mgtr.hooksExecutor.OnSuccess(false); err != nil { - return err - } mgtr.migrationContext.Log.Infof("Done moving tables %v from %s to %s (%s)", mgtr.migrationContext.MoveTables.TableNames, sql.EscapeName(mgtr.migrationContext.DatabaseName), sql.EscapeName(mgtr.migrationContext.GetTargetDatabaseName()), mgtr.migrationContext.MoveTables.TargetHost) @@ -925,6 +937,167 @@ func (mgtr *Migrator) MoveTables() (err error) { return nil } +// moveTablesCutOver orchestrates the cooperative cutover protocol for move-tables +// mode. It implements the T0-T6 transitions described in +// docs/learning/design-refs/coop_cutover.md §1.3. +// +// NOT the standard cutOver() path: every internal call from cutOver() (throttle, +// atomicCutOver, waitForEventsUpToLock, heartbeat-lag) was built on a +// single-server assumption that no longer holds when the applier writes target +// and the streamer reads source. Each is replaced or dropped here. +// +// Crash safety (persisting the drain GTID before T3) is #8210. Enriched hook +// env vars (GH_OST_DRAIN_GTID, GH_OST_TARGET_*) are #8211. Target-side +// throttling is #8212. None of those are wired here. +func (mgtr *Migrator) moveTablesCutOver() (err error) { + if mgtr.migrationContext.Noop { + mgtr.migrationContext.Log.Debugf("Noop operation; not really moving tables") + return nil + } + + // ----- Postpone gate (precedes T0) ----- + // Mirrors standard cutOver()'s sleepWhileTrue postpone structure but DROPS the + // heartbeat-lag branch: move-tables mode disables _ghc heartbeat writes (#8206), + // so TimeSinceLastHeartbeatOnChangelog() returns ~58 years (time.Since(zero)) + // and would deadlock the gate forever. KEEPS the postpone-flag-file + + // unpostpone-socket gate because per coop_cutover.md §1.1 P4, operator-removes- + // postpone is the trigger for the entire cutover phase. + mgtr.migrationContext.Log.Debugf("checking for cut-over postpone") + if err := mgtr.sleepWhileTrue(func() (bool, error) { + if mgtr.migrationContext.PostponeCutOverFlagFile == "" { + return false, nil + } + if atomic.LoadInt64(&mgtr.migrationContext.UserCommandedUnpostponeFlag) > 0 { + atomic.StoreInt64(&mgtr.migrationContext.UserCommandedUnpostponeFlag, 0) + return false, nil + } + if base.FileExists(mgtr.migrationContext.PostponeCutOverFlagFile) { + if atomic.LoadInt64(&mgtr.migrationContext.IsPostponingCutOver) == 0 { + if err := mgtr.hooksExecutor.OnBeginPostponed(); err != nil { + return true, err + } + } + atomic.StoreInt64(&mgtr.migrationContext.IsPostponingCutOver, 1) + return true, nil + } + return false, nil + }); err != nil { + return err + } + atomic.StoreInt64(&mgtr.migrationContext.IsPostponingCutOver, 0) + mgtr.migrationContext.Log.Debugf("checking for cut-over postpone: complete") + + // ----- T0: on-before-cut-over hook ----- + // Non-zero hook exit aborts cutover BEFORE any source DDL fires. + if err := mgtr.hooksExecutor.OnBeforeCutOver(); err != nil { + return fmt.Errorf("on-before-cut-over hook failed: %w", err) + } + + // ----- T1 + T2: RENAME then capture @@gtid_executed on the same connection ----- + // Pin both operations to a single *sql.Conn so MySQL's within-session + // ordering guarantee makes it impossible for T2 to observe a state that + // pre-dates T1's commit. Using mgtr.inspector.db directly would let the + // pool schedule T1 and T2 on different underlying TCP connections (or, with + // a proxy, different servers), breaking the happens-before relationship. + // + // No retry on the RENAME: it is not idempotent — a partial success leaves + // the table already renamed and a retry would fail. The operator re-runs + // the whole hook chain on failure. + pinnedConn, err := mgtr.inspector.db.Conn(context.Background()) + if err != nil { + return fmt.Errorf("failed to pin connection for T1/T2: %w", err) + } + defer pinnedConn.Close() + + sourceDB := mgtr.migrationContext.DatabaseName + sourceTable := mgtr.migrationContext.OriginalTableName + delTable := mgtr.migrationContext.GetOldTableName() + renameQuery := fmt.Sprintf("RENAME TABLE %s.%s TO %s.%s", + sql.EscapeName(sourceDB), sql.EscapeName(sourceTable), + sql.EscapeName(sourceDB), sql.EscapeName(delTable)) + mgtr.migrationContext.Log.Infof("T1: renaming source table: %s", renameQuery) + if _, err := pinnedConn.ExecContext(context.Background(), renameQuery); err != nil { + return fmt.Errorf("RENAME failed: %w", err) + } + + // ----- T2: capture @@gtid_executed on the SAME connection as T1 ----- + // @@GLOBAL scope is explicit so the intent is unambiguous in the SQL itself. + // Design: https://github.com/github/gh-ost-tablemove-poc/blob/9dc6df75c4c88ff473906a497836c7518f5614ec/design/coop_cutover.md#32-correctness-verification-for-p4 + var drainGTIDStr string + if err := pinnedConn.QueryRowContext(context.Background(), "select @@gtid_executed").Scan(&drainGTIDStr); err != nil { + return fmt.Errorf("drain GTID capture failed: %w", err) + } + drainGTID, err := mysql.NewGTIDBinlogCoordinates(drainGTIDStr) + if err != nil { + return fmt.Errorf("drain GTID parse failed: %w", err) + } + mgtr.migrationContext.Log.Infof("T2: captured drain GTID: %s", drainGTID.DisplayString()) + + // ----- T3: drain poll ----- + // Wait until applier.CurrentCoordinates catches up to drainGTID. The drain + // is complete when the applier's coords are not strictly smaller than the + // drain target (i.e. the applier contains every GTID in drainGTID). Reads + // of CurrentCoordinates hold the mutex per applier.go:75. Per-iteration + // logging is Debug only to avoid spamming Info on a hot loop. + drainTimeout := time.Duration(mgtr.migrationContext.CutOverLockTimeoutSeconds) * time.Second + mgtr.migrationContext.Log.Infof("T3: draining applier to drain GTID (timeout %s, poll %s)", + drainTimeout, moveTablesCutOverDrainPollInterval) + drainCtx, cancel := context.WithTimeout(context.Background(), drainTimeout) + defer cancel() + ticker := time.NewTicker(moveTablesCutOverDrainPollInterval) + defer ticker.Stop() + for { + if err := mgtr.checkAbort(); err != nil { + return err + } + mgtr.applier.CurrentCoordinatesMutex.Lock() + applierCoords := mgtr.applier.CurrentCoordinates + mgtr.applier.CurrentCoordinatesMutex.Unlock() + applyBacklog := len(mgtr.applyEventsQueue) + streamerBacklog := 0 + if mgtr.eventsStreamer != nil { + streamerBacklog = len(mgtr.eventsStreamer.eventsChannel) + } + if applierCoords != nil && !applierCoords.IsEmpty() && !applierCoords.SmallerThan(drainGTID) && applyBacklog == 0 && streamerBacklog == 0 { + mgtr.migrationContext.Log.Infof("T3: drain complete; applier caught up to drain GTID") + break + } + if applierCoords != nil && !applierCoords.IsEmpty() && !applierCoords.SmallerThan(drainGTID) { + mgtr.migrationContext.Log.Debugf("T3: drain GTID reached but backlog remains (apply=%d, streamer=%d)", applyBacklog, streamerBacklog) + } else { + mgtr.migrationContext.Log.Debugf("T3: applier still behind drain GTID, polling") + } + select { + case <-drainCtx.Done(): + return fmt.Errorf("drain poll timed out after %s: applier did not catch up to drain GTID", drainTimeout) + case <-ticker.C: + // next iteration + } + } + + // ----- T4: set CutOverCompleteFlag ----- + // MUST be set before T5 so the streamer's canStopStreaming loop (migrator.go:256) + // can wind down in parallel with the (potentially slow) on-success hook. + // Forgetting this has no visible failure at the call site — the run silently + // hangs after cutover because eventsStreamer.StreamEvents() never returns. + atomic.StoreInt64(&mgtr.migrationContext.CutOverCompleteFlag, 1) + mgtr.migrationContext.Log.Debugf("T4: CutOverCompleteFlag set") + + // ----- T5: on-success hook ----- + // Hook unlocks user_rw@target via db-user-management and flips the + // write_cutover? feature flag. Standard env vars only — GH_OST_DRAIN_GTID + + // GH_OST_TARGET_* are #8211 (1.7), not this PR. The pre-protocol placeholder + // OnSuccess call that used to live in MoveTables() (after finalCleanup) has + // been removed so the hook fires in the order coop_cutover.md §3.2 step 6 + // requires (T5 between T4 and T6, BEFORE finalCleanup). + if err := mgtr.hooksExecutor.OnSuccess(false); err != nil { + return fmt.Errorf("on-success hook failed: %w", err) + } + + // ----- T6: return nil ----- + return nil +} + // ExecOnFailureHook executes the onFailure hook, and this method is provided as the only external // hook access point func (mgtr *Migrator) ExecOnFailureHook() (err error) { diff --git a/go/logic/migrator_move_tables_cutover_test.go b/go/logic/migrator_move_tables_cutover_test.go new file mode 100644 index 000000000..22426be7d --- /dev/null +++ b/go/logic/migrator_move_tables_cutover_test.go @@ -0,0 +1,366 @@ +package logic + +import ( + "context" + gosql "database/sql" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/github/gh-ost/go/base" + "github.com/github/gh-ost/go/binlog" + "github.com/github/gh-ost/go/mysql" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + "github.com/testcontainers/testcontainers-go" + testmysql "github.com/testcontainers/testcontainers-go/modules/mysql" +) + +// ----------------------------------------------------------------------------- +// Pure unit tests - no MySQL. These exercise the orchestration branches that +// run BEFORE T1's RENAME, so they do not require a real inspector.db. Per the +// Option A decision in commit 3's plan, the "RENAME was not attempted" check +// is a proxy assertion: m.inspector is nil, so if T1 were reached the test +// would panic instead of silently passing. +// ----------------------------------------------------------------------------- + +// TestMoveTablesCutOver_NoopShortCircuits maps to the Noop semantics decision +// in #8209-implement-protocol (Noop returns immediately, no postpone gate, no +// hooks, no RENAME). +func TestMoveTablesCutOver_NoopShortCircuits(t *testing.T) { + var calls []string + fakeHooks := &recordingHooks{name: "fake", calls: &calls} + + ctx := base.NewMigrationContext() + ctx.Noop = true + ctx.Hooks = fakeHooks + + m := NewMigrator(ctx, "test") + + require.Equal(t, int64(0), atomic.LoadInt64(&ctx.CutOverCompleteFlag), "pre-state: flag must be 0") + require.Empty(t, calls, "pre-state: no hooks recorded") + + require.NoError(t, m.moveTablesCutOver()) + + require.Equal(t, int64(0), atomic.LoadInt64(&ctx.CutOverCompleteFlag), + "post-state: Noop must not set CutOverCompleteFlag") + require.Empty(t, calls, "post-state: Noop must not fire any hook") +} + +// TestMoveTablesCutOver_OnBeforeCutOverHookAbortsBeforeRename maps to T0 in +// coop_cutover.md section 1.3 ("non-zero return code aborts cutover"). The "aborts +// BEFORE source DDL" assertion is enforced as a proxy: m.inspector is nil, so +// if T1 RENAME executed via mgtr.inspector.db, this test would panic. +func TestMoveTablesCutOver_OnBeforeCutOverHookAbortsBeforeRename(t *testing.T) { + var calls []string + boom := errors.New("hook says no") + fakeHooks := &recordingHooks{name: "fake", calls: &calls, errOn: "OnBeforeCutOver", errVal: boom} + + ctx := base.NewMigrationContext() + ctx.Hooks = fakeHooks + ctx.DatabaseName = "test" + ctx.OriginalTableName = "t" + + m := NewMigrator(ctx, "test") + + require.Equal(t, int64(0), atomic.LoadInt64(&ctx.CutOverCompleteFlag), "pre-state: flag must be 0") + require.Empty(t, calls, "pre-state: no hooks recorded") + + err := m.moveTablesCutOver() + require.Error(t, err) + require.ErrorIs(t, err, boom) + require.Contains(t, err.Error(), "on-before-cut-over hook failed") + + require.Equal(t, int64(0), atomic.LoadInt64(&ctx.CutOverCompleteFlag), + "post-state: T0 abort must leave CutOverCompleteFlag unset") + require.Equal(t, []string{"fake:OnBeforeCutOver"}, calls, + "post-state: only the failing T0 hook fires; no OnSuccess, no OnBeginPostponed") +} + +// TestMoveTablesCutOver_PostponeGateFiresOnBeginPostponedOnce maps to the +// postpone-gate decision in #8209-implement-protocol (keep OnBeginPostponed +// firing logic with the same once-per-cutover semantics as standard cutOver). +// Also exercises Edge Case Test Quality #5 by asserting both pre-state and +// post-state of IsPostponingCutOver. +func TestMoveTablesCutOver_PostponeGateFiresOnBeginPostponedOnce(t *testing.T) { + flagPath := filepath.Join(t.TempDir(), "postpone.flag") + require.NoError(t, os.WriteFile(flagPath, nil, 0o644)) + + var calls []string + boom := errors.New("we got past the gate") + fakeHooks := &recordingHooks{name: "fake", calls: &calls, errOn: "OnBeforeCutOver", errVal: boom} + + ctx := base.NewMigrationContext() + ctx.PostponeCutOverFlagFile = flagPath + ctx.Hooks = fakeHooks + + m := NewMigrator(ctx, "test") + + require.Equal(t, int64(0), atomic.LoadInt64(&ctx.IsPostponingCutOver), "pre-state: gate flag must be 0") + require.FileExists(t, flagPath, "pre-state: postpone flag file present") + require.Empty(t, calls, "pre-state: no hooks recorded") + + // Remove the flag file during the gate's first 1s sleep so the next + // poll exits cleanly. sleepWhileTrue uses time.Sleep(1 * time.Second). + go func() { + time.Sleep(500 * time.Millisecond) + _ = os.Remove(flagPath) + }() + + err := m.moveTablesCutOver() + require.ErrorIs(t, err, boom, "expect to bail at T0 hook after gate releases") + + onBegin := 0 + for _, c := range calls { + if c == "fake:OnBeginPostponed" { + onBegin++ + } + } + require.Equal(t, 1, onBegin, + "post-state: OnBeginPostponed must fire exactly once per cutover (idempotent via IsPostponingCutOver)") + require.Equal(t, int64(0), atomic.LoadInt64(&ctx.IsPostponingCutOver), + "post-state: gate must reset IsPostponingCutOver to 0 after exit") + require.Contains(t, calls, "fake:OnBeforeCutOver", + "post-state: T0 hook must fire after gate releases") + require.Equal(t, int64(0), atomic.LoadInt64(&ctx.CutOverCompleteFlag), + "post-state: hook failure must leave CutOverCompleteFlag unset") +} + +// ----------------------------------------------------------------------------- +// Integration tests - real MySQL via testcontainers, exercise T1/T2/T3. +// +// These live under a dedicated suite (NOT MigratorTestSuite) so the parent +// function name TestMoveTablesCutOver matches the `-run MoveTablesCutOver` +// verifier alongside the pure unit tests above. SetupSuite duplicates the +// testcontainer setup pattern used by MigratorTestSuite intentionally to +// keep this commit's diff strictly additive (no edits to existing tests or +// shared helpers, per the commit-3 prompt). +// +// Known-environmental flakes when Docker/testcontainers is unavailable mirror +// the same class as #8206; they are not regressions of #8209. +// ----------------------------------------------------------------------------- + +type MoveTablesCutOverSuite struct { + suite.Suite + mysqlContainer testcontainers.Container + db *gosql.DB +} + +func (s *MoveTablesCutOverSuite) SetupSuite() { + ctx := context.Background() + mysqlContainer, err := testmysql.Run(ctx, + testMysqlContainerImage, + testmysql.WithDatabase(testMysqlDatabase), + testmysql.WithUsername(testMysqlUser), + testmysql.WithPassword(testMysqlPass), + testmysql.WithConfigFile("my.cnf.test"), + ) + s.Require().NoError(err) + s.mysqlContainer = mysqlContainer + + dsn, err := mysqlContainer.ConnectionString(ctx) + s.Require().NoError(err) + db, err := gosql.Open("mysql", dsn) + s.Require().NoError(err) + s.db = db +} + +func (s *MoveTablesCutOverSuite) TearDownSuite() { + s.Assert().NoError(s.db.Close()) + s.Assert().NoError(testcontainers.TerminateContainer(s.mysqlContainer)) +} + +func (s *MoveTablesCutOverSuite) SetupTest() { + _, err := s.db.ExecContext(context.Background(), "CREATE DATABASE IF NOT EXISTS "+testMysqlDatabase) + s.Require().NoError(err) +} + +func (s *MoveTablesCutOverSuite) TearDownTest() { + ctx := context.Background() + _, _ = s.db.ExecContext(ctx, "DROP TABLE IF EXISTS "+getTestTableName()) + _, _ = s.db.ExecContext(ctx, "DROP TABLE IF EXISTS "+getTestOldTableName()) +} + +// containingDrainGTID returns a fabricated GTID set guaranteed to contain any +// GTID the test container will assign during the test. RENAME's GTID will be +// a strict subset of this range, so T3's containment poll passes on iteration 1. +func (s *MoveTablesCutOverSuite) containingDrainGTID() *mysql.GTIDBinlogCoordinates { + var serverUUID string + s.Require().NoError(s.db.QueryRow("SELECT @@server_uuid").Scan(&serverUUID)) + g, err := mysql.NewGTIDBinlogCoordinates(fmt.Sprintf("%s:1-99999999", serverUUID)) + s.Require().NoError(err) + return g +} + +// buildMigrator wires a Migrator with the test container's *sql.DB pinned to +// inspector.db and a fresh Applier. initialCoords may be nil for the drain- +// timeout case. +func (s *MoveTablesCutOverSuite) buildMigrator(fakeHooks *recordingHooks, initialCoords mysql.BinlogCoordinates) (*Migrator, *base.MigrationContext) { + ctx := context.Background() + connectionConfig, err := getTestConnectionConfig(ctx, s.mysqlContainer) + s.Require().NoError(err) + + mc := newTestMigrationContext() + mc.ApplierConnectionConfig = connectionConfig + mc.InspectorConnectionConfig = connectionConfig + mc.SetConnectionConfig("innodb") + mc.Hooks = fakeHooks + + m := NewMigrator(mc, "test") + m.inspector = &Inspector{db: s.db, migrationContext: mc} + m.applier = NewApplier(mc) + if initialCoords != nil { + m.applier.CurrentCoordinatesMutex.Lock() + m.applier.CurrentCoordinates = initialCoords + m.applier.CurrentCoordinatesMutex.Unlock() + } + return m, mc +} + +// TestHappyPath drives the full T0-T6 protocol against the test container. +// Asserts hook ordering (T0 then T5), T4 flag set, and the source-side rename. +// Maps to acceptance criterion #8209 "RENAME executes; drain completes; +// CutOverCompleteFlag set; on-success hook fires". +func (s *MoveTablesCutOverSuite) TestHappyPath() { + ctx := context.Background() + _, err := s.db.ExecContext(ctx, fmt.Sprintf("CREATE TABLE %s (id INT PRIMARY KEY)", getTestTableName())) + s.Require().NoError(err) + + var calls []string + fakeHooks := &recordingHooks{name: "fake", calls: &calls} + m, mc := s.buildMigrator(fakeHooks, s.containingDrainGTID()) + + s.Require().Equal(int64(0), atomic.LoadInt64(&mc.CutOverCompleteFlag), "pre-state: flag must be 0") + s.Require().Empty(calls, "pre-state: no hooks recorded") + + s.Require().NoError(m.moveTablesCutOver()) + + s.Require().Equal(int64(1), atomic.LoadInt64(&mc.CutOverCompleteFlag), + "post-state: T4 must set CutOverCompleteFlag before T5/T6") + s.Require().Equal([]string{"fake:OnBeforeCutOver", "fake:OnSuccess"}, calls, + "post-state: T0 hook precedes T5 hook") + + // Source-side post-state (Edge Case Test Quality #5: assert both sides). + var renamed string + s.Require().NoError(s.db.QueryRow(fmt.Sprintf("SHOW TABLES IN %s LIKE '_%s_del'", + testMysqlDatabase, testMysqlTableName)).Scan(&renamed)) + s.Require().Equal("_"+testMysqlTableName+"_del", renamed) + + err = s.db.QueryRow(fmt.Sprintf("SHOW TABLES IN %s LIKE '%s'", + testMysqlDatabase, testMysqlTableName)).Scan(&renamed) + s.Require().ErrorIs(err, gosql.ErrNoRows, "original table must no longer exist under its old name") +} + +// TestRenameFailurePropagates maps to T1 failure handling: no source table -> +// RENAME errors. Verify the wrapped error, no CutOverCompleteFlag set, no +// OnSuccess fired. +func (s *MoveTablesCutOverSuite) TestRenameFailurePropagates() { + // Deliberately no CREATE TABLE. + var calls []string + fakeHooks := &recordingHooks{name: "fake", calls: &calls} + m, mc := s.buildMigrator(fakeHooks, s.containingDrainGTID()) + + s.Require().Equal(int64(0), atomic.LoadInt64(&mc.CutOverCompleteFlag), "pre-state: flag must be 0") + + err := m.moveTablesCutOver() + s.Require().Error(err) + s.Require().Contains(err.Error(), "RENAME failed") + + s.Require().Equal(int64(0), atomic.LoadInt64(&mc.CutOverCompleteFlag), + "post-state: RENAME failure must leave CutOverCompleteFlag unset") + for _, c := range calls { + s.Require().NotEqual("fake:OnSuccess", c, "OnSuccess must not fire when RENAME fails") + } + s.Require().Equal([]string{"fake:OnBeforeCutOver"}, calls, + "post-state: only T0 fires before the failed RENAME") +} + +// TestDrainTimeoutPropagates maps to T3 timeout handling: applier coordinates +// never reach the drain GTID -> drain poll bounded by CutOverLockTimeoutSeconds +// returns a wrapped error and the flag is not set. +func (s *MoveTablesCutOverSuite) TestDrainTimeoutPropagates() { + ctx := context.Background() + _, err := s.db.ExecContext(ctx, fmt.Sprintf("CREATE TABLE %s (id INT PRIMARY KEY)", getTestTableName())) + s.Require().NoError(err) + + // Patch poll interval and use a 1-second drain timeout for a bounded test. + origPoll := moveTablesCutOverDrainPollInterval + moveTablesCutOverDrainPollInterval = 50 * time.Millisecond + s.T().Cleanup(func() { + moveTablesCutOverDrainPollInterval = origPoll + }) + + var calls []string + fakeHooks := &recordingHooks{name: "fake", calls: &calls} + // initialCoords nil - drain comparison never satisfies. + m, mc := s.buildMigrator(fakeHooks, nil) + mc.CutOverLockTimeoutSeconds = 1 + + s.Require().Equal(int64(0), atomic.LoadInt64(&mc.CutOverCompleteFlag), "pre-state: flag must be 0") + + err = m.moveTablesCutOver() + s.Require().Error(err) + s.Require().Contains(err.Error(), "drain poll timed out") + s.Require().True(strings.HasPrefix(err.Error(), "drain poll timed out"), + "drain timeout error must name the drain") + + s.Require().Equal(int64(0), atomic.LoadInt64(&mc.CutOverCompleteFlag), + "post-state: drain timeout must abort before T4 flag set") + for _, c := range calls { + s.Require().NotEqual("fake:OnSuccess", c, "OnSuccess must not fire on drain timeout") + } + s.Require().Equal([]string{"fake:OnBeforeCutOver"}, calls, + "post-state: only T0 fires before the drain loop") +} + +// TestDrainWaitsForQueuedDML ensures T3 does not declare success just because +// applier.CurrentCoordinates already contains the drain GTID while there is +// still source-table DML queued for application. +func (s *MoveTablesCutOverSuite) TestDrainWaitsForQueuedDML() { + ctx := context.Background() + _, err := s.db.ExecContext(ctx, fmt.Sprintf("CREATE TABLE %s (id INT PRIMARY KEY)", getTestTableName())) + s.Require().NoError(err) + + origPoll := moveTablesCutOverDrainPollInterval + moveTablesCutOverDrainPollInterval = 50 * time.Millisecond + s.T().Cleanup(func() { + moveTablesCutOverDrainPollInterval = origPoll + }) + + var calls []string + fakeHooks := &recordingHooks{name: "fake", calls: &calls} + m, mc := s.buildMigrator(fakeHooks, s.containingDrainGTID()) + mc.CutOverLockTimeoutSeconds = 1 + m.applyEventsQueue <- newApplyEventStructByDML(&binlog.BinlogEntry{ + DmlEvent: &binlog.BinlogDMLEvent{ + DatabaseName: testMysqlDatabase, + TableName: testMysqlTableName, + DML: binlog.InsertDML, + }, + Coordinates: s.containingDrainGTID(), + }) + + s.Require().Equal(int64(0), atomic.LoadInt64(&mc.CutOverCompleteFlag), "pre-state: flag must be 0") + err = m.moveTablesCutOver() + s.Require().Error(err) + s.Require().Contains(err.Error(), "drain poll timed out") + s.Require().Equal(int64(0), atomic.LoadInt64(&mc.CutOverCompleteFlag), + "post-state: queued DML must keep T3 from reaching T4") + for _, c := range calls { + s.Require().NotEqual("fake:OnSuccess", c, "OnSuccess must not fire while backlog remains") + } + s.Require().Equal([]string{"fake:OnBeforeCutOver"}, calls, + "post-state: only T0 fires before the drain loop times out") +} + +func TestMoveTablesCutOver(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration suite in short mode") + } + suite.Run(t, new(MoveTablesCutOverSuite)) +} diff --git a/script/move-tables/README.md b/script/move-tables/README.md index 52d138d09..db57f750a 100644 --- a/script/move-tables/README.md +++ b/script/move-tables/README.md @@ -24,157 +24,10 @@ script/build --cli Run gh-ost to move tables: ```bash -./bin/gh-ost --move-tables=gh_ost_test --host=localhost --port=3308 --user root --password opensesame --database=gh_ost_test_db --target-host=localhost --target-port=3309 --target-user root --target-password opensesame --target-database=gh_ost_test_db --execute --verbose +./script/build --cli; ./bin/gh-ost --move-tables=gh_ost_test --host=localhost --port=3307 --user root --password opensesame --database=gh_ost_test_db --target-host=localhost --target-port=3309 --target-user root --target-password opensesame --target-database=gh_ost_test_db --postpone-cut-over-flag-file=/tmp/ghost-move-tables.postpone.flag --execute --verbose ``` -### WIP - -Current state based on the current outer dev loop: - - -```bash - - -\u2718 \e[2m\u2388 (\u2205) gh-ost:(move-tables/1.2-skip-ghost-tables) -> rm /tmp/gh-ost.gh_ost_test_db..sock; ./script/build --cli && ./bin/gh-ost --move-tables=gh_ost_test --host=localhost --port=3308 --user root --password opensesame --database=gh_ost_test_db --target-host=localhost --target-port=3309 --target-user root --target-password opensesame --target-database=gh_ost_test_db --execute --verbose -rm: /tmp/gh-ost.gh_ost_test_db..sock: No such file or directory -go version go1.25.9 darwin/arm64 found in : Go Binary: /opt/homebrew/bin/go -++ '[' '!' -L .gopath/src/github.com/github/gh-ost ']' -++ export GOPATH=/Users/chriskirkland/git/src/github.com/github/gh-ost/.gopath:/Users/chriskirkland/git/src/github.com/github/gh-ost/.vendor -++ GOPATH=/Users/chriskirkland/git/src/github.com/github/gh-ost/.gopath:/Users/chriskirkland/git/src/github.com/github/gh-ost/.vendor -+ mkdir -p bin -+ bindir=/Users/chriskirkland/git/src/github.com/github/gh-ost/bin -+ scriptdir=/Users/chriskirkland/git/src/github.com/github/gh-ost/script -++ git rev-parse HEAD -+ version=0508dd782e1871de9dcaa51d3f59e5ba4cd92117 -++ git describe --tags --always --dirty -+ describe=v1.1.9-19-g0508dd78-dirty -+ export GOPATH=/Users/chriskirkland/git/src/github.com/github/gh-ost/.gopath -+ GOPATH=/Users/chriskirkland/git/src/github.com/github/gh-ost/.gopath -+ cd .gopath/src/github.com/github/gh-ost -+ go build -o /Users/chriskirkland/git/src/github.com/github/gh-ost/bin/gh-ost -ldflags '-X main.AppVersion=0508dd782e1871de9dcaa51d3f59e5ba4cd92117 -X main.BuildDescribe=v1.1.9-19-g0508dd78-dirty' ./go/cmd/gh-ost/main.go -2026-06-01 16:41:13 INFO starting gh-ost 0508dd782e1871de9dcaa51d3f59e5ba4cd92117 (git commit: unknown) -2026-06-01 16:41:13 INFO Moving tables [gh_ost_test] from `gh_ost_test_db` to `gh_ost_test_db` (localhost) -2026-06-01 16:41:13 INFO inspector connection validated on localhost:3308 -2026-06-01 16:41:13 INFO User has SUPER, REPLICATION SLAVE privileges, and has ALL privileges on `gh_ost_test_db`.* -2026-06-01 16:41:13 INFO binary logs validated on localhost:3308 -2026-06-01 16:41:13 INFO Restarting replication on localhost:3308 to make sure binlog settings apply to replication thread -2026-06-01 16:41:13 INFO Inspector initiated on 3e162abb4a14:3308, version 8.0.41 -2026-06-01 16:41:13 INFO Inspector validating original table -2026-06-01 16:41:13 INFO Table found. Engine=InnoDB -2026-06-01 16:41:13 INFO Estimated number of rows via EXPLAIN: 20 -2026-06-01 16:41:13 INFO Inspector validated original table -2026-06-01 16:41:13 INFO Inspector inspected original table -2026-06-01 16:41:13 INFO log_slave_updates validated on localhost:3308 -2026-06-01 16:41:13 INFO Inspector validated and initialized -2026-06-01 16:41:13 INFO applier connection validated on localhost:3309 -2026-06-01 16:41:13 INFO applier connection validated on localhost:3309 -2026-06-01 16:41:13 INFO will use time_zone='SYSTEM' on applier -2026-06-01 16:41:13 INFO applier connection validated on localhost:3309 -2026-06-01 16:41:13 INFO Applier initiated on 381ee87dc2c6:3309, version 8.0.41 -2026-06-01 16:41:13 INFO Fetching create table statement for `gh_ost_test_db.gh_ost_test` -2026-06-01 16:41:13 INFO Create table statement: CREATE TABLE `gh_ost_test` ( - `id` bigint NOT NULL AUTO_INCREMENT, - `column1` int NOT NULL, - `column2` smallint unsigned NOT NULL, - `column3` mediumint unsigned NOT NULL, - `column4` tinyint unsigned NOT NULL, - `column5` int NOT NULL, - `column6` int NOT NULL, - PRIMARY KEY (`id`), - KEY `c12_ix` (`column1`,`column2`) -) ENGINE=InnoDB AUTO_INCREMENT=21 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci -2026-06-01 16:41:13 INFO Creating target table `gh_ost_test_db`.`gh_ost_test` -2026-06-01 16:41:13 INFO Target table created -2026-06-01 16:41:13 INFO streamer connection validated on localhost:3308 -[2026/06/01 16:41:13] [info] binlogsyncer.go:191 create BinlogSyncer with config {ServerID:99999 Flavor:mysql Host:localhost Port:3308 User:root Password: Localhost: Charset: SemiSyncEnabled:false RawModeEnabled:false TLSConfig: ParseTime:false TimestampStringLocation:UTC UseDecimal:true RecvBufferSize:0 HeartbeatPeriod:0s ReadTimeout:0s MaxReconnectAttempts:0 DisableRetrySync:false VerifyChecksum:false DumpCommandFlag:0 Option: Logger:0x14000494a20 Dialer:0x100c8c0c0 RowsEventDecodeFunc: TableMapOptionalMetaDecodeFunc: DiscardGTIDSet:false EventCacheCount:10240 SynchronousEventHandler:} -2026-06-01 16:41:13 INFO Connecting binlog streamer at mysql-bin.000003:2987908 -[2026/06/01 16:41:13] [info] binlogsyncer.go:443 begin to sync binlog from position (mysql-bin.000003, 2987908) -[2026/06/01 16:41:13] [info] binlogsyncer.go:409 Connected to mysql 8.0.41 server -2026-06-01 16:41:13 INFO Skipping stream of the changelog table [] -[2026/06/01 16:41:13] [info] binlogsyncer.go:868 rotate to (mysql-bin.000003, 2987908) -2026-06-01 16:41:13 INFO rotate to next log from mysql-bin.000003:0 to mysql-bin.000003 -2026-06-01 16:41:13 INFO Listening on unix socket file: /tmp/gh-ost.gh_ost_test_db..sock -2026-06-01 16:41:13 INFO Adding listener for gh_ost_test_db.gh_ost_test -2026-06-01 16:41:13 INFO Reading migration range according to key: PRIMARY ( - select /* gh-ost `gh_ost_test_db`.`gh_ost_test` */ `id` - from - `gh_ost_test_db`.`gh_ost_test` - force index (PRIMARY) - order by - `id` asc - limit 1) -2026-06-01 16:41:13 INFO Migration min values: [1] -2026-06-01 16:41:13 INFO Migration max values: [20] -2026-06-01 16:41:13 INFO Skipping throttling in move tables mode [] -# Migrating `gh_ost_test_db`.`gh_ost_test`; Target table is `gh_ost_test_db`.`gh_ost_test` -# Migrating 381ee87dc2c6:3309; inspecting 3e162abb4a14:3308; executing on Chriss-MBP-2 -# Migration started at Mon Jun 01 16:41:13 -0600 2026 -# chunk-size: 1000; max-lag-millis: 1500ms; dml-batch-size: 10; max-load: ; critical-load: ; nice-ratio: 0.000000 -# throttle-additional-flag-file: /tmp/gh-ost.throttle -# Serving on unix socket: /tmp/gh-ost.gh_ost_test_db..sock -Copy: 0/20 0.0%; Applied: 0; Backlog: 0/1000; Time: 0s(total), 0s(copy); streamer: mysql-bin.000003:0; Lag: 0.00s, HeartbeatLag: 9223372036.85s, State: migrating; ETA: N/A -2026-06-01 16:41:13 INFO Copy: 0/20 0.0%; Applied: 0; Backlog: 0/1000; Time: 0s(total), 0s(copy); streamer: mysql-bin.000003:0; Lag: 0.00s, HeartbeatLag: 9223372036.85s, State: migrating; ETA: N/A [] -Copy: 0/20 0.0%; Applied: 0; Backlog: 0/1000; Time: 1s(total), 1s(copy); streamer: mysql-bin.000003:0; Lag: 0.00s, HeartbeatLag: 9223372036.85s, State: migrating; ETA: N/A -2026-06-01 16:41:14 INFO [execWriteFuncs] Processing row copy function [] -2026-06-01 16:41:14 INFO Copy: 0/20 0.0%; Applied: 0; Backlog: 0/1000; Time: 1s(total), 1s(copy); streamer: mysql-bin.000003:0; Lag: 0.00s, HeartbeatLag: 9223372036.85s, State: migrating; ETA: N/A [] -2026-06-01 16:41:14 INFO ApplyIterationInsertQuery affected 20 rows -2026-06-01 16:41:14 INFO [execWriteFuncs] Processing row copy function [] -2026-06-01 16:41:14 INFO [execWriteFuncs] Processing row copy function [] -2026-06-01 16:41:14 INFO Row copy complete -2026-06-01 16:41:14 INFO Writing changelog state: Migrated -[2026/06/01 16:41:14] [info] binlogsyncer.go:225 syncer is closing... -2026-06-01 16:41:14 INFO StreamEvents encountered unexpected error: Sync was closed -github.com/go-mysql-org/go-mysql/replication.init - :1 -runtime.doInit1 - /Users/chriskirkland/git/src/github.com/github/gh-ost/.gopath/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.9.darwin-arm64/src/runtime/proc.go:7670 -runtime.doInit - /Users/chriskirkland/git/src/github.com/github/gh-ost/.gopath/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.9.darwin-arm64/src/runtime/proc.go:7637 -runtime.main - /Users/chriskirkland/git/src/github.com/github/gh-ost/.gopath/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.9.darwin-arm64/src/runtime/proc.go:256 -runtime.goexit - /Users/chriskirkland/git/src/github.com/github/gh-ost/.gopath/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.9.darwin-arm64/src/runtime/asm_arm64.s:1268 -[2026/06/01 16:41:14] [info] binlogsyncer.go:988 kill last connection id 405 -[2026/06/01 16:41:14] [info] binlogsyncer.go:255 syncer is closed -2026-06-01 16:41:14 INFO Closed streamer connection. err= -2026-06-01 16:41:14 INFO Done moving tables [gh_ost_test] from `gh_ost_test_db` to `gh_ost_test_db` (localhost) -2026-06-01 16:41:14 INFO Removing socket file: /tmp/gh-ost.gh_ost_test_db..sock -2026-06-01 16:41:14 INFO Tearing down inspector -2026-06-01 16:41:14 INFO Tearing down applier -2026-06-01 16:41:14 INFO Tearing down streamer -# Done - -``` - -:tada: :tada: :tada: :tada: -```bash - -\u2714 \e[2m\u2388 (\u2205) gh-ost-tablemove-poc:(chriskirkland/move-tables) -> ./script/move-tables/mysql-target-primary -D "gh_ost_test_db" -e "SELECT * FROM gh_ost_test;" -+----+---------+---------+---------+---------+------------+------------+ -| id | column1 | column2 | column3 | column4 | column5 | column6 | -+----+---------+---------+---------+---------+------------+------------+ -| 1 | 1001 | 100 | 500000 | 10 | 1700000001 | 1700000002 | -| 2 | 1002 | 200 | 600000 | 20 | 1700000003 | 1700000004 | -| 3 | 1003 | 300 | 700000 | 30 | 1700000005 | 1700000006 | -| 4 | 1004 | 400 | 800000 | 40 | 1700000007 | 1700000008 | -| 5 | 1005 | 500 | 900000 | 50 | 1700000009 | 1700000010 | -| 6 | 1006 | 600 | 1000000 | 60 | 1700000011 | 1700000012 | -| 7 | 1007 | 700 | 1100000 | 70 | 1700000013 | 1700000014 | -| 8 | 1008 | 800 | 1200000 | 80 | 1700000015 | 1700000016 | -| 9 | 1009 | 900 | 1300000 | 90 | 1700000017 | 1700000018 | -| 10 | 1010 | 1000 | 1400000 | 100 | 1700000019 | 1700000020 | -| 11 | 1011 | 1100 | 1500000 | 110 | 1700000021 | 1700000022 | -| 12 | 1012 | 1200 | 1600000 | 120 | 1700000023 | 1700000024 | -| 13 | 1013 | 1300 | 1700000 | 130 | 1700000025 | 1700000026 | -| 14 | 1014 | 1400 | 1800000 | 140 | 1700000027 | 1700000028 | -| 15 | 1015 | 1500 | 1900000 | 150 | 1700000029 | 1700000030 | -| 16 | 1016 | 1600 | 2000000 | 160 | 1700000031 | 1700000032 | -| 17 | 1017 | 1700 | 2100000 | 170 | 1700000033 | 1700000034 | -| 18 | 1018 | 1800 | 2200000 | 180 | 1700000035 | 1700000036 | -| 19 | 1019 | 1900 | 2300000 | 190 | 1700000037 | 1700000038 | -| 20 | 1020 | 2000 | 2400000 | 200 | 1700000039 | 1700000040 | -+----+---------+---------+---------+---------+------------+------------+ - -``` \ No newline at end of file +Note: replicas in this local topology are configured with `read_only=ON` and +`super_read_only=ON`. If you point `--host` at `mysql-source-replica` (3308), +the cutover `RENAME TABLE` step will fail by design. Use source primary (3307) +as the inspected host when you want cutover to rename on source. diff --git a/script/move-tables/insert-source-primary-loop b/script/move-tables/insert-source-primary-loop new file mode 100755 index 000000000..c5571ff1c --- /dev/null +++ b/script/move-tables/insert-source-primary-loop @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Continuously insert new rows into gh_ost_test on source primary. +# Usage: +# script/move-tables/insert-source-primary-loop [start_column1] [sleep_seconds] +# Example: +# script/move-tables/insert-source-primary-loop 100000 0.2 + +start_i="${1:-100000}" +delay="${2:-0.2}" +i="$start_i" + +echo "Starting continuous inserts on source primary. Press Ctrl+C to stop." +echo "start_column1=$start_i sleep_seconds=$delay" + +trap 'echo; echo "Stopped."; exit 0' INT TERM + +while true; do + ts="$(date +%s)" + + script/move-tables/mysql-source-primary -D gh_ost_test_db -e " + INSERT INTO gh_ost_test (column1, column2, column3, column4, column5, column6) + VALUES ($i, $((i % 65535)), $((i % 16777215)), $((i % 255)), $ts, $((ts + 1))); + " + + echo "inserted row: column1=$i ts=$ts" + i=$((i + 1)) + sleep "$delay" +done diff --git a/script/move-tables/setup b/script/move-tables/setup index 05fd1ece9..813e7203e 100755 --- a/script/move-tables/setup +++ b/script/move-tables/setup @@ -98,6 +98,9 @@ setup() { exec-mysql-source-replica -e "change master to master_host='mysql-source-primary', master_port=3307, master_user='repl', master_password='repl', master_auto_position=1;" exec-mysql-source-replica -e "start slave;" fi + # Keep replicas non-writable so local move-tables runs that point --host at + # a replica fail at cutover (RENAME) instead of mutating only the replica. + exec-mysql-source-replica -e "set global read_only=ON; set global super_read_only=ON;" echo "OK" # Setup replication for target cluster @@ -115,6 +118,7 @@ setup() { exec-mysql-target-replica -e "change master to master_host='mysql-target-primary', master_port=3309, master_user='repl', master_password='repl', master_auto_position=1;" exec-mysql-target-replica -e "start slave;" fi + exec-mysql-target-replica -e "set global read_only=ON; set global super_read_only=ON;" echo "OK" echo -n "Initializing '$DATABASE_NAME' database into each cluster..." From bd49227701a41c4fd442abe9104077e30a5bf831 Mon Sep 17 00:00:00 2001 From: womoruyi Date: Fri, 12 Jun 2026 20:43:39 +0000 Subject: [PATCH 05/23] move-tables: create and validate target tables Create target tables in the destination database and abort safely when target tables already exist. Refs: #1710 --- go/logic/applier.go | 37 +++++- go/logic/applier_test.go | 108 ++++++++++++++++++ go/logic/migrator.go | 4 +- go/logic/migrator_move_tables_cutover_test.go | 2 +- 4 files changed, 144 insertions(+), 7 deletions(-) diff --git a/go/logic/applier.go b/go/logic/applier.go index bc355549e..fe9c9ad3b 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -669,7 +669,8 @@ func (apl *Applier) AnalyzeGhostTable() error { } // createTargetTableFromStatement creates the table on the applier host to which the applier will -// apply changes. +// apply changes. In move-tables mode this executes on moveTablesTargetDB (the target cluster); +// in standard mode it executes on apl.db (the source/applier host). func (apl *Applier) createTargetTableFromStatement(targetTableName, createStatement string) error { targetDatabase := apl.migrationContext.GetTargetDatabaseName() apl.migrationContext.Log.Infof("Creating target table %s.%s", @@ -677,8 +678,13 @@ func (apl *Applier) createTargetTableFromStatement(targetTableName, createStatem sql.EscapeName(targetTableName), ) + db := apl.db + if apl.migrationContext.IsMoveTablesMode() { + db = apl.moveTablesTargetDB + } + err := func() error { - tx, err := apl.db.Begin() + tx, err := db.Begin() if err != nil { return err } @@ -713,12 +719,35 @@ func (apl *Applier) CreateGhostTable() error { return apl.createTargetTable(apl.migrationContext.GetGhostTableName()) } -// CreateTargetTable creates the target table on the target host (for move-tables) +// CreateTargetTable creates the target table on the target host (for move-tables). +// It aborts with an error if the target table already exists on the target cluster, +// to prevent silently writing into a table that has unrelated data or a different +// schema (move_table_mode.md §1.3: "Don't use IF NOT EXISTS for the target table. +// An existing table is an error condition, not a no-op."). func (apl *Applier) CreateTargetTable(createStatement string) error { if !apl.migrationContext.IsMoveTablesMode() { return errors.New("CreateTargetTable is only available in MoveTables mode") } - return apl.createTargetTableFromStatement(apl.originalTableName(), createStatement) + targetTableName := apl.originalTableName() + targetDatabase := apl.migrationContext.GetTargetDatabaseName() + + // Explicit pre-check: abort before any data is copied if the target table + // already exists. The CREATE TABLE would also fail (MySQL ERROR 1050), but + // this gives operators a clear gh-ost error message explaining what to do. + var count int + err := apl.moveTablesTargetDB.QueryRow( + "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema=? AND table_name=?", + targetDatabase, targetTableName, + ).Scan(&count) + if err != nil { + return fmt.Errorf("failed to check for existing target table: %w", err) + } + if count > 0 { + return fmt.Errorf("target table %s.%s already exists on the target cluster. Aborting to prevent writing into a table with unrelated data. Drop the table manually if this is intentional", + sql.EscapeName(targetDatabase), sql.EscapeName(targetTableName)) + } + + return apl.createTargetTableFromStatement(targetTableName, createStatement) } // AlterGhost applies `alter` statement on ghost table diff --git a/go/logic/applier_test.go b/go/logic/applier_test.go index f3d620ca3..7cdc0fa6d 100644 --- a/go/logic/applier_test.go +++ b/go/logic/applier_test.go @@ -879,6 +879,114 @@ func (suite *ApplierTestSuite) TestAnalyzeGhostTable() { suite.Require().ErrorContains(applier.AnalyzeGhostTable(), "failed; refusing cut-over") } +// TestCreateTargetTable_HappyPath exercises #8207 AC #1: +// "A move table run targeting a clean target schema creates the migrated table +// with a SHOW CREATE TABLE output equivalent to the source's." +// +// It calls CreateTargetTable (not just IsMoveTablesMode()), asserts the table +// exists on the target database, verifies schema equivalence via SHOW CREATE TABLE, +// and confirms no table was accidentally created on the source. +func (suite *ApplierTestSuite) TestCreateTargetTable_HappyPath() { + ctx := context.Background() + + _, err := suite.db.ExecContext(ctx, fmt.Sprintf("CREATE TABLE %s (id INT PRIMARY KEY, name VARCHAR(64), updated_at DATETIME);", getTestTableName())) + suite.Require().NoError(err) + + connectionConfig, err := getTestConnectionConfig(ctx, suite.mysqlContainer) + suite.Require().NoError(err) + + migrationContext := newTestMigrationContext() + migrationContext.MoveTables.TableNames = []string{testMysqlTableName} + migrationContext.MoveTables.TargetDatabase = testMysqlDatabaseOther + migrationContext.ApplierConnectionConfig = connectionConfig + migrationContext.MoveTables.ConnectionConfig = connectionConfig + migrationContext.SetConnectionConfig("innodb") + migrationContext.OriginalTableColumns = sql.NewColumnList([]string{"id", "name", "updated_at"}) + + applier := NewApplier(migrationContext) + defer applier.Teardown() + + err = applier.InitDBConnections() + suite.Require().NoError(err) + + var dummy, sourceCreateDDL string + err = suite.db.QueryRow(fmt.Sprintf("SHOW CREATE TABLE %s", getTestTableName())).Scan(&dummy, &sourceCreateDDL) + suite.Require().NoError(err) + + var count int + err = suite.otherDB.QueryRow( + "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema=? AND table_name=?", + testMysqlDatabaseOther, testMysqlTableName, + ).Scan(&count) + suite.Require().NoError(err) + suite.Require().Equal(0, count, "precondition: target table must not exist before CreateTargetTable") + + err = applier.CreateTargetTable(sourceCreateDDL) + suite.Require().NoError(err) + + var targetTableName, targetCreateDDL string + err = suite.otherDB.QueryRow(fmt.Sprintf("SHOW CREATE TABLE `%s`.`%s`", testMysqlDatabaseOther, testMysqlTableName)).Scan(&targetTableName, &targetCreateDDL) + suite.Require().NoError(err) + suite.Require().Equal(testMysqlTableName, targetTableName) + suite.Require().Equal(sourceCreateDDL, targetCreateDDL, "target table schema must be equivalent to source") + + err = suite.otherDB.QueryRow( + "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema=? AND table_name=?", + testMysqlDatabaseOther, testMysqlTableName, + ).Scan(&count) + suite.Require().NoError(err) + suite.Require().Equal(1, count, "target table must exist exactly once on the target database") +} + +// TestCreateTargetTable_AbortsIfExists exercises #8207 AC #2: +// "A move table run aborts before any data is copied if the target table already exists." +// +// It pre-creates the target table, then calls CreateTargetTable and asserts it +// returns a descriptive error (not just MySQL's raw ERROR 1050). +func (suite *ApplierTestSuite) TestCreateTargetTable_AbortsIfExists() { + ctx := context.Background() + + _, err := suite.db.ExecContext(ctx, fmt.Sprintf("CREATE TABLE %s (id INT PRIMARY KEY);", getTestTableName())) + suite.Require().NoError(err) + + _, err = suite.otherDB.ExecContext(ctx, fmt.Sprintf("CREATE TABLE `%s`.`%s` (id INT PRIMARY KEY);", testMysqlDatabaseOther, testMysqlTableName)) + suite.Require().NoError(err) + + var count int + err = suite.otherDB.QueryRow( + "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema=? AND table_name=?", + testMysqlDatabaseOther, testMysqlTableName, + ).Scan(&count) + suite.Require().NoError(err) + suite.Require().Equal(1, count, "precondition: target table must exist before CreateTargetTable") + + connectionConfig, err := getTestConnectionConfig(ctx, suite.mysqlContainer) + suite.Require().NoError(err) + + migrationContext := newTestMigrationContext() + migrationContext.MoveTables.TableNames = []string{testMysqlTableName} + migrationContext.MoveTables.TargetDatabase = testMysqlDatabaseOther + migrationContext.ApplierConnectionConfig = connectionConfig + migrationContext.MoveTables.ConnectionConfig = connectionConfig + migrationContext.SetConnectionConfig("innodb") + migrationContext.OriginalTableColumns = sql.NewColumnList([]string{"id"}) + + applier := NewApplier(migrationContext) + defer applier.Teardown() + + err = applier.InitDBConnections() + suite.Require().NoError(err) + + var dummy, sourceCreateDDL string + err = suite.db.QueryRow(fmt.Sprintf("SHOW CREATE TABLE %s", getTestTableName())).Scan(&dummy, &sourceCreateDDL) + suite.Require().NoError(err) + + err = applier.CreateTargetTable(sourceCreateDDL) + suite.Require().Error(err, "CreateTargetTable must return an error when target table already exists") + suite.Require().Contains(err.Error(), "already exists", "error message must mention 'already exists'") + suite.Require().Contains(err.Error(), testMysqlTableName, "error message must name the table") +} + func (suite *ApplierTestSuite) TestPanicOnWarningsInApplyIterationInsertQuerySucceedsWithUniqueKeyWarningInsertedByDMLEvent() { ctx := context.Background() diff --git a/go/logic/migrator.go b/go/logic/migrator.go index b0a4d05e1..74c2483cd 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -963,7 +963,7 @@ func (mgtr *Migrator) moveTablesCutOver() (err error) { // unpostpone-socket gate because per coop_cutover.md §1.1 P4, operator-removes- // postpone is the trigger for the entire cutover phase. mgtr.migrationContext.Log.Debugf("checking for cut-over postpone") - if err := mgtr.sleepWhileTrue(func() (bool, error) { + if err := mgtr.sleepWhileTrue("cut_over_postpone", func() (bool, error) { if mgtr.migrationContext.PostponeCutOverFlagFile == "" { return false, nil } @@ -1027,7 +1027,7 @@ func (mgtr *Migrator) moveTablesCutOver() (err error) { if err := pinnedConn.QueryRowContext(context.Background(), "select @@gtid_executed").Scan(&drainGTIDStr); err != nil { return fmt.Errorf("drain GTID capture failed: %w", err) } - drainGTID, err := mysql.NewGTIDBinlogCoordinates(drainGTIDStr) + drainGTID, err := mysql.NewGTIDBinlogCoordinates(mysql.FlavorFor(mgtr.migrationContext.InspectorMySQLVersion), drainGTIDStr) if err != nil { return fmt.Errorf("drain GTID parse failed: %w", err) } diff --git a/go/logic/migrator_move_tables_cutover_test.go b/go/logic/migrator_move_tables_cutover_test.go index 22426be7d..b6f7aaa75 100644 --- a/go/logic/migrator_move_tables_cutover_test.go +++ b/go/logic/migrator_move_tables_cutover_test.go @@ -192,7 +192,7 @@ func (s *MoveTablesCutOverSuite) TearDownTest() { func (s *MoveTablesCutOverSuite) containingDrainGTID() *mysql.GTIDBinlogCoordinates { var serverUUID string s.Require().NoError(s.db.QueryRow("SELECT @@server_uuid").Scan(&serverUUID)) - g, err := mysql.NewGTIDBinlogCoordinates(fmt.Sprintf("%s:1-99999999", serverUUID)) + g, err := mysql.NewGTIDBinlogCoordinates(mysql.MySQLFlavor, fmt.Sprintf("%s:1-99999999", serverUUID)) s.Require().NoError(err) return g } From a30d90fc18da7a5ab058c10c36c70573ecf06a4c Mon Sep 17 00:00:00 2001 From: Daniel Joos Date: Mon, 15 Jun 2026 09:24:04 +0000 Subject: [PATCH 06/23] move-tables: throttle through target credentials Use target-cluster credentials and replica status for control-replica throttling during move-table migrations. Refs: #1709 --- go/cmd/gh-ost/main.go | 2 +- go/logic/applier.go | 6 +++- go/logic/applier_test.go | 56 ++++++++++++++++++++++++++++++++++++++ go/logic/migrator.go | 10 ++----- go/logic/throttler.go | 26 ++++++++++++++++-- go/logic/throttler_test.go | 35 ++++++++++++++++++++++++ go/mysql/utils.go | 26 ++++++++++++++++++ 7 files changed, 150 insertions(+), 11 deletions(-) diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index 3d520f2e9..5028f03c1 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -134,7 +134,7 @@ func main() { maxLagMillis := flag.Int64("max-lag-millis", 1500, "replication lag at which to throttle operation") replicationLagQuery := flag.String("replication-lag-query", "", "Deprecated. gh-ost uses an internal, subsecond resolution query") - throttleControlReplicas := flag.String("throttle-control-replicas", "", "List of replicas on which to check for lag; comma delimited. Example: myhost1.com:3306,myhost2.com,myhost3.com:3307") + throttleControlReplicas := flag.String("throttle-control-replicas", "", "List of replicas on which to check for lag; comma delimited. Example: myhost1.com:3306,myhost2.com,myhost3.com:3307. In move-tables mode, these replicas are expected to be in the target cluster. Specified target credentials will be used for the connection.") throttleQuery := flag.String("throttle-query", "", "when given, issued (every second) to check if operation should throttle. Expecting to return zero for no-throttle, >0 for throttle. Query is issued on the migrated server. Make sure this query is lightweight") throttleHTTP := flag.String("throttle-http", "", "when given, gh-ost checks given URL via HEAD request; any response code other than 200 (OK) causes throttling; make sure it has low latency response") flag.Int64Var(&migrationContext.ThrottleHTTPIntervalMillis, "throttle-http-interval-millis", 100, "Number of milliseconds to wait before triggering another HTTP throttle check") diff --git a/go/logic/applier.go b/go/logic/applier.go index fe9c9ad3b..1bc3bbc88 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -2009,8 +2009,12 @@ func (apl *Applier) AtomicCutoverRename(sessionIdChan chan int64, tablesRenamed } func (apl *Applier) ShowStatusVariable(variableName string) (result int64, err error) { + targetDB := apl.db + if apl.migrationContext.IsMoveTablesMode() { + targetDB = apl.moveTablesTargetDB + } query := fmt.Sprintf(`show /* gh-ost */ global status like '%s'`, variableName) - if err := apl.db.QueryRow(query).Scan(&variableName, &result); err != nil { + if err := targetDB.QueryRow(query).Scan(&variableName, &result); err != nil { return 0, err } return result, nil diff --git a/go/logic/applier_test.go b/go/logic/applier_test.go index 7cdc0fa6d..c22667514 100644 --- a/go/logic/applier_test.go +++ b/go/logic/applier_test.go @@ -2181,6 +2181,62 @@ func (suite *ApplierTestSuite) TestApplyIterationMoveTableCopyQueriesNoRows() { suite.Require().Equal(0, count) } +func (suite *ApplierTestSuite) TestShowStatusVariable() { + ctx := context.Background() + + _, err := suite.db.ExecContext(ctx, fmt.Sprintf("CREATE TABLE %s (id INT, item_id INT);", getTestTableName())) + suite.Require().NoError(err) + + connectionConfig, err := getTestConnectionConfig(ctx, suite.mysqlContainer) + suite.Require().NoError(err) + + migrationContext := newTestMigrationContext() + migrationContext.ApplierConnectionConfig = connectionConfig + migrationContext.SetConnectionConfig("innodb") + + applier := NewApplier(migrationContext) + defer applier.Teardown() + + suite.Require().NoError(applier.InitDBConnections()) + + // Uptime is always present in `SHOW GLOBAL STATUS` and is non-negative. + result, err := applier.ShowStatusVariable("Uptime") + suite.Require().NoError(err) + suite.Require().GreaterOrEqual(result, int64(0)) +} + +func (suite *ApplierTestSuite) TestShowStatusVariableMoveTablesMode() { + ctx := context.Background() + + _, err := suite.db.ExecContext(ctx, fmt.Sprintf("CREATE TABLE %s (id INT, item_id INT);", getTestTableName())) + suite.Require().NoError(err) + + connectionConfig, err := getTestConnectionConfig(ctx, suite.mysqlContainer) + suite.Require().NoError(err) + + migrationContext := newTestMigrationContext() + migrationContext.ApplierConnectionConfig = connectionConfig + migrationContext.MoveTables.ConnectionConfig = connectionConfig + migrationContext.SetConnectionConfig("innodb") + migrationContext.OriginalTableColumns = sql.NewColumnList([]string{"id", "item_id"}) + migrationContext.MoveTables.TableNames = []string{testMysqlTableName} + migrationContext.MoveTables.TargetDatabase = testMysqlDatabaseOther + + applier := NewApplier(migrationContext) + defer applier.Teardown() + + suite.Require().NoError(applier.InitDBConnections()) + + // In move-tables mode the status variable must be read from the + // move-tables target DB connection rather than the applier DB. + suite.Require().True(migrationContext.IsMoveTablesMode()) + suite.Require().NotNil(applier.moveTablesTargetDB) + + result, err := applier.ShowStatusVariable("Uptime") + suite.Require().NoError(err) + suite.Require().GreaterOrEqual(result, int64(0)) +} + func TestApplier(t *testing.T) { if testing.Short() { t.Skip("skipping applier test suite in short mode") diff --git a/go/logic/migrator.go b/go/logic/migrator.go index 74c2483cd..789859d41 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -1896,17 +1896,13 @@ func (mgtr *Migrator) addDMLEventsListener() error { // initiateThrottler kicks in the throttling collection and the throttling checks. func (mgtr *Migrator) initiateThrottler() { - if mgtr.migrationContext.IsMoveTablesMode() { - // TODO(chriskirkland): throttle against the target cluster - mgtr.migrationContext.Log.Info("Skipping throttling in move tables mode") - return - } - mgtr.throttler = NewThrottler(mgtr.migrationContext, mgtr.applier, mgtr.inspector, mgtr.appVersion) go mgtr.throttler.initiateThrottlerCollection(mgtr.firstThrottlingCollected) mgtr.migrationContext.Log.Infof("Waiting for first throttle metrics to be collected") - <-mgtr.firstThrottlingCollected // replication lag + if !mgtr.migrationContext.IsMoveTablesMode() { + <-mgtr.firstThrottlingCollected // replication lag + } <-mgtr.firstThrottlingCollected // HTTP status <-mgtr.firstThrottlingCollected // other, general metrics mgtr.migrationContext.Log.Infof("First throttle metrics collected") diff --git a/go/logic/throttler.go b/go/logic/throttler.go index ee6e3d132..42a93f574 100644 --- a/go/logic/throttler.go +++ b/go/logic/throttler.go @@ -184,6 +184,17 @@ func (thlr *Throttler) collectReplicationLag(firstThrottlingCollected chan<- boo } } +// controlReplicaConnectionConfig returns the connection config used to read +// replication lag from a control replica. In move-tables mode the lag is read +// from the target cluster's replicas, otherwise from the source (inspector) +// cluster's replicas. +func (thlr *Throttler) controlReplicaConnectionConfig(replicaKey mysql.InstanceKey) *mysql.ConnectionConfig { + if thlr.migrationContext.IsMoveTablesMode() { + return thlr.migrationContext.MoveTables.ConnectionConfig.DuplicateCredentials(replicaKey) + } + return thlr.migrationContext.InspectorConnectionConfig.DuplicateCredentials(replicaKey) +} + // collectControlReplicasLag polls all the control replicas to get maximum lag value func (thlr *Throttler) collectControlReplicasLag() { if atomic.LoadInt64(&thlr.migrationContext.HibernateUntil) > 0 { @@ -206,6 +217,14 @@ func (thlr *Throttler) collectControlReplicasLag() { return lag, err } + if thlr.migrationContext.IsMoveTablesMode() { + dbVersion, err := mysql.GetDBVersion(thlr.migrationContext.Uuid, dbUri) + if err != nil { + return lag, err + } + return mysql.GetReplicationLagFromSlaveStatus(dbVersion, db) + } + if err := db.QueryRow(replicationLagQuery).Scan(&heartbeatValue); err != nil { return lag, err } @@ -221,7 +240,8 @@ func (thlr *Throttler) collectControlReplicasLag() { } lagResults := make(chan *mysql.ReplicationLagResult, instanceKeyMap.Len()) for replicaKey := range *instanceKeyMap { - connectionConfig := thlr.migrationContext.InspectorConnectionConfig.DuplicateCredentials(replicaKey) + connectionConfig := thlr.controlReplicaConnectionConfig(replicaKey) + if err := connectionConfig.RegisterTLSConfig(); err != nil { return &mysql.ReplicationLagResult{Err: err} } @@ -451,7 +471,9 @@ func (thlr *Throttler) collectGeneralThrottleMetrics() error { // that may affect throttling. There are several components, all running independently, // that collect such metrics. func (thlr *Throttler) initiateThrottlerCollection(firstThrottlingCollected chan<- bool) { - go thlr.collectReplicationLag(firstThrottlingCollected) + if !thlr.migrationContext.IsMoveTablesMode() { + go thlr.collectReplicationLag(firstThrottlingCollected) + } go thlr.collectControlReplicasLag() go thlr.collectThrottleHTTPStatus(firstThrottlingCollected) diff --git a/go/logic/throttler_test.go b/go/logic/throttler_test.go index 01104805b..7a9b77e2e 100644 --- a/go/logic/throttler_test.go +++ b/go/logic/throttler_test.go @@ -14,6 +14,7 @@ import ( "github.com/stretchr/testify/require" "github.com/github/gh-ost/go/base" + "github.com/github/gh-ost/go/mysql" ) func newTestThrottler() *Throttler { @@ -87,6 +88,40 @@ func TestThrottleReturnsOnContextCancellation(t *testing.T) { } } +func TestControlReplicaConnectionConfig(t *testing.T) { + replicaKey := mysql.InstanceKey{Hostname: "replica-host", Port: 3307} + + t.Run("uses inspector connection config when not in move-tables mode", func(t *testing.T) { + thlr := newTestThrottler() + thlr.migrationContext.InspectorConnectionConfig.Key = mysql.InstanceKey{Hostname: "source-host", Port: 3306} + thlr.migrationContext.InspectorConnectionConfig.User = "source-user" + thlr.migrationContext.InspectorConnectionConfig.Password = "source-pass" + + connectionConfig := thlr.controlReplicaConnectionConfig(replicaKey) + + assert.False(t, thlr.migrationContext.IsMoveTablesMode()) + assert.Equal(t, replicaKey, connectionConfig.Key) + assert.Equal(t, "source-user", connectionConfig.User) + assert.Equal(t, "source-pass", connectionConfig.Password) + }) + + t.Run("uses move-tables connection config when in move-tables mode", func(t *testing.T) { + thlr := newTestThrottler() + thlr.migrationContext.MoveTables.TableNames = []string{"my_table"} + thlr.migrationContext.MoveTables.ConnectionConfig = mysql.NewConnectionConfig() + thlr.migrationContext.MoveTables.ConnectionConfig.Key = mysql.InstanceKey{Hostname: "target-host", Port: 3306} + thlr.migrationContext.MoveTables.ConnectionConfig.User = "target-user" + thlr.migrationContext.MoveTables.ConnectionConfig.Password = "target-pass" + + connectionConfig := thlr.controlReplicaConnectionConfig(replicaKey) + + assert.True(t, thlr.migrationContext.IsMoveTablesMode()) + assert.Equal(t, replicaKey, connectionConfig.Key) + assert.Equal(t, "target-user", connectionConfig.User) + assert.Equal(t, "target-pass", connectionConfig.Password) + }) +} + func TestThrottleCallsOnThrottledCallback(t *testing.T) { thlr := newTestThrottler() thlr.migrationContext.SetThrottled(true, "test", base.NoThrottleReasonHint) diff --git a/go/mysql/utils.go b/go/mysql/utils.go index ad3c09487..0169c4861 100644 --- a/go/mysql/utils.go +++ b/go/mysql/utils.go @@ -50,6 +50,8 @@ func (rlg *ReplicationLagResult) HasLag() bool { // knownDBs is a DB cache by uri var knownDBs map[string]*gosql.DB = make(map[string]*gosql.DB) var knownDBsMutex = &sync.Mutex{} +var knownDBsVersions map[string]string = make(map[string]string) +var knownDBsVersionsMutex = &sync.Mutex{} // initConnector wraps a driver.Connector to run a fixed set of statements on // every newly established connection (e.g. setting the transaction isolation @@ -123,6 +125,30 @@ func GetDB(migrationUuid string, mysql_uri string) (db *gosql.DB, exists bool, e return db, exists, nil } +// GetDBVersion returns the MySQL version for a given mysql_uri, and caches it for future calls. +// Uses GetDB to get a connection to the database. +func GetDBVersion(migrationUuid string, mysql_uri string) (dbVersion string, err error) { + cacheKey := migrationUuid + ":" + mysql_uri + + knownDBsVersionsMutex.Lock() + defer knownDBsVersionsMutex.Unlock() + + if dbVersion, exists := knownDBsVersions[cacheKey]; exists { + return dbVersion, nil + } + + db, _, err := GetDB(migrationUuid, mysql_uri) + if err != nil { + return "", err + } + var version string + if err := db.QueryRow(`select @@global.version`).Scan(&version); err != nil { + return "", err + } + knownDBsVersions[cacheKey] = version + return version, nil +} + // GetReplicationLagFromSlaveStatus returns replication lag for a given db; via SHOW SLAVE STATUS func GetReplicationLagFromSlaveStatus(dbVersion string, informationSchemaDb *gosql.DB) (replicationLag time.Duration, err error) { showReplicaStatusQuery := fmt.Sprintf("show %s", ReplicaTermFor(dbVersion, `slave status`)) From 71d4d63607938c58d71b877238a8a3f64f01c912 Mon Sep 17 00:00:00 2001 From: Daniel Joos Date: Tue, 16 Jun 2026 09:34:45 +0000 Subject: [PATCH 07/23] move-tables: expose hook environment variables Expose move-table source, target, and migration details to lifecycle hooks. Refs: #1711 --- doc/hooks.md | 7 ++- go/base/context.go | 14 +++++ go/logic/hooks.go | 11 +++- go/logic/hooks_test.go | 122 +++++++++++++++++++++++++++++++++++++++++ go/logic/migrator.go | 1 + 5 files changed, 152 insertions(+), 3 deletions(-) diff --git a/doc/hooks.md b/doc/hooks.md index 03ce26db0..d9d4ffa46 100644 --- a/doc/hooks.md +++ b/doc/hooks.md @@ -73,15 +73,20 @@ The following variables are available on all hooks: - `GH_OST_MIGRATED_HOST` - `GH_OST_INSPECTED_HOST` - `GH_OST_EXECUTING_HOST` +- `GH_OST_TARGET_HOST` - the target/applier hostname (in `--move-tables`, this is the target cluster host) - `GH_OST_HOOKS_HINT` - copy of `--hooks-hint` value - `GH_OST_HOOKS_HINT_OWNER` - copy of `--hooks-hint-owner` value - `GH_OST_HOOKS_HINT_TOKEN` - copy of `--hooks-hint-token` value - `GH_OST_DRY_RUN` - whether or not the `gh-ost` run is a dry run - `GH_OST_REVERT` - whether or not `gh-ost` is running in revert mode +- `GH_OST_MOVE_TABLES` - whether or not `gh-ost` is running in `--move-tables` mode +- `GH_OST_TARGET_DATABASE_NAME` - operation target database name (in `--move-tables`, this is the explicit target database) +- `GH_OST_TARGET_TABLE_NAME` - operation target table name (mode-dependent) -The following variable are available on particular hooks: +The following variables are available on particular hooks: - `GH_OST_INSTANT_DDL` is only available in `gh-ost-on-success`. The value is `true` if instant DDL was successful, and `false` if it was not. +- `GH_OST_DRAIN_GTID` is only available in `gh-ost-on-success` and only in `--move-tables` mode. It contains the source `@@gtid_executed` captured immediately after the source `RENAME TABLE` (cutover) and represents the drain target the applier waits for. - `GH_OST_COMMAND` is only available in `gh-ost-on-interactive-command` - `GH_OST_STATUS` is only available in `gh-ost-on-status` - `GH_OST_LAST_BATCH_COPY_ERROR` is only available in `gh-ost-on-batch-copy-retry` diff --git a/go/base/context.go b/go/base/context.go index 54e157885..9da775c8e 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -289,6 +289,8 @@ type MigrationContext struct { TargetPass string // Target password for the move. If not specified, it will default to the source password. TargetDatabase string // Target database name for the move. If not specified, it will default to the source database name. ConnectionConfig *mysql.ConnectionConfig + + DrainGTID mysql.BinlogCoordinates // Source @@gtid_executed captured immediately after the source RENAME TABLE; the applier drains until it reaches this coordinate (move-tables only). } Log Logger @@ -497,6 +499,18 @@ func (mctx *MigrationContext) GetInspectorHostname() string { return mctx.InspectorConnectionConfig.ImpliedKey.Hostname } +// GetTargetHostname is a safe access method to the target hostname. +// In move-tables mode, this is the hostname of the target database, +// otherwise it's the same as the applier hostname. +func (mctx *MigrationContext) GetTargetHostname() string { + if mctx.IsMoveTablesMode() && + mctx.MoveTables.ConnectionConfig != nil && + mctx.MoveTables.ConnectionConfig.ImpliedKey != nil { + return mctx.MoveTables.ConnectionConfig.ImpliedKey.Hostname + } + return mctx.GetApplierHostname() +} + // InspectorIsAlsoApplier is `true` when the both inspector and applier are the // same database instance. This would be true when running directly on master or when // testing on replica. diff --git a/go/logic/hooks.go b/go/logic/hooks.go index 1b36ede63..ceec4b6b9 100644 --- a/go/logic/hooks.go +++ b/go/logic/hooks.go @@ -233,6 +233,7 @@ func (he *HooksExecutor) applyEnvironmentVariables(extraVariables ...string) []s env = append(env, fmt.Sprintf("GH_OST_MIGRATED_HOST=%s", he.migrationContext.GetApplierHostname())) env = append(env, fmt.Sprintf("GH_OST_INSPECTED_HOST=%s", he.migrationContext.GetInspectorHostname())) env = append(env, fmt.Sprintf("GH_OST_EXECUTING_HOST=%s", he.migrationContext.Hostname)) + env = append(env, fmt.Sprintf("GH_OST_TARGET_HOST=%s", he.migrationContext.GetTargetHostname())) env = append(env, fmt.Sprintf("GH_OST_INSPECTED_LAG=%f", he.migrationContext.GetCurrentLagDuration().Seconds())) env = append(env, fmt.Sprintf("GH_OST_HEARTBEAT_LAG=%f", he.migrationContext.TimeSinceLastHeartbeatOnChangelog().Seconds())) env = append(env, fmt.Sprintf("GH_OST_PROGRESS=%f", he.migrationContext.GetProgressPct())) @@ -242,6 +243,9 @@ func (he *HooksExecutor) applyEnvironmentVariables(extraVariables ...string) []s env = append(env, fmt.Sprintf("GH_OST_HOOKS_HINT_TOKEN=%s", he.migrationContext.HooksHintToken)) env = append(env, fmt.Sprintf("GH_OST_DRY_RUN=%t", he.migrationContext.Noop)) env = append(env, fmt.Sprintf("GH_OST_REVERT=%t", he.migrationContext.Revert)) + env = append(env, fmt.Sprintf("GH_OST_MOVE_TABLES=%t", he.migrationContext.IsMoveTablesMode())) + env = append(env, fmt.Sprintf("GH_OST_TARGET_DATABASE_NAME=%s", he.migrationContext.GetTargetDatabaseName())) + env = append(env, fmt.Sprintf("GH_OST_TARGET_TABLE_NAME=%s", he.migrationContext.GetTargetTableName())) env = append(env, extraVariables...) return env @@ -320,8 +324,11 @@ func (he *HooksExecutor) OnInteractiveCommand(command string) error { } func (he *HooksExecutor) OnSuccess(instantDDL bool) error { - v := fmt.Sprintf("GH_OST_INSTANT_DDL=%t", instantDDL) - return he.executeHooks(onSuccess, v) + v := []string{fmt.Sprintf("GH_OST_INSTANT_DDL=%t", instantDDL)} + if he.migrationContext.IsMoveTablesMode() && he.migrationContext.MoveTables.DrainGTID != nil { + v = append(v, fmt.Sprintf("GH_OST_DRAIN_GTID=%s", he.migrationContext.MoveTables.DrainGTID.String())) + } + return he.executeHooks(onSuccess, v...) } func (he *HooksExecutor) OnFailure() error { diff --git a/go/logic/hooks_test.go b/go/logic/hooks_test.go index 0729df31f..2865ebac6 100644 --- a/go/logic/hooks_test.go +++ b/go/logic/hooks_test.go @@ -20,6 +20,7 @@ import ( "github.com/stretchr/testify/require" "github.com/github/gh-ost/go/base" + "github.com/github/gh-ost/go/mysql" ) type recordingHooks struct { @@ -242,9 +243,130 @@ func TestHooksExecutorExecuteHooks(t *testing.T) { require.Equal(t, migrationContext.OriginalTableName, split[1]) case "GH_OST_INSTANT_DDL": require.Equal(t, "false", split[1]) + case "GH_OST_MOVE_TABLES": + require.Equal(t, "false", split[1]) + case "GH_OST_TARGET_DATABASE_NAME": + require.Equal(t, migrationContext.DatabaseName, split[1]) + case "GH_OST_TARGET_TABLE_NAME": + require.Equal(t, fmt.Sprintf("_%s_gho", migrationContext.OriginalTableName), split[1]) + case "GH_OST_TARGET_HOST": + require.Equal(t, migrationContext.GetApplierHostname(), split[1]) case "TEST": require.Equal(t, t.Name(), split[1]) } } }) } + +func TestHooksExecutorMoveTablesEnvironmentVariables(t *testing.T) { + migrationContext := base.NewMigrationContext() + migrationContext.DatabaseName = "source_db" + migrationContext.OriginalTableName = "tablename" + migrationContext.MoveTables.TableNames = []string{"tablename"} + migrationContext.MoveTables.TargetDatabase = "target_db" + migrationContext.MoveTables.ConnectionConfig = mysql.NewConnectionConfig() + migrationContext.MoveTables.ConnectionConfig.Key.Hostname = "target.example.com" + migrationContext.MoveTables.ConnectionConfig.ImpliedKey = &migrationContext.MoveTables.ConnectionConfig.Key + + hooksExecutor := NewHooksExecutor(migrationContext) + + hooksPath, err := os.MkdirTemp("", "TestHooksExecutorMoveTablesEnvironmentVariables") + require.NoError(t, err) + defer os.RemoveAll(hooksPath) + migrationContext.HooksPath = hooksPath + + require.NoError(t, os.WriteFile( + filepath.Join(hooksPath, "success-hook"), + []byte("#!/bin/sh\nenv"), + 0o777, + )) + + var buf bytes.Buffer + hooksExecutor.writer = &buf + require.Nil(t, hooksExecutor.executeHooks("success-hook")) + + scanner := bufio.NewScanner(&buf) + for scanner.Scan() { + split := strings.SplitN(scanner.Text(), "=", 2) + switch split[0] { + case "GH_OST_MOVE_TABLES": + require.Equal(t, "true", split[1]) + case "GH_OST_TARGET_DATABASE_NAME": + require.Equal(t, "target_db", split[1]) + case "GH_OST_TARGET_TABLE_NAME": + require.Equal(t, "tablename", split[1]) + case "GH_OST_TARGET_HOST": + require.Equal(t, "target.example.com", split[1]) + } + } +} + +func TestHooksExecutorOnSuccessEnvironmentVariables(t *testing.T) { + writeOnSuccessHook := func(t *testing.T, hooksPath string) { + t.Helper() + require.NoError(t, os.WriteFile( + filepath.Join(hooksPath, onSuccess), + []byte("#!/bin/sh\nenv"), + 0o777, + )) + } + + envFromBuffer := func(buf *bytes.Buffer) map[string]string { + envMap := map[string]string{} + scanner := bufio.NewScanner(buf) + for scanner.Scan() { + split := strings.SplitN(scanner.Text(), "=", 2) + if len(split) == 2 { + envMap[split[0]] = split[1] + } + } + return envMap + } + + t.Run("move-tables-includes-drain-gtid", func(t *testing.T) { + migrationContext := base.NewMigrationContext() + migrationContext.DatabaseName = "source_db" + migrationContext.OriginalTableName = "tablename" + migrationContext.MoveTables.TableNames = []string{"tablename"} + migrationContext.MoveTables.DrainGTID = &mysql.FileBinlogCoordinates{LogFile: "mysql-bin.000001", LogPos: 12345} + + hooksExecutor := NewHooksExecutor(migrationContext) + + hooksPath, err := os.MkdirTemp("", "TestHooksExecutorOnSuccessEnvironmentVariables-move-tables") + require.NoError(t, err) + defer os.RemoveAll(hooksPath) + migrationContext.HooksPath = hooksPath + writeOnSuccessHook(t, hooksPath) + + var buf bytes.Buffer + hooksExecutor.writer = &buf + require.NoError(t, hooksExecutor.OnSuccess(true)) + + envMap := envFromBuffer(&buf) + require.Equal(t, "true", envMap["GH_OST_INSTANT_DDL"]) + require.Equal(t, migrationContext.MoveTables.DrainGTID.String(), envMap["GH_OST_DRAIN_GTID"]) + }) + + t.Run("non-move-tables-omits-drain-gtid", func(t *testing.T) { + migrationContext := base.NewMigrationContext() + migrationContext.DatabaseName = "source_db" + migrationContext.OriginalTableName = "tablename" + + hooksExecutor := NewHooksExecutor(migrationContext) + + hooksPath, err := os.MkdirTemp("", "TestHooksExecutorOnSuccessEnvironmentVariables-standard") + require.NoError(t, err) + defer os.RemoveAll(hooksPath) + migrationContext.HooksPath = hooksPath + writeOnSuccessHook(t, hooksPath) + + var buf bytes.Buffer + hooksExecutor.writer = &buf + require.NoError(t, hooksExecutor.OnSuccess(false)) + + envMap := envFromBuffer(&buf) + require.Equal(t, "false", envMap["GH_OST_INSTANT_DDL"]) + _, exists := envMap["GH_OST_DRAIN_GTID"] + require.False(t, exists) + }) +} diff --git a/go/logic/migrator.go b/go/logic/migrator.go index 789859d41..ea6bcf6ed 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -1090,6 +1090,7 @@ func (mgtr *Migrator) moveTablesCutOver() (err error) { // OnSuccess call that used to live in MoveTables() (after finalCleanup) has // been removed so the hook fires in the order coop_cutover.md §3.2 step 6 // requires (T5 between T4 and T6, BEFORE finalCleanup). + mgtr.migrationContext.MoveTables.DrainGTID = drainGTID if err := mgtr.hooksExecutor.OnSuccess(false); err != nil { return fmt.Errorf("on-success hook failed: %w", err) } From 7231d9e12d78d270010e8fe0319b07aff6fa0a7b Mon Sep 17 00:00:00 2001 From: Daniel Joos Date: Wed, 17 Jun 2026 15:07:20 +0200 Subject: [PATCH 08/23] move-tables: initialize migration configuration Initialize move-table configuration consistently before migration setup. Refs: #1712 --- go/base/context.go | 10 ++++++++-- go/cmd/gh-ost/main.go | 7 ------- go/logic/applier.go | 4 ++-- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/go/base/context.go b/go/base/context.go index 9da775c8e..a55ea2113 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -989,8 +989,14 @@ func (mctx *MigrationContext) ApplyCredentials() { Hostname: mctx.MoveTables.TargetHost, Port: mctx.MoveTables.TargetPort, }) - mctx.MoveTables.ConnectionConfig.User = mctx.MoveTables.TargetUser - mctx.MoveTables.ConnectionConfig.Password = mctx.MoveTables.TargetPass + if mctx.MoveTables.TargetUser != "" { + // Override + mctx.MoveTables.ConnectionConfig.User = mctx.MoveTables.TargetUser + } + if mctx.MoveTables.TargetPass != "" { + // Override + mctx.MoveTables.ConnectionConfig.Password = mctx.MoveTables.TargetPass + } } } diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index 5028f03c1..f0b96c5eb 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -384,13 +384,6 @@ func main() { // For now, we only support moving a single table at a time. log.Fatal("--move-tables currently supports only a single table") } - - if migrationContext.MoveTables.TargetUser == "" { - migrationContext.MoveTables.TargetUser = migrationContext.CliUser - } - if migrationContext.MoveTables.TargetPass == "" { - migrationContext.MoveTables.TargetPass = migrationContext.CliPassword - } if migrationContext.MoveTables.TargetDatabase == "" { migrationContext.MoveTables.TargetDatabase = migrationContext.DatabaseName } diff --git a/go/logic/applier.go b/go/logic/applier.go index 1bc3bbc88..6cb493a93 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -198,11 +198,11 @@ func buildMigrationLockName(db, table string) string { // preventing two gh-ost processes from migrating the same table concurrently // on the same MySQL server. func (apl *Applier) AcquireMigrationLock(ctx context.Context) error { - lockName := buildMigrationLockName(apl.migrationContext.DatabaseName, apl.originalTableName()) + lockName := buildMigrationLockName(apl.migrationContext.GetTargetDatabaseName(), apl.originalTableName()) // Use a dedicated *sql.DB so the pinned connection does not consume a // slot in apl.db's small pool (mysql.MaxDBPoolConnections). - lockURI := apl.connectionConfig.GetDBUri(apl.migrationContext.DatabaseName) + lockURI := apl.connectionConfig.GetDBUri(apl.migrationContext.GetTargetDatabaseName()) lockDB, err := mysql.OpenDB(lockURI) if err != nil { return fmt.Errorf("failed to open migration lock DB: %w", err) From 7c4c719680d753c794a881f434b539ebf2fce33e Mon Sep 17 00:00:00 2001 From: Daniel Joos Date: Thu, 18 Jun 2026 17:13:09 +0200 Subject: [PATCH 09/23] move-tables: enable throttling Enable the throttler in move-table mode with the move-table-specific configuration. Refs: #1713 --- go/logic/migrator.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/go/logic/migrator.go b/go/logic/migrator.go index ea6bcf6ed..9f79b6002 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -2266,11 +2266,7 @@ func (mgtr *Migrator) executeWriteFuncs() error { return nil } - if !mgtr.migrationContext.IsMoveTablesMode() { - // disable throttling in move-tables mode for now - // https://github.com/github/database-infrastructure/issues/8212 - mgtr.throttler.throttle(nil) - } + mgtr.throttler.throttle(nil) // We give higher priority to event processing, then secondary priority to // rowcopy From f5ce1e7f77171722cc49f71cbb76b7a805ef3a61 Mon Sep 17 00:00:00 2001 From: Zach Sierakowski Date: Thu, 18 Jun 2026 17:38:10 +0000 Subject: [PATCH 10/23] move-tables: add crash-safe resume Persist per-table checkpoints and drain GTIDs so copying and cutover can resume safely after interruption. Refs: #1708 --- go/cmd/gh-ost/main.go | 8 + go/logic/applier.go | 126 +++++- go/logic/applier_test.go | 148 +++++++ go/logic/checkpoint.go | 12 +- go/logic/migrator.go | 386 +++++++++++++++--- go/logic/migrator_move_tables_cutover_test.go | 147 ++++++- go/sql/builder.go | 67 ++- go/sql/builder_test.go | 29 +- script/move-tables/README.md | 29 +- script/move-tables/insert-source-primary-loop | 31 +- script/move-tables/teardown | 6 +- 11 files changed, 883 insertions(+), 106 deletions(-) diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index f0b96c5eb..ba285f54e 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -374,6 +374,14 @@ func main() { if migrationContext.PostponeCutOverFlagFile == "" { log.Fatal("--postpone-cut-over-flag-file must be specified when using --move-tables") } + if !migrationContext.Checkpoint { + log.Infof("--move-tables requires checkpointing; enabling --checkpoint") + migrationContext.Checkpoint = true + } + if !migrationContext.UseGTIDs { + log.Infof("--move-tables requires GTID coordinates for cutover drain checks; enabling --gtid") + migrationContext.UseGTIDs = true + } migrationContext.MoveTables.TableNames = strings.Split(*moveTables, ",") for i := range migrationContext.MoveTables.TableNames { migrationContext.MoveTables.TableNames[i] = strings.TrimSpace(migrationContext.MoveTables.TableNames[i]) diff --git a/go/logic/applier.go b/go/logic/applier.go index 6cb493a93..6b83a603d 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -111,6 +111,35 @@ func NewApplier(migrationContext *base.MigrationContext) *Applier { } } +func (apl *Applier) checkpointDB() *gosql.DB { + if apl.migrationContext.IsMoveTablesMode() && apl.moveTablesTargetDB != nil { + return apl.moveTablesTargetDB + } + return apl.db +} + +func (apl *Applier) checkpointDatabaseName() string { + if apl.migrationContext.IsMoveTablesMode() { + return apl.migrationContext.GetTargetDatabaseName() + } + return apl.migrationContext.DatabaseName +} + +func (apl *Applier) checkpointDrainGTIDString(chk *Checkpoint) string { + if chk == nil || chk.MoveTablesCutOverDrainGTID == nil || chk.MoveTablesCutOverDrainGTID.IsEmpty() { + return "" + } + return chk.MoveTablesCutOverDrainGTID.String() +} + +func (apl *Applier) checkpointRangeColumnNames() (minColumnNames []string, maxColumnNames []string) { + for _, col := range apl.migrationContext.UniqueKey.Columns.Columns() { + minColumnNames = append(minColumnNames, sql.TruncateColumnName(col.Name, sql.MaxColumnNameLength-4)+"_min") + maxColumnNames = append(maxColumnNames, sql.TruncateColumnName(col.Name, sql.MaxColumnNameLength-4)+"_max") + } + return minColumnNames, maxColumnNames +} + // compileMigrationKeyWarningRegex compiles a regex pattern that matches duplicate key warnings // for the migration's unique key. Duplicate warnings are formatted differently across MySQL versions, // hence the optional table name prefix. Metacharacters in table/index names are escaped to avoid @@ -359,9 +388,10 @@ func (apl *Applier) prepareQueries() (err error) { } if apl.migrationContext.Checkpoint { if apl.checkpointInsertQueryBuilder, err = sql.NewCheckpointQueryBuilder( - apl.migrationContext.DatabaseName, + apl.checkpointDatabaseName(), apl.migrationContext.GetCheckpointTableName(), &apl.migrationContext.UniqueKey.Columns, + apl.migrationContext.IsMoveTablesMode(), ); err != nil { return err } @@ -854,6 +884,12 @@ func (apl *Applier) CreateCheckpointTable() error { "`gh_ost_dml_applied` bigint", "`gh_ost_is_cutover` tinyint(1) DEFAULT '0'", } + if apl.migrationContext.IsMoveTablesMode() { + colDefs = append(colDefs, + "`gh_ost_move_tables_cutover_started` tinyint(1) DEFAULT '0'", + "`gh_ost_move_tables_drain_gtid` text charset ascii", + ) + } for _, col := range apl.migrationContext.UniqueKey.Columns.Columns() { if col.MySQLType == "" { return fmt.Errorf("column %s has no type information. applyColumnTypes must be called", sql.EscapeName(col.Name)) @@ -870,12 +906,12 @@ func (apl *Applier) CreateCheckpointTable() error { } query := fmt.Sprintf("create /* gh-ost */ table %s.%s (\n %s\n)", - sql.EscapeName(apl.migrationContext.DatabaseName), + sql.EscapeName(apl.checkpointDatabaseName()), sql.EscapeName(apl.migrationContext.GetCheckpointTableName()), strings.Join(colDefs, ",\n "), ) apl.migrationContext.Log.Infof("Created checkpoint table") - if _, err := sqlutils.ExecNoPrepare(apl.db, query); err != nil { + if _, err := sqlutils.ExecNoPrepare(apl.checkpointDB(), query); err != nil { return err } return nil @@ -884,14 +920,14 @@ func (apl *Applier) CreateCheckpointTable() error { // dropTable drops a given table on the applied host func (apl *Applier) dropTable(tableName string) error { query := fmt.Sprintf(`drop /* gh-ost */ table if exists %s.%s`, - sql.EscapeName(apl.migrationContext.DatabaseName), + sql.EscapeName(apl.checkpointDatabaseName()), sql.EscapeName(tableName), ) apl.migrationContext.Log.Infof("Dropping table %s.%s", - sql.EscapeName(apl.migrationContext.DatabaseName), + sql.EscapeName(apl.checkpointDatabaseName()), sql.EscapeName(tableName), ) - if _, err := sqlutils.ExecNoPrepare(apl.db, query); err != nil { + if _, err := sqlutils.ExecNoPrepare(apl.checkpointDB(), query); err != nil { return err } apl.migrationContext.Log.Infof("Table dropped") @@ -1056,8 +1092,11 @@ func (apl *Applier) WriteCheckpoint(chk *Checkpoint) (int64, error) { return insertId, err } args := sqlutils.Args(chk.LastTrxCoords.String(), chk.Iteration, chk.RowsCopied, chk.DMLApplied, chk.IsCutover) + if apl.migrationContext.IsMoveTablesMode() { + args = append(args, chk.MoveTablesCutOverStarted, apl.checkpointDrainGTIDString(chk)) + } args = append(args, uniqueKeyArgs...) - res, err := apl.db.Exec(query, args...) + res, err := apl.checkpointDB().Exec(query, args...) if err != nil { return insertId, err } @@ -1065,15 +1104,39 @@ func (apl *Applier) WriteCheckpoint(chk *Checkpoint) (int64, error) { } func (apl *Applier) ReadLastCheckpoint() (*Checkpoint, error) { - row := apl.db.QueryRow(fmt.Sprintf(`select /* gh-ost */ * from %s.%s order by gh_ost_chk_id desc limit 1`, sql.EscapeName(apl.migrationContext.DatabaseName), sql.EscapeName(apl.migrationContext.GetCheckpointTableName()))) + minColumnNames, maxColumnNames := apl.checkpointRangeColumnNames() + selectColumns := []string{ + "gh_ost_chk_id", + "gh_ost_chk_timestamp", + "gh_ost_chk_coords", + "gh_ost_chk_iteration", + "gh_ost_rows_copied", + "gh_ost_dml_applied", + "gh_ost_is_cutover", + } + if apl.migrationContext.IsMoveTablesMode() { + selectColumns = append(selectColumns, "gh_ost_move_tables_cutover_started", "gh_ost_move_tables_drain_gtid") + } + selectColumns = append(selectColumns, minColumnNames...) + selectColumns = append(selectColumns, maxColumnNames...) + + row := apl.checkpointDB().QueryRow(fmt.Sprintf( + `select /* gh-ost */ %s from %s.%s order by gh_ost_chk_id desc limit 1`, + strings.Join(selectColumns, ", "), + sql.EscapeName(apl.checkpointDatabaseName()), + sql.EscapeName(apl.migrationContext.GetCheckpointTableName()), + )) chk := &Checkpoint{ IterationRangeMin: sql.NewColumnValues(apl.migrationContext.UniqueKey.Columns.Len()), IterationRangeMax: sql.NewColumnValues(apl.migrationContext.UniqueKey.Columns.Len()), } - var coordStr string + var coordStr, drainGTIDStr string var timestamp int64 ptrs := []interface{}{&chk.Id, ×tamp, &coordStr, &chk.Iteration, &chk.RowsCopied, &chk.DMLApplied, &chk.IsCutover} + if apl.migrationContext.IsMoveTablesMode() { + ptrs = append(ptrs, &chk.MoveTablesCutOverStarted, &drainGTIDStr) + } ptrs = append(ptrs, chk.IterationRangeMin.ValuesPointers...) ptrs = append(ptrs, chk.IterationRangeMax.ValuesPointers...) err := row.Scan(ptrs...) @@ -1097,6 +1160,51 @@ func (apl *Applier) ReadLastCheckpoint() (*Checkpoint, error) { } chk.LastTrxCoords = fileCoords } + if apl.migrationContext.IsMoveTablesMode() && drainGTIDStr != "" { + drainGTID, err := mysql.NewGTIDBinlogCoordinates(drainGTIDStr) + if err != nil { + return nil, err + } + chk.MoveTablesCutOverDrainGTID = drainGTID + } + return chk, nil +} + +func (apl *Applier) ReadMoveTablesCutOverCheckpoint() (*Checkpoint, error) { + row := apl.checkpointDB().QueryRow(fmt.Sprintf(`select /* gh-ost */ gh_ost_chk_id, gh_ost_chk_timestamp, gh_ost_chk_coords, gh_ost_chk_iteration, gh_ost_rows_copied, gh_ost_dml_applied, gh_ost_is_cutover, gh_ost_move_tables_cutover_started, gh_ost_move_tables_drain_gtid from %s.%s where gh_ost_move_tables_cutover_started = 1 and gh_ost_move_tables_drain_gtid is not null and gh_ost_move_tables_drain_gtid != '' order by gh_ost_chk_id desc limit 1`, sql.EscapeName(apl.checkpointDatabaseName()), sql.EscapeName(apl.migrationContext.GetCheckpointTableName()))) + chk := &Checkpoint{} + var coordStr, drainGTIDStr string + var timestamp int64 + err := row.Scan(&chk.Id, ×tamp, &coordStr, &chk.Iteration, &chk.RowsCopied, &chk.DMLApplied, &chk.IsCutover, &chk.MoveTablesCutOverStarted, &drainGTIDStr) + if err != nil { + if errors.Is(err, gosql.ErrNoRows) { + return nil, ErrNoCheckpointFound + } + return nil, err + } + chk.Timestamp = time.Unix(timestamp, 0) + if coordStr != "" { + if apl.migrationContext.UseGTIDs { + coords, err := mysql.NewGTIDBinlogCoordinates(coordStr) + if err != nil { + return nil, err + } + chk.LastTrxCoords = coords + } else { + coords, err := mysql.ParseFileBinlogCoordinates(coordStr) + if err != nil { + return nil, err + } + chk.LastTrxCoords = coords + } + } + if drainGTIDStr != "" { + drainGTID, err := mysql.NewGTIDBinlogCoordinates(drainGTIDStr) + if err != nil { + return nil, err + } + chk.MoveTablesCutOverDrainGTID = drainGTID + } return chk, nil } diff --git a/go/logic/applier_test.go b/go/logic/applier_test.go index c22667514..1e1183692 100644 --- a/go/logic/applier_test.go +++ b/go/logic/applier_test.go @@ -1222,6 +1222,154 @@ func (suite *ApplierTestSuite) TestWriteCheckpoint() { suite.Require().Equal(chk.RowsCopied, gotChk.RowsCopied) suite.Require().Equal(chk.DMLApplied, gotChk.DMLApplied) suite.Require().Equal(chk.IsCutover, gotChk.IsCutover) + suite.Require().False(gotChk.MoveTablesCutOverStarted) + suite.Require().Nil(gotChk.MoveTablesCutOverDrainGTID) +} + +func (suite *ApplierTestSuite) TestWriteCheckpointMoveTables() { + ctx := context.Background() + + var err error + + _, err = suite.db.ExecContext(ctx, fmt.Sprintf("CREATE TABLE %s (id int not null, id2 char(4) CHARACTER SET utf8mb4, primary key(id, id2))", getTestTableName())) + suite.Require().NoError(err) + + _, err = suite.db.ExecContext(ctx, fmt.Sprintf("INSERT INTO %s (id, id2) VALUES (?,?), (?,?), (?,?)", getTestTableName()), 411, "君子懷德", 411, "小人懷土", 212, "君子不器") + suite.Require().NoError(err) + + connectionConfig, err := getTestConnectionConfig(ctx, suite.mysqlContainer) + suite.Require().NoError(err) + + migrationContext := newTestMigrationContext() + migrationContext.ApplierConnectionConfig = connectionConfig + migrationContext.InspectorConnectionConfig = connectionConfig + migrationContext.SetConnectionConfig("innodb") + migrationContext.UseGTIDs = true + + migrationContext.OriginalTableColumns = sql.NewColumnList([]string{"id", "id2"}) + migrationContext.SharedColumns = sql.NewColumnList([]string{"id", "id2"}) + migrationContext.MappedSharedColumns = sql.NewColumnList([]string{"id", "id2"}) + migrationContext.Checkpoint = true + migrationContext.MoveTables.TableNames = []string{testMysqlTableName} + migrationContext.MoveTables.TargetDatabase = testMysqlDatabase + migrationContext.MoveTables.ConnectionConfig = connectionConfig + migrationContext.UniqueKey = &sql.UniqueKey{ + Name: "PRIMARY", + NameInGhostTable: "PRIMARY", + Columns: *sql.NewColumnList([]string{"id", "id2"}), + } + + inspector := NewInspector(migrationContext) + suite.Require().NoError(inspector.InitDBConnections()) + + err = inspector.applyColumnTypes(testMysqlDatabase, testMysqlTableName, &migrationContext.UniqueKey.Columns) + suite.Require().NoError(err) + + applier := NewApplier(migrationContext) + + err = applier.InitDBConnections() + suite.Require().NoError(err) + + err = applier.CreateCheckpointTable() + suite.Require().NoError(err) + + err = applier.prepareQueries() + suite.Require().NoError(err) + + err = applier.ReadMigrationRangeValues(inspector.db) + suite.Require().NoError(err) + + coords, err := mysql.NewGTIDBinlogCoordinates("00000000-0000-0000-0000-000000000001:1-10") + suite.Require().NoError(err) + drainGTID, err := mysql.NewGTIDBinlogCoordinates("00000000-0000-0000-0000-000000000001:1-20") + suite.Require().NoError(err) + + chk := &Checkpoint{ + LastTrxCoords: coords, + IterationRangeMin: applier.migrationContext.MigrationRangeMinValues, + IterationRangeMax: applier.migrationContext.MigrationRangeMaxValues, + Iteration: 3, + RowsCopied: 1000, + DMLApplied: 2000, + IsCutover: false, + MoveTablesCutOverStarted: true, + MoveTablesCutOverDrainGTID: drainGTID, + } + id, err := applier.WriteCheckpoint(chk) + suite.Require().NoError(err) + suite.Require().Equal(int64(1), id) + + gotChk, err := applier.ReadLastCheckpoint() + suite.Require().NoError(err) + + suite.Require().Equal(chk.Iteration, gotChk.Iteration) + suite.Require().Equal(chk.LastTrxCoords.String(), gotChk.LastTrxCoords.String()) + suite.Require().Equal(chk.IterationRangeMin.String(), gotChk.IterationRangeMin.String()) + suite.Require().Equal(chk.IterationRangeMax.String(), gotChk.IterationRangeMax.String()) + suite.Require().Equal(chk.RowsCopied, gotChk.RowsCopied) + suite.Require().Equal(chk.DMLApplied, gotChk.DMLApplied) + suite.Require().Equal(chk.IsCutover, gotChk.IsCutover) + suite.Require().True(gotChk.MoveTablesCutOverStarted) + suite.Require().NotNil(gotChk.MoveTablesCutOverDrainGTID) + suite.Require().Equal(drainGTID.String(), gotChk.MoveTablesCutOverDrainGTID.String()) +} + +func (suite *ApplierTestSuite) TestReadMoveTablesCutOverCheckpointIgnoresRowCopyCheckpoints() { + ctx := context.Background() + + var err error + + _, err = suite.db.ExecContext(ctx, fmt.Sprintf("CREATE TABLE %s (id int not null primary key)", getTestTableName())) + suite.Require().NoError(err) + + _, err = suite.db.ExecContext(ctx, fmt.Sprintf("INSERT INTO %s (id) VALUES (1), (2), (3)", getTestTableName())) + suite.Require().NoError(err) + + connectionConfig, err := getTestConnectionConfig(ctx, suite.mysqlContainer) + suite.Require().NoError(err) + + migrationContext := newTestMigrationContext() + migrationContext.ApplierConnectionConfig = connectionConfig + migrationContext.InspectorConnectionConfig = connectionConfig + migrationContext.SetConnectionConfig("innodb") + migrationContext.Checkpoint = true + migrationContext.MoveTables.TableNames = []string{testMysqlTableName} + migrationContext.MoveTables.TargetDatabase = testMysqlDatabase + migrationContext.MoveTables.ConnectionConfig = connectionConfig + migrationContext.OriginalTableColumns = sql.NewColumnList([]string{"id"}) + migrationContext.SharedColumns = sql.NewColumnList([]string{"id"}) + migrationContext.MappedSharedColumns = sql.NewColumnList([]string{"id"}) + migrationContext.UniqueKey = &sql.UniqueKey{ + Name: "PRIMARY", + NameInGhostTable: "PRIMARY", + Columns: *sql.NewColumnList([]string{"id"}), + } + + inspector := NewInspector(migrationContext) + suite.Require().NoError(inspector.InitDBConnections()) + err = inspector.applyColumnTypes(testMysqlDatabase, testMysqlTableName, &migrationContext.UniqueKey.Columns) + suite.Require().NoError(err) + + applier := NewApplier(migrationContext) + suite.Require().NoError(applier.InitDBConnections()) + suite.Require().NoError(applier.CreateCheckpointTable()) + suite.Require().NoError(applier.prepareQueries()) + suite.Require().NoError(applier.ReadMigrationRangeValues(inspector.db)) + + coords := mysql.NewFileBinlogCoordinates("mysql-bin.000003", int64(1234)) + chk := &Checkpoint{ + LastTrxCoords: coords, + IterationRangeMin: applier.migrationContext.MigrationRangeMinValues, + IterationRangeMax: applier.migrationContext.MigrationRangeMaxValues, + Iteration: 1, + RowsCopied: 3, + DMLApplied: 0, + } + _, err = applier.WriteCheckpoint(chk) + suite.Require().NoError(err) + + _, err = applier.ReadMoveTablesCutOverCheckpoint() + suite.Require().ErrorIs(err, ErrNoCheckpointFound) } func (suite *ApplierTestSuite) TestPanicOnWarningsWithDuplicateKeyOnNonMigrationIndex() { diff --git a/go/logic/checkpoint.go b/go/logic/checkpoint.go index cffe08c4b..f81a2bb16 100644 --- a/go/logic/checkpoint.go +++ b/go/logic/checkpoint.go @@ -24,9 +24,11 @@ type Checkpoint struct { IterationRangeMin *sql.ColumnValues // IterationRangeMax is the max shared key value // for the chunk copier range. - IterationRangeMax *sql.ColumnValues - Iteration int64 - RowsCopied int64 - DMLApplied int64 - IsCutover bool + IterationRangeMax *sql.ColumnValues + Iteration int64 + RowsCopied int64 + DMLApplied int64 + IsCutover bool + MoveTablesCutOverStarted bool + MoveTablesCutOverDrainGTID mysql.BinlogCoordinates } diff --git a/go/logic/migrator.go b/go/logic/migrator.go index 9f79b6002..70f3c6815 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -102,8 +102,9 @@ type Migrator struct { rowCopyCompleteFlag int64 // copyRowsQueue should not be buffered; if buffered some non-damaging but // excessive work happens at the end of the iteration as new copy-jobs arrive before realizing the copy is complete - copyRowsQueue chan tableWriteFunc - applyEventsQueue chan *applyEventStruct + copyRowsQueue chan tableWriteFunc + applyEventsQueue chan *applyEventStruct + applyEventsInFlight int64 finishedMigrating int64 } @@ -812,6 +813,183 @@ func (mgtr *Migrator) prepareMoveTablesCopyState() { mgtr.migrationContext.MappedSharedColumns = mgtr.migrationContext.OriginalTableColumns } +func (mgtr *Migrator) hydrateMoveTablesStateFromTarget() error { + probeContext := base.NewMigrationContext() + probeContext.DatabaseName = mgtr.migrationContext.GetTargetDatabaseName() + targetInspector := &Inspector{db: mgtr.applier.moveTablesTargetDB, migrationContext: probeContext} + + columns, virtualColumns, uniqueKeys, err := targetInspector.InspectTableColumnsAndUniqueKeys(mgtr.migrationContext.GetTargetTableName()) + if err != nil { + return err + } + + mgtr.migrationContext.OriginalTableColumns = columns + mgtr.migrationContext.OriginalTableVirtualColumns = virtualColumns + mgtr.migrationContext.OriginalTableUniqueKeys = uniqueKeys + mgtr.migrationContext.UniqueKey = targetInspector.selectUniqueKey(uniqueKeys) + mgtr.migrationContext.SharedColumns = columns + mgtr.migrationContext.MappedSharedColumns = columns + return nil +} + +func (mgtr *Migrator) persistMoveTablesCutOverCheckpoint(drainGTID mysql.BinlogCoordinates, isCutover bool) error { + mgtr.applier.CurrentCoordinatesMutex.Lock() + safeCoords := mgtr.applier.CurrentCoordinates + mgtr.applier.CurrentCoordinatesMutex.Unlock() + + if safeCoords == nil || safeCoords.IsEmpty() { + // In move-tables mode CurrentCoordinates may never advance on a quiet source + // (no _ghc heartbeats, no DML). If there is no backlog, the streamer's + // frontier is a safe fallback for checkpointing. + if mgtr.eventsStreamer != nil && len(mgtr.applyEventsQueue) == 0 && len(mgtr.eventsStreamer.eventsChannel) == 0 { + safeCoords = mgtr.eventsStreamer.GetCurrentBinlogCoordinates() + } + if safeCoords == nil || safeCoords.IsEmpty() { + return errors.New("current coordinates are empty, cannot checkpoint move-tables cutover") + } + } + safeCoords = safeCoords.Clone() + + chk := &Checkpoint{ + LastTrxCoords: safeCoords, + IterationRangeMin: sql.NewColumnValues(mgtr.migrationContext.UniqueKey.Len()), + IterationRangeMax: sql.NewColumnValues(mgtr.migrationContext.UniqueKey.Len()), + Iteration: mgtr.migrationContext.GetIteration(), + RowsCopied: atomic.LoadInt64(&mgtr.migrationContext.TotalRowsCopied), + DMLApplied: atomic.LoadInt64(&mgtr.migrationContext.TotalDMLEventsApplied), + IsCutover: isCutover, + MoveTablesCutOverStarted: true, + MoveTablesCutOverDrainGTID: drainGTID, + } + mgtr.applier.LastIterationRangeMutex.Lock() + if mgtr.applier.LastIterationRangeMinValues != nil { + chk.IterationRangeMin = mgtr.applier.LastIterationRangeMinValues.Clone() + } + if mgtr.applier.LastIterationRangeMaxValues != nil { + chk.IterationRangeMax = mgtr.applier.LastIterationRangeMaxValues.Clone() + } + mgtr.applier.LastIterationRangeMutex.Unlock() + id, err := mgtr.applier.WriteCheckpoint(chk) + chk.Id = id + return err +} + +// moveTablesDrainCoordinateReached returns true when current is at-or-ahead of +// drain within the same coordinate family. For GTID drains, current must also +// be GTID-backed; mixed GTID/file-pos comparisons are treated as not reached. +func moveTablesDrainCoordinateReached(current mysql.BinlogCoordinates, drain mysql.BinlogCoordinates) bool { + if current == nil || current.IsEmpty() || drain == nil || drain.IsEmpty() { + return false + } + switch drain.(type) { + case *mysql.GTIDBinlogCoordinates: + if _, ok := current.(*mysql.GTIDBinlogCoordinates); !ok { + return false + } + } + return !current.SmallerThan(drain) +} + +// moveTablesDrainProvenByStreamerProgress is the non-DML-tail fallback used +// in T3: if both queues are empty and the streamer has advanced to drain, +// drain is considered complete even when applier coords did not move. +func moveTablesDrainProvenByStreamerProgress(drain mysql.BinlogCoordinates, streamer mysql.BinlogCoordinates, applyBacklog int, streamerBacklog int) bool { + if applyBacklog != 0 || streamerBacklog != 0 { + return false + } + return moveTablesDrainCoordinateReached(streamer, drain) +} + +func (mgtr *Migrator) drainMoveTablesCutOver(drainGTID mysql.BinlogCoordinates) error { + drainTimeout := time.Duration(mgtr.migrationContext.CutOverLockTimeoutSeconds) * time.Second + mgtr.migrationContext.Log.Infof("T3: draining applier to drain GTID (timeout %s, poll %s)", + drainTimeout, moveTablesCutOverDrainPollInterval) + drainCtx, cancel := context.WithTimeout(context.Background(), drainTimeout) + defer cancel() + ticker := time.NewTicker(moveTablesCutOverDrainPollInterval) + defer ticker.Stop() + for { + if err := mgtr.checkAbort(); err != nil { + return err + } + // Primary signal: applier's coordinate (advances when relevant apply work runs). + mgtr.applier.CurrentCoordinatesMutex.Lock() + applierCoords := mgtr.applier.CurrentCoordinates + mgtr.applier.CurrentCoordinatesMutex.Unlock() + drainReached := moveTablesDrainCoordinateReached(applierCoords, drainGTID) + // Backlogs gate completion: if either queue is non-empty, drain is not done. + applyBacklog := len(mgtr.applyEventsQueue) + streamerBacklog := 0 + var streamerCoords mysql.BinlogCoordinates + applierDisplay := "" + if applierCoords != nil { + applierDisplay = applierCoords.DisplayString() + } + if mgtr.eventsStreamer != nil { + streamerBacklog = len(mgtr.eventsStreamer.eventsChannel) + if mgtr.eventsStreamer.binlogReader != nil { + // Secondary signal: streamer's latest source position. + streamerCoords = mgtr.eventsStreamer.GetCurrentBinlogCoordinates() + } + } + applyInFlight := atomic.LoadInt64(&mgtr.applyEventsInFlight) + // Normal completion path: applier reached drain, both queues are empty, + // and no apply handler is still running. + if drainReached && applyBacklog == 0 && streamerBacklog == 0 && applyInFlight == 0 { + mgtr.migrationContext.Log.Infof("T3: drain complete; applier caught up to drain GTID") + return nil + } + // Fallback for non-DML tail: GTID can advance due to unrelated/non-row events, + // so applier may stop moving while streamer has already crossed drain. + if applyInFlight == 0 && moveTablesDrainProvenByStreamerProgress(drainGTID, streamerCoords, applyBacklog, streamerBacklog) { + mgtr.migrationContext.Log.Infof("T3: drain complete via streamer frontier (non-DML tail after T2)") + return nil + } + if drainReached { + mgtr.migrationContext.Log.Debugf("T3: drain GTID reached but backlog remains (apply=%d, streamer=%d, in_flight=%d)", applyBacklog, streamerBacklog, applyInFlight) + } else { + mgtr.migrationContext.Log.Debugf("T3: applier still behind drain GTID, polling (applier=%s drain=%s in_flight=%d)", applierDisplay, drainGTID.DisplayString(), applyInFlight) + } + select { + case <-drainCtx.Done(): + streamerDisplay := "" + if streamerCoords != nil { + streamerDisplay = streamerCoords.DisplayString() + } + return fmt.Errorf("drain poll timed out after %s: applier did not catch up to drain GTID (applier=%s drain=%s streamer=%s apply_backlog=%d streamer_backlog=%d in_flight=%d)", + drainTimeout, applierDisplay, drainGTID.DisplayString(), streamerDisplay, applyBacklog, streamerBacklog, applyInFlight) + case <-ticker.C: + } + } +} + +func (mgtr *Migrator) resumeMoveTablesCutOverFromCheckpoint(chk *Checkpoint) error { + if chk == nil || !chk.MoveTablesCutOverStarted || chk.MoveTablesCutOverDrainGTID == nil || chk.MoveTablesCutOverDrainGTID.IsEmpty() { + return errors.New("checkpoint does not contain move-tables cutover resume state") + } + if chk.LastTrxCoords != nil && !chk.LastTrxCoords.IsEmpty() { + mgtr.applier.CurrentCoordinatesMutex.Lock() + mgtr.applier.CurrentCoordinates = chk.LastTrxCoords.Clone() + mgtr.applier.CurrentCoordinatesMutex.Unlock() + } + mgtr.migrationContext.Log.Infof("Resuming move-tables cutover from checkpoint at coords=%+v drain_gtid=%s", + chk.LastTrxCoords, chk.MoveTablesCutOverDrainGTID.DisplayString()) + if err := mgtr.drainMoveTablesCutOver(chk.MoveTablesCutOverDrainGTID); err != nil { + return err + } + if mgtr.migrationContext.Checkpoint { + if err := mgtr.persistMoveTablesCutOverCheckpoint(chk.MoveTablesCutOverDrainGTID, true); err != nil { + mgtr.migrationContext.Log.Warningf("failed to checkpoint drained move-tables cutover: %+v", err) + } + } + atomic.StoreInt64(&mgtr.migrationContext.CutOverCompleteFlag, 1) + mgtr.migrationContext.Log.Debugf("T4: CutOverCompleteFlag set") + if err := mgtr.hooksExecutor.OnSuccess(false); err != nil { + return fmt.Errorf("on-success hook failed: %w", err) + } + return nil +} + func (mgtr *Migrator) MoveTables() (err error) { mgtr.migrationContext.Log.Infof("Moving tables %v from %s to %s (%s)", mgtr.migrationContext.MoveTables.TableNames, @@ -841,18 +1019,100 @@ func (mgtr *Migrator) MoveTables() (err error) { // so we don't leave things hanging around defer mgtr.teardown() + if mgtr.migrationContext.Checkpoint && mgtr.migrationContext.Resume { + mgtr.migrationContext.ApplierConnectionConfig = mgtr.migrationContext.MoveTables.ConnectionConfig + mgtr.applier = NewApplier(mgtr.migrationContext) + if err := mgtr.applier.InitDBConnections(); err != nil { + return err + } + cutoverResumeCheckpoint, err := mgtr.applier.ReadMoveTablesCutOverCheckpoint() + if err != nil && !errors.Is(err, ErrNoCheckpointFound) { + return err + } + if cutoverResumeCheckpoint != nil && cutoverResumeCheckpoint.MoveTablesCutOverStarted && cutoverResumeCheckpoint.MoveTablesCutOverDrainGTID != nil && !cutoverResumeCheckpoint.MoveTablesCutOverDrainGTID.IsEmpty() { + mgtr.migrationContext.InitialStreamerCoords = cutoverResumeCheckpoint.LastTrxCoords + mgtr.migrationContext.Iteration = cutoverResumeCheckpoint.Iteration + atomic.StoreInt64(&mgtr.migrationContext.TotalRowsCopied, cutoverResumeCheckpoint.RowsCopied) + atomic.StoreInt64(&mgtr.migrationContext.TotalDMLEventsApplied, cutoverResumeCheckpoint.DMLApplied) + if err := mgtr.hydrateMoveTablesStateFromTarget(); err != nil { + return fmt.Errorf("failed to hydrate move-tables resume state from target: %w", err) + } + if err := mgtr.createFlagFiles(); err != nil { + return err + } + if err := mgtr.initiateStreaming(); err != nil { + return err + } + if err := mgtr.applier.prepareQueries(); err != nil { + return err + } + if err := mgtr.hooksExecutor.OnValidated(); err != nil { + return err + } + if err := mgtr.initiateServer(); err != nil { + return err + } + defer mgtr.server.RemoveSocketFile() + if err := mgtr.addDMLEventsListener(); err != nil { + return err + } + mgtr.initiateThrottler() + go func() { + if err := mgtr.executeWriteFuncs(); err != nil { + _ = base.SendWithContext(mgtr.migrationContext.GetContext(), mgtr.migrationContext.PanicAbort, err) + } + }() + // Do not initiate status ticker in cutover resume path: inspector is not initialized, + // and we're only doing drain polling + hooks before exit (no row copy to monitor). + if err := mgtr.resumeMoveTablesCutOverFromCheckpoint(cutoverResumeCheckpoint); err != nil { + return err + } + if err := mgtr.finalCleanup(); err != nil { + return nil + } + mgtr.migrationContext.Log.Infof("Done moving tables %v from %s to %s (%s)", + mgtr.migrationContext.MoveTables.TableNames, sql.EscapeName(mgtr.migrationContext.DatabaseName), + sql.EscapeName(mgtr.migrationContext.GetTargetDatabaseName()), mgtr.migrationContext.MoveTables.TargetHost) + if err := mgtr.checkAbort(); err != nil { + return err + } + return nil + } + // Do not teardown this preflight applier on the miss path. Its DB handles + // come from the shared connection cache keyed by migration UUID, and + // closing them here would poison the later inspector/applier init path + // with "sql: database is closed". + mgtr.applier = nil + } + if err := mgtr.initiateInspector(); err != nil { return err } if err := mgtr.checkAbort(); err != nil { return err } + mgtr.prepareMoveTablesCopyState() if err := mgtr.initiateApplier(); err != nil { return err } if err := mgtr.checkAbort(); err != nil { return err } + if mgtr.migrationContext.Checkpoint && mgtr.migrationContext.Resume { + lastCheckpoint, err := mgtr.applier.ReadLastCheckpoint() + if err != nil { + return mgtr.migrationContext.Log.Errorf("no checkpoint found, unable to resume: %+v", err) + } + mgtr.migrationContext.Log.Infof("Resuming move-tables from checkpoint coords=%+v range_min=%+v range_max=%+v iteration=%d", + lastCheckpoint.LastTrxCoords, lastCheckpoint.IterationRangeMin.String(), lastCheckpoint.IterationRangeMax.String(), lastCheckpoint.Iteration) + + mgtr.migrationContext.MigrationIterationRangeMinValues = lastCheckpoint.IterationRangeMin + mgtr.migrationContext.MigrationIterationRangeMaxValues = lastCheckpoint.IterationRangeMax + mgtr.migrationContext.Iteration = lastCheckpoint.Iteration + atomic.StoreInt64(&mgtr.migrationContext.TotalRowsCopied, lastCheckpoint.RowsCopied) + atomic.StoreInt64(&mgtr.migrationContext.TotalDMLEventsApplied, lastCheckpoint.DMLApplied) + mgtr.migrationContext.InitialStreamerCoords = lastCheckpoint.LastTrxCoords + } if err := mgtr.createFlagFiles(); err != nil { return err } @@ -866,12 +1126,15 @@ func (mgtr *Migrator) MoveTables() (err error) { return err } - mgtr.prepareMoveTablesCopyState() - // this function assumes that the unique key constraint has been set. if err := mgtr.applier.prepareQueries(); err != nil { return err } + if mgtr.migrationContext.Checkpoint && !mgtr.migrationContext.Resume { + if err := mgtr.applier.CreateCheckpointTable(); err != nil { + mgtr.migrationContext.Log.Errorf("unable to create checkpoint table, see further error details") + } + } // Validation complete! Run on-validated hook. if err := mgtr.hooksExecutor.OnValidated(); err != nil { @@ -908,6 +1171,9 @@ func (mgtr *Migrator) MoveTables() (err error) { go mgtr.iterateChunks() mgtr.migrationContext.MarkRowCopyStartTime() go mgtr.initiateStatus() + if mgtr.migrationContext.Checkpoint { + go mgtr.checkpointLoop() + } mgtr.migrationContext.Log.Debugf("Operating until row copy is complete") mgtr.consumeRowCopyComplete() @@ -945,15 +1211,12 @@ func (mgtr *Migrator) MoveTables() (err error) { // atomicCutOver, waitForEventsUpToLock, heartbeat-lag) was built on a // single-server assumption that no longer holds when the applier writes target // and the streamer reads source. Each is replaced or dropped here. -// -// Crash safety (persisting the drain GTID before T3) is #8210. Enriched hook -// env vars (GH_OST_DRAIN_GTID, GH_OST_TARGET_*) are #8211. Target-side -// throttling is #8212. None of those are wired here. func (mgtr *Migrator) moveTablesCutOver() (err error) { if mgtr.migrationContext.Noop { mgtr.migrationContext.Log.Debugf("Noop operation; not really moving tables") return nil } + defer atomic.StoreInt64(&mgtr.migrationContext.InCutOverCriticalSectionFlag, 0) // ----- Postpone gate (precedes T0) ----- // Mirrors standard cutOver()'s sleepWhileTrue postpone structure but DROPS the @@ -987,6 +1250,9 @@ func (mgtr *Migrator) moveTablesCutOver() (err error) { atomic.StoreInt64(&mgtr.migrationContext.IsPostponingCutOver, 0) mgtr.migrationContext.Log.Debugf("checking for cut-over postpone: complete") + // Disables throttling and background checkpoint loop + atomic.StoreInt64(&mgtr.migrationContext.InCutOverCriticalSectionFlag, 1) + // ----- T0: on-before-cut-over hook ----- // Non-zero hook exit aborts cutover BEFORE any source DDL fires. if err := mgtr.hooksExecutor.OnBeforeCutOver(); err != nil { @@ -1003,7 +1269,8 @@ func (mgtr *Migrator) moveTablesCutOver() (err error) { // No retry on the RENAME: it is not idempotent — a partial success leaves // the table already renamed and a retry would fail. The operator re-runs // the whole hook chain on failure. - pinnedConn, err := mgtr.inspector.db.Conn(context.Background()) + cutOverCtx := mgtr.migrationContext.GetContext() + pinnedConn, err := mgtr.inspector.db.Conn(cutOverCtx) if err != nil { return fmt.Errorf("failed to pin connection for T1/T2: %w", err) } @@ -1016,7 +1283,7 @@ func (mgtr *Migrator) moveTablesCutOver() (err error) { sql.EscapeName(sourceDB), sql.EscapeName(sourceTable), sql.EscapeName(sourceDB), sql.EscapeName(delTable)) mgtr.migrationContext.Log.Infof("T1: renaming source table: %s", renameQuery) - if _, err := pinnedConn.ExecContext(context.Background(), renameQuery); err != nil { + if _, err := pinnedConn.ExecContext(cutOverCtx, renameQuery); err != nil { return fmt.Errorf("RENAME failed: %w", err) } @@ -1024,7 +1291,7 @@ func (mgtr *Migrator) moveTablesCutOver() (err error) { // @@GLOBAL scope is explicit so the intent is unambiguous in the SQL itself. // Design: https://github.com/github/gh-ost-tablemove-poc/blob/9dc6df75c4c88ff473906a497836c7518f5614ec/design/coop_cutover.md#32-correctness-verification-for-p4 var drainGTIDStr string - if err := pinnedConn.QueryRowContext(context.Background(), "select @@gtid_executed").Scan(&drainGTIDStr); err != nil { + if err := pinnedConn.QueryRowContext(cutOverCtx, "select @@global.gtid_executed").Scan(&drainGTIDStr); err != nil { return fmt.Errorf("drain GTID capture failed: %w", err) } drainGTID, err := mysql.NewGTIDBinlogCoordinates(mysql.FlavorFor(mgtr.migrationContext.InspectorMySQLVersion), drainGTIDStr) @@ -1032,46 +1299,18 @@ func (mgtr *Migrator) moveTablesCutOver() (err error) { return fmt.Errorf("drain GTID parse failed: %w", err) } mgtr.migrationContext.Log.Infof("T2: captured drain GTID: %s", drainGTID.DisplayString()) - - // ----- T3: drain poll ----- - // Wait until applier.CurrentCoordinates catches up to drainGTID. The drain - // is complete when the applier's coords are not strictly smaller than the - // drain target (i.e. the applier contains every GTID in drainGTID). Reads - // of CurrentCoordinates hold the mutex per applier.go:75. Per-iteration - // logging is Debug only to avoid spamming Info on a hot loop. - drainTimeout := time.Duration(mgtr.migrationContext.CutOverLockTimeoutSeconds) * time.Second - mgtr.migrationContext.Log.Infof("T3: draining applier to drain GTID (timeout %s, poll %s)", - drainTimeout, moveTablesCutOverDrainPollInterval) - drainCtx, cancel := context.WithTimeout(context.Background(), drainTimeout) - defer cancel() - ticker := time.NewTicker(moveTablesCutOverDrainPollInterval) - defer ticker.Stop() - for { - if err := mgtr.checkAbort(); err != nil { - return err - } - mgtr.applier.CurrentCoordinatesMutex.Lock() - applierCoords := mgtr.applier.CurrentCoordinates - mgtr.applier.CurrentCoordinatesMutex.Unlock() - applyBacklog := len(mgtr.applyEventsQueue) - streamerBacklog := 0 - if mgtr.eventsStreamer != nil { - streamerBacklog = len(mgtr.eventsStreamer.eventsChannel) - } - if applierCoords != nil && !applierCoords.IsEmpty() && !applierCoords.SmallerThan(drainGTID) && applyBacklog == 0 && streamerBacklog == 0 { - mgtr.migrationContext.Log.Infof("T3: drain complete; applier caught up to drain GTID") - break - } - if applierCoords != nil && !applierCoords.IsEmpty() && !applierCoords.SmallerThan(drainGTID) { - mgtr.migrationContext.Log.Debugf("T3: drain GTID reached but backlog remains (apply=%d, streamer=%d)", applyBacklog, streamerBacklog) - } else { - mgtr.migrationContext.Log.Debugf("T3: applier still behind drain GTID, polling") + if mgtr.migrationContext.Checkpoint { + if err := mgtr.persistMoveTablesCutOverCheckpoint(drainGTID, false); err != nil { + return fmt.Errorf("failed to persist move-tables cutover checkpoint: %w", err) } - select { - case <-drainCtx.Done(): - return fmt.Errorf("drain poll timed out after %s: applier did not catch up to drain GTID", drainTimeout) - case <-ticker.C: - // next iteration + } + + if err := mgtr.drainMoveTablesCutOver(drainGTID); err != nil { + return err + } + if mgtr.migrationContext.Checkpoint { + if err := mgtr.persistMoveTablesCutOverCheckpoint(drainGTID, true); err != nil { + mgtr.migrationContext.Log.Warningf("failed to checkpoint drained move-tables cutover: %+v", err) } } @@ -1920,13 +2159,20 @@ func (mgtr *Migrator) initiateApplier() error { } if mgtr.migrationContext.IsMoveTablesMode() { - createTableStatement, err := mgtr.inspector.showCreateTable(mgtr.migrationContext.MoveTables.TableNames[0]) - if err != nil { - return fmt.Errorf("failed to fetch create table statement: %w", err) - } - if err := mgtr.applier.CreateTargetTable(createTableStatement); err != nil { - mgtr.migrationContext.Log.Errorf("unable to create target table, see further error details. Perhaps a previous migration failed without dropping the table? Bailing out") - return err + if !mgtr.migrationContext.Resume { + createTableStatement, err := mgtr.inspector.showCreateTable(mgtr.migrationContext.MoveTables.TableNames[0]) + if err != nil { + return fmt.Errorf("failed to fetch create table statement: %w", err) + } + if err := mgtr.applier.CreateTargetTable(createTableStatement); err != nil { + mgtr.migrationContext.Log.Errorf("unable to create target table, see further error details. Perhaps a previous migration failed without dropping the table? Bailing out") + return err + } + } else { + mgtr.migrationContext.Log.Infof("Resuming move-tables; reusing existing target table %s.%s", + sql.EscapeName(mgtr.migrationContext.GetTargetDatabaseName()), + sql.EscapeName(mgtr.migrationContext.GetTargetTableName()), + ) } } else { if mgtr.migrationContext.Revert { @@ -2097,6 +2343,9 @@ func (mgtr *Migrator) iterateChunks() error { } func (mgtr *Migrator) onApplyEventStruct(eventStruct *applyEventStruct) error { + atomic.AddInt64(&mgtr.applyEventsInFlight, 1) + defer atomic.AddInt64(&mgtr.applyEventsInFlight, -1) + handleNonDMLEventStruct := func(eventStruct *applyEventStruct) error { if eventStruct.writeFunc != nil { if err := mgtr.retryOperation(*eventStruct.writeFunc); err != nil { @@ -2183,6 +2432,17 @@ func (mgtr *Migrator) Checkpoint(ctx context.Context) (*Checkpoint, error) { mgtr.applier.CurrentCoordinatesMutex.Unlock() return chk, err } + // In move-tables mode we do not emit heartbeat rows into _ghc, so + // CurrentCoordinates may not advance while the system is otherwise idle. + // If there is no backlog in either queue, it is safe to treat the current + // streamer coordinates as applied for checkpointing purposes. + if mgtr.migrationContext.IsMoveTablesMode() && len(mgtr.applyEventsQueue) == 0 && (mgtr.eventsStreamer == nil || len(mgtr.eventsStreamer.eventsChannel) == 0) { + mgtr.applier.CurrentCoordinates = coords.Clone() + id, err := mgtr.applier.WriteCheckpoint(chk) + chk.Id = id + mgtr.applier.CurrentCoordinatesMutex.Unlock() + return chk, err + } mgtr.applier.CurrentCoordinatesMutex.Unlock() sleepDuration := 500 * time.Millisecond metrics.RecordSleep(mgtr.migrationContext.Metrics, "replica_wait", sleepDuration) @@ -2359,9 +2619,17 @@ func (mgtr *Migrator) finalCleanup() error { } if mgtr.migrationContext.IsMoveTablesMode() { + if mgtr.migrationContext.Checkpoint { + if mgtr.migrationContext.OkToDropTable { + if err := mgtr.retryOperation(mgtr.applier.DropCheckpointTable); err != nil { + return err + } + } else if !mgtr.migrationContext.Noop { + mgtr.migrationContext.Log.Infof("Am not dropping checkpoint table without `--ok-to-drop-table`. To drop the checkpoint table, issue:") + mgtr.migrationContext.Log.Infof("-- drop table %s.%s", sql.EscapeName(mgtr.migrationContext.GetTargetDatabaseName()), sql.EscapeName(mgtr.migrationContext.GetCheckpointTableName())) + } + } // for move-tables mode, we're done at this point - // TODO(zacharysierakowski): when we add the checkpoint table in for 1.6, make sure we cleanup - // the checkpoint table here first before returning (looks like that's a few lines below changelog table cleanup) return nil } diff --git a/go/logic/migrator_move_tables_cutover_test.go b/go/logic/migrator_move_tables_cutover_test.go index b6f7aaa75..a62ba80f5 100644 --- a/go/logic/migrator_move_tables_cutover_test.go +++ b/go/logic/migrator_move_tables_cutover_test.go @@ -8,6 +8,7 @@ import ( "os" "path/filepath" "strings" + "sync" "sync/atomic" "testing" "time" @@ -82,6 +83,21 @@ func TestMoveTablesCutOver_OnBeforeCutOverHookAbortsBeforeRename(t *testing.T) { "post-state: only the failing T0 hook fires; no OnSuccess, no OnBeginPostponed") } +type onSuccessCheckHooks struct { + *recordingHooks + onSuccessCheck func() error +} + +func (h *onSuccessCheckHooks) OnSuccess(bool) error { + if err := h.record("OnSuccess"); err != nil { + return err + } + if h.onSuccessCheck != nil { + return h.onSuccessCheck() + } + return nil +} + // TestMoveTablesCutOver_PostponeGateFiresOnBeginPostponedOnce maps to the // postpone-gate decision in #8209-implement-protocol (keep OnBeginPostponed // firing logic with the same once-per-cutover semantics as standard cutOver). @@ -131,6 +147,43 @@ func TestMoveTablesCutOver_PostponeGateFiresOnBeginPostponedOnce(t *testing.T) { "post-state: hook failure must leave CutOverCompleteFlag unset") } +// TestResumeMoveTablesCutOverFromCheckpointAlreadyDrained verifies the crash- +// safe resume branch skips T1 entirely and proceeds directly to T5 when the +// persisted checkpoint already shows a drain-satisfied position. +func TestResumeMoveTablesCutOverFromCheckpointAlreadyDrained(t *testing.T) { + var calls []string + fakeHooks := &recordingHooks{name: "fake", calls: &calls} + + ctx := base.NewMigrationContext() + ctx.Hooks = fakeHooks + ctx.Checkpoint = false + + m := NewMigrator(ctx, "test") + m.applier = NewApplier(ctx) + + drainGTID, err := mysql.NewGTIDBinlogCoordinates("11111111-1111-1111-1111-111111111111:1-10") + require.NoError(t, err) + + chk := &Checkpoint{ + LastTrxCoords: drainGTID, + MoveTablesCutOverStarted: true, + MoveTablesCutOverDrainGTID: drainGTID, + } + + require.Equal(t, int64(0), atomic.LoadInt64(&ctx.CutOverCompleteFlag), "pre-state: flag must be 0") + require.Empty(t, calls, "pre-state: no hooks recorded") + + require.NoError(t, m.resumeMoveTablesCutOverFromCheckpoint(chk)) + + require.Equal(t, int64(1), atomic.LoadInt64(&ctx.CutOverCompleteFlag), + "post-state: resume path must set CutOverCompleteFlag before exiting") + require.Equal(t, []string{"fake:OnSuccess"}, calls, + "post-state: resume path should jump directly to T5 without rerunning T0/T1") + if m.applier.CurrentCoordinates != nil { + require.Equal(t, drainGTID.String(), m.applier.CurrentCoordinates.String()) + } +} + // ----------------------------------------------------------------------------- // Integration tests - real MySQL via testcontainers, exercise T1/T2/T3. // @@ -200,7 +253,7 @@ func (s *MoveTablesCutOverSuite) containingDrainGTID() *mysql.GTIDBinlogCoordina // buildMigrator wires a Migrator with the test container's *sql.DB pinned to // inspector.db and a fresh Applier. initialCoords may be nil for the drain- // timeout case. -func (s *MoveTablesCutOverSuite) buildMigrator(fakeHooks *recordingHooks, initialCoords mysql.BinlogCoordinates) (*Migrator, *base.MigrationContext) { +func (s *MoveTablesCutOverSuite) buildMigrator(fakeHooks base.Hooks, initialCoords mysql.BinlogCoordinates) (*Migrator, *base.MigrationContext) { ctx := context.Background() connectionConfig, err := getTestConnectionConfig(ctx, s.mysqlContainer) s.Require().NoError(err) @@ -358,6 +411,98 @@ func (s *MoveTablesCutOverSuite) TestDrainWaitsForQueuedDML() { "post-state: only T0 fires before the drain loop times out") } +// TestDrainWaitsForInFlightApplyEvent ensures T3 does not complete while an +// apply handler is still running. The blocked handler simulates the last source +// DML not yet landing on the target. If T3 exits early, OnSuccess observes the +// target missing that row and fails immediately. +func (s *MoveTablesCutOverSuite) TestDrainWaitsForInFlightApplyEvent() { + ctx := context.Background() + _, err := s.db.ExecContext(ctx, fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s", testMysqlDatabaseOther)) + s.Require().NoError(err) + _, err = s.db.ExecContext(ctx, fmt.Sprintf("CREATE TABLE %s (id INT PRIMARY KEY)", getTestTableName())) + s.Require().NoError(err) + _, err = s.db.ExecContext(ctx, fmt.Sprintf("CREATE TABLE %s (id INT PRIMARY KEY)", getTestOtherTableName())) + s.Require().NoError(err) + s.T().Cleanup(func() { + _, _ = s.db.ExecContext(context.Background(), "DROP TABLE IF EXISTS "+getTestOtherTableName()) + }) + + drainGTID := s.containingDrainGTID() + const pendingID = 42 + raceErr := errors.New("on-success observed target missing in-flight apply") + + origPoll := moveTablesCutOverDrainPollInterval + moveTablesCutOverDrainPollInterval = 50 * time.Millisecond + s.T().Cleanup(func() { + moveTablesCutOverDrainPollInterval = origPoll + }) + + var calls []string + fakeHooks := &onSuccessCheckHooks{ + recordingHooks: &recordingHooks{name: "fake", calls: &calls}, + onSuccessCheck: func() error { + var count int + query := fmt.Sprintf("SELECT COUNT(*) FROM %s WHERE id = ?", getTestOtherTableName()) + if err := s.db.QueryRowContext(context.Background(), query, pendingID).Scan(&count); err != nil { + return err + } + if count == 0 { + return raceErr + } + return nil + }, + } + m, mc := s.buildMigrator(fakeHooks, drainGTID) + mc.CutOverLockTimeoutSeconds = 1 + m.applier.CurrentCoordinatesMutex.Lock() + m.applier.CurrentCoordinates = drainGTID + m.applier.CurrentCoordinatesMutex.Unlock() + + started := make(chan struct{}) + release := make(chan struct{}) + var startedOnce sync.Once + var releaseOnce sync.Once + releaseApply := func() { + releaseOnce.Do(func() { + close(release) + }) + } + s.T().Cleanup(releaseApply) + blockApply := tableWriteFunc(func() error { + startedOnce.Do(func() { + close(started) + }) + <-release + _, err := s.db.ExecContext(context.Background(), fmt.Sprintf("INSERT INTO %s VALUES (?)", getTestOtherTableName()), pendingID) + return err + }) + go func() { + <-started + time.Sleep(200 * time.Millisecond) + releaseApply() + }() + go func() { + _ = m.onApplyEventStruct(newApplyEventStructByFunc(&blockApply)) + }() + + select { + case <-started: + case <-time.After(time.Second): + s.Require().FailNow("blocked apply event never started") + } + + s.Require().Equal(int64(0), atomic.LoadInt64(&mc.CutOverCompleteFlag), "pre-state: flag must be 0") + err = m.moveTablesCutOver() + s.Require().NoError(err) + + var count int + query := fmt.Sprintf("SELECT COUNT(*) FROM %s WHERE id = ?", getTestOtherTableName()) + s.Require().NoError(s.db.QueryRowContext(context.Background(), query, pendingID).Scan(&count)) + s.Require().Equal(1, count, "post-state: target must contain the pending row by the time cutover succeeds") + s.Require().Equal([]string{"fake:OnBeforeCutOver", "fake:OnSuccess"}, calls, + "post-state: cutover should only reach OnSuccess after the target row is present") +} + func TestMoveTablesCutOver(t *testing.T) { if testing.Short() { t.Skip("skipping integration suite in short mode") diff --git a/go/sql/builder.go b/go/sql/builder.go index 1c3c612fa..3f4c375ef 100644 --- a/go/sql/builder.go +++ b/go/sql/builder.go @@ -120,11 +120,12 @@ func BuildEqualsPreparedComparison(columns []string) (result string, err error) // It holds the prepared query statement so it doesn't need to be recreated every time. type CheckpointInsertQueryBuilder struct { - uniqueKeyColumns *ColumnList - preparedStatement string + uniqueKeyColumns *ColumnList + preparedStatement string + includeMoveTablesCutOverColumns bool } -func NewCheckpointQueryBuilder(databaseName, tableName string, uniqueKeyColumns *ColumnList) (*CheckpointInsertQueryBuilder, error) { +func NewCheckpointQueryBuilder(databaseName, tableName string, uniqueKeyColumns *ColumnList, includeMoveTablesCutOverColumns bool) (*CheckpointInsertQueryBuilder, error) { if uniqueKeyColumns.Len() == 0 { return nil, fmt.Errorf("got 0 columns in BuildSetCheckpointInsertQuery") } @@ -139,26 +140,48 @@ func NewCheckpointQueryBuilder(databaseName, tableName string, uniqueKeyColumns } databaseName = EscapeName(databaseName) tableName = EscapeName(tableName) - stmt := fmt.Sprintf(` - insert /* gh-ost */ - into %s.%s - (gh_ost_chk_timestamp, gh_ost_chk_coords, gh_ost_chk_iteration, - gh_ost_rows_copied, gh_ost_dml_applied, gh_ost_is_cutover, - %s, %s) - values - (unix_timestamp(now()), ?, ?, - ?, ?, ?, - %s, %s)`, - databaseName, tableName, - strings.Join(minUniqueColNames, ", "), - strings.Join(maxUniqueColNames, ", "), - strings.Join(values, ", "), - strings.Join(values, ", "), - ) - b := &CheckpointInsertQueryBuilder{ - uniqueKeyColumns: uniqueKeyColumns, - preparedStatement: stmt, + uniqueKeyColumns: uniqueKeyColumns, + preparedStatement: func() string { + if includeMoveTablesCutOverColumns { + return fmt.Sprintf(` + insert /* gh-ost */ + into %s.%s + (gh_ost_chk_timestamp, gh_ost_chk_coords, gh_ost_chk_iteration, + gh_ost_rows_copied, gh_ost_dml_applied, gh_ost_is_cutover, + gh_ost_move_tables_cutover_started, gh_ost_move_tables_drain_gtid, + %s, %s) + values + (unix_timestamp(now()), ?, ?, + ?, ?, ?, + ?, ?, + %s, %s)`, + databaseName, tableName, + strings.Join(minUniqueColNames, ", "), + strings.Join(maxUniqueColNames, ", "), + strings.Join(values, ", "), + strings.Join(values, ", "), + ) + } + + return fmt.Sprintf(` + insert /* gh-ost */ + into %s.%s + (gh_ost_chk_timestamp, gh_ost_chk_coords, gh_ost_chk_iteration, + gh_ost_rows_copied, gh_ost_dml_applied, gh_ost_is_cutover, + %s, %s) + values + (unix_timestamp(now()), ?, ?, + ?, ?, ?, + %s, %s)`, + databaseName, tableName, + strings.Join(minUniqueColNames, ", "), + strings.Join(maxUniqueColNames, ", "), + strings.Join(values, ", "), + strings.Join(values, ", "), + ) + }(), + includeMoveTablesCutOverColumns: includeMoveTablesCutOverColumns, } return b, nil } diff --git a/go/sql/builder_test.go b/go/sql/builder_test.go index 0fcf31441..38b5043ab 100644 --- a/go/sql/builder_test.go +++ b/go/sql/builder_test.go @@ -1347,7 +1347,7 @@ func TestCheckpointQueryBuilder(t *testing.T) { tableName := "_tbl_ghk" valueArgs := []interface{}{"mona", "mascot", int8(-17), "anothername", "anotherposition", int8(-2)} uniqueKeyColumns := NewColumnList([]string{"name", "position", "my_very_long_column_that_is_64_utf8_characters_long_很长很长很长很长很长很长"}) - builder, err := NewCheckpointQueryBuilder(databaseName, tableName, uniqueKeyColumns) + builder, err := NewCheckpointQueryBuilder(databaseName, tableName, uniqueKeyColumns, false) require.NoError(t, err) query, uniqueKeyArgs, err := builder.BuildQuery(valueArgs) require.NoError(t, err) @@ -1366,3 +1366,30 @@ func TestCheckpointQueryBuilder(t *testing.T) { require.Equal(t, normalizeQuery(expected), normalizeQuery(query)) require.Equal(t, []interface{}{"mona", "mascot", int8(-17), "anothername", "anotherposition", int8(-2)}, uniqueKeyArgs) } + +func TestMoveTablesCheckpointQueryBuilder(t *testing.T) { + databaseName := "mydb" + tableName := "_tbl_ghk" + valueArgs := []interface{}{"mona", "mascot", int8(-17), "anothername", "anotherposition", int8(-2)} + uniqueKeyColumns := NewColumnList([]string{"name", "position", "my_very_long_column_that_is_64_utf8_characters_long_很长很长很长很长很长很长"}) + builder, err := NewCheckpointQueryBuilder(databaseName, tableName, uniqueKeyColumns, true) + require.NoError(t, err) + query, uniqueKeyArgs, err := builder.BuildQuery(valueArgs) + require.NoError(t, err) + expected := ` + insert /* gh-ost */ into mydb._tbl_ghk + (gh_ost_chk_timestamp, gh_ost_chk_coords, gh_ost_chk_iteration, + gh_ost_rows_copied, gh_ost_dml_applied, gh_ost_is_cutover, + gh_ost_move_tables_cutover_started, gh_ost_move_tables_drain_gtid, + name_min, position_min, my_very_long_column_that_is_64_utf8_characters_long_很长很长很长很长_min, + name_max, position_max, my_very_long_column_that_is_64_utf8_characters_long_很长很长很长很长_max) + values + (unix_timestamp(now()), ?, ?, + ?, ?, ?, + ?, ?, + ?, ?, ?, + ?, ?, ?) + ` + require.Equal(t, normalizeQuery(expected), normalizeQuery(query)) + require.Equal(t, []interface{}{"mona", "mascot", int8(-17), "anothername", "anotherposition", int8(-2)}, uniqueKeyArgs) +} diff --git a/script/move-tables/README.md b/script/move-tables/README.md index db57f750a..398ec7517 100644 --- a/script/move-tables/README.md +++ b/script/move-tables/README.md @@ -24,10 +24,37 @@ script/build --cli Run gh-ost to move tables: ```bash -./script/build --cli; ./bin/gh-ost --move-tables=gh_ost_test --host=localhost --port=3307 --user root --password opensesame --database=gh_ost_test_db --target-host=localhost --target-port=3309 --target-user root --target-password opensesame --target-database=gh_ost_test_db --postpone-cut-over-flag-file=/tmp/ghost-move-tables.postpone.flag --execute --verbose +./script/build --cli; ./bin/gh-ost --move-tables=gh_ost_test --host=localhost --port=3307 --user root --password opensesame --database=gh_ost_test_db --target-host=localhost --target-port=3309 --target-user root --target-password opensesame --target-database=gh_ost_test_db --postpone-cut-over-flag-file=/tmp/ghost-move-tables.postpone.flag --execute --verbose --checkpoint --checkpoint-seconds 10 ``` Note: replicas in this local topology are configured with `read_only=ON` and `super_read_only=ON`. If you point `--host` at `mysql-source-replica` (3308), the cutover `RENAME TABLE` step will fail by design. Use source primary (3307) as the inspected host when you want cutover to rename on source. + +Start continuous inserts against the source. +```bash +script/move-tables/insert-source-primary-loop +``` + +Check the target - it should have the initial data from the source and should be receiving the new data. +```bash +script/move-tables/mysql-target-primary -D gh_ost_test_db -e "SELECT * FROM gh_ost_test;" +``` + +Remove the cutover flag file. +```bash +rm /tmp/ghost-move-tables.postpone.flag +``` + +You'll see the continuous inserts will stop because of the table rename. + +Check the source - table has been renamed. +```bash +script/move-tables/mysql-source-primary -D gh_ost_test_db -e "SELECT * FROM _gh_ost_test_del;" +``` + +Check the target has the same set of data. +```bash +script/move-tables/mysql-target-primary -D gh_ost_test_db -e "SELECT * FROM gh_ost_test;" +``` \ No newline at end of file diff --git a/script/move-tables/insert-source-primary-loop b/script/move-tables/insert-source-primary-loop index c5571ff1c..d2a0afddc 100755 --- a/script/move-tables/insert-source-primary-loop +++ b/script/move-tables/insert-source-primary-loop @@ -3,28 +3,45 @@ set -euo pipefail # Continuously insert new rows into gh_ost_test on source primary. # Usage: -# script/move-tables/insert-source-primary-loop [start_column1] [sleep_seconds] +# script/move-tables/insert-source-primary-loop [start_column1] [sleep_seconds] [rows_per_batch] # Example: -# script/move-tables/insert-source-primary-loop 100000 0.2 +# script/move-tables/insert-source-primary-loop 100000 0.2 1 +# Fast example: +# script/move-tables/insert-source-primary-loop 100000 0 50 start_i="${1:-100000}" delay="${2:-0.2}" +rows_per_batch="${3:-1}" i="$start_i" echo "Starting continuous inserts on source primary. Press Ctrl+C to stop." -echo "start_column1=$start_i sleep_seconds=$delay" +echo "start_column1=$start_i sleep_seconds=$delay rows_per_batch=$rows_per_batch" trap 'echo; echo "Stopped."; exit 0' INT TERM while true; do ts="$(date +%s)" + values="" + batch_start="$i" + + for ((n=0; n/dev/null || true echo "Removing containers..." -docker rm -f mysql-source-replica mysql-source-primary mysql-target-replica mysql-target-primary 2>/dev/null || true \ No newline at end of file +docker rm -f mysql-source-replica mysql-source-primary mysql-target-replica mysql-target-primary 2>/dev/null || true + +echo "Cleaning up Docker resources..." +docker system prune -f +docker volume prune -f \ No newline at end of file From 4ad73148520ab4c93ee363c6fd6cb14d5682407d Mon Sep 17 00:00:00 2001 From: Zach Sierakowski Date: Fri, 19 Jun 2026 17:19:01 +0000 Subject: [PATCH 11/23] move-tables: clean up migration artifacts Remove target artifacts and migration state after successful or failed move-table operations. Refs: #1717 --- go/base/context.go | 1 + go/logic/applier.go | 9 -- go/logic/migrator.go | 120 ++++++++++++++---- go/logic/migrator_move_tables_cleanup_test.go | 71 +++++++++++ go/logic/streamer.go | 27 ++++ 5 files changed, 197 insertions(+), 31 deletions(-) create mode 100644 go/logic/migrator_move_tables_cleanup_test.go diff --git a/go/base/context.go b/go/base/context.go index a55ea2113..33c0b4794 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -226,6 +226,7 @@ type MigrationContext struct { UserCommandedUnpostponeFlag int64 CutOverCompleteFlag int64 InCutOverCriticalSectionFlag int64 + MoveTablesSourceRenamedFlag int64 PanicAbort chan error // Context for cancellation signaling across all goroutines diff --git a/go/logic/applier.go b/go/logic/applier.go index 6b83a603d..94c437cb6 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -199,15 +199,6 @@ func (apl *Applier) InitDBConnections() (err error) { return err } } - if apl.moveTablesConnectionConfig != nil { - moveTablesURI := apl.moveTablesConnectionConfig.GetDBUri(apl.migrationContext.GetTargetDatabaseName()) + "&multiStatements=true" - if apl.moveTablesTargetDB, _, err = mysql.GetDB(apl.migrationContext.Uuid, moveTablesURI); err != nil { - return err - } - if _, err := base.ValidateConnection(apl.moveTablesTargetDB, apl.moveTablesConnectionConfig, apl.migrationContext, apl.name); err != nil { - return err - } - } apl.migrationContext.Log.Infof("Applier initiated on %+v, version %+v", apl.connectionConfig.ImpliedKey, apl.migrationContext.ApplierMySQLVersion) return nil } diff --git a/go/logic/migrator.go b/go/logic/migrator.go index 70f3c6815..73a55df64 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -967,6 +967,10 @@ func (mgtr *Migrator) resumeMoveTablesCutOverFromCheckpoint(chk *Checkpoint) err if chk == nil || !chk.MoveTablesCutOverStarted || chk.MoveTablesCutOverDrainGTID == nil || chk.MoveTablesCutOverDrainGTID.IsEmpty() { return errors.New("checkpoint does not contain move-tables cutover resume state") } + // The checkpoint proves the source RENAME already happened in a prior run, so + // `__del` exists on the source. Mark it so a failed resume emits the rollback + // hint. + atomic.StoreInt64(&mgtr.migrationContext.MoveTablesSourceRenamedFlag, 1) if chk.LastTrxCoords != nil && !chk.LastTrxCoords.IsEmpty() { mgtr.applier.CurrentCoordinatesMutex.Lock() mgtr.applier.CurrentCoordinates = chk.LastTrxCoords.Clone() @@ -1019,6 +1023,17 @@ func (mgtr *Migrator) MoveTables() (err error) { // so we don't leave things hanging around defer mgtr.teardown() + // If the run fails after the source RENAME, the source `__del` table is the + // rollback handle. Emit a clear rollback hint on any error return once the + // rename has happened. finalCleanup errors are + // swallowed below (they return nil), so this never fires on a successful + // cutover whose only failure was post-success cleanup. + defer func() { + if err != nil && atomic.LoadInt64(&mgtr.migrationContext.MoveTablesSourceRenamedFlag) > 0 { + mgtr.logMoveTablesRollbackHint() + } + }() + if mgtr.migrationContext.Checkpoint && mgtr.migrationContext.Resume { mgtr.migrationContext.ApplierConnectionConfig = mgtr.migrationContext.MoveTables.ConnectionConfig mgtr.applier = NewApplier(mgtr.migrationContext) @@ -1286,6 +1301,10 @@ func (mgtr *Migrator) moveTablesCutOver() (err error) { if _, err := pinnedConn.ExecContext(cutOverCtx, renameQuery); err != nil { return fmt.Errorf("RENAME failed: %w", err) } + // The source `__del` table now exists and is the rollback handle. Mark the + // rename as done so any later failure emits the rollback hint (and never + // drops `__del`). + atomic.StoreInt64(&mgtr.migrationContext.MoveTablesSourceRenamedFlag, 1) // ----- T2: capture @@gtid_executed on the SAME connection as T1 ----- // @@GLOBAL scope is explicit so the intent is unambiguous in the SQL itself. @@ -2601,36 +2620,28 @@ func (mgtr *Migrator) executeDMLWriteFuncs() error { func (mgtr *Migrator) finalCleanup() error { atomic.StoreInt64(&mgtr.migrationContext.CleanupImminentFlag, 1) - mgtr.migrationContext.Log.Infof("Writing changelog state: %+v", Migrated) - if _, err := mgtr.applier.WriteChangelogState(string(Migrated)); err != nil { - return err - } + if !mgtr.migrationContext.IsMoveTablesMode() { + mgtr.migrationContext.Log.Infof("Writing changelog state: %+v", Migrated) + if _, err := mgtr.applier.WriteChangelogState(string(Migrated)); err != nil { + return err + } - if mgtr.migrationContext.Noop { - if createTableStatement, err := mgtr.inspector.showCreateTable(mgtr.migrationContext.GetGhostTableName()); err == nil { - mgtr.migrationContext.Log.Infof("New table structure follows") - fmt.Println(createTableStatement) - } else if !mgtr.migrationContext.IsMoveTablesMode() { - mgtr.migrationContext.Log.Errore(fmt.Errorf("error showing create table: %w", err)) + if mgtr.migrationContext.Noop { + if createTableStatement, err := mgtr.inspector.showCreateTable(mgtr.migrationContext.GetGhostTableName()); err == nil { + mgtr.migrationContext.Log.Infof("New table structure follows") + fmt.Println(createTableStatement) + } else { + mgtr.migrationContext.Log.Errore(fmt.Errorf("error showing create table: %w", err)) + } } } + if err := mgtr.eventsStreamer.Close(); err != nil { mgtr.migrationContext.Log.Errore(err) } if mgtr.migrationContext.IsMoveTablesMode() { - if mgtr.migrationContext.Checkpoint { - if mgtr.migrationContext.OkToDropTable { - if err := mgtr.retryOperation(mgtr.applier.DropCheckpointTable); err != nil { - return err - } - } else if !mgtr.migrationContext.Noop { - mgtr.migrationContext.Log.Infof("Am not dropping checkpoint table without `--ok-to-drop-table`. To drop the checkpoint table, issue:") - mgtr.migrationContext.Log.Infof("-- drop table %s.%s", sql.EscapeName(mgtr.migrationContext.GetTargetDatabaseName()), sql.EscapeName(mgtr.migrationContext.GetCheckpointTableName())) - } - } - // for move-tables mode, we're done at this point - return nil + return mgtr.moveTablesFinalCleanup() } if err := mgtr.retryOperation(mgtr.applier.DropChangelogTable); err != nil { @@ -2660,6 +2671,71 @@ func (mgtr *Migrator) finalCleanup() error { return nil } +// moveTablesFinalCleanup handles artifact cleanup after a successful move-tables +// run. A successful run leaves two artifacts behind: +// the source `__del` table (the post-cutover rollback handle) and the target +// checkpoint table. There are no `_ghc`/`__gho` tables in move-tables mode. +// +// This runs only on the success path; on failure `__del` is never dropped and +// survives as the rollback handle (see logMoveTablesRollbackHint). +func (mgtr *Migrator) moveTablesFinalCleanup() error { + sourceDatabaseName := mgtr.migrationContext.DatabaseName + delTableName := mgtr.migrationContext.GetOldTableName() + targetDatabaseName := mgtr.migrationContext.GetTargetDatabaseName() + checkpointTableName := mgtr.migrationContext.GetCheckpointTableName() + + if mgtr.migrationContext.OkToDropTable { + // The source `__del` rollback handle only exists after a real cutover, + // never in Noop runs. The streamer owns the live source connection in + // both the normal and cutover-resume paths (its `db` handle uses the + // source config and Close() above only closed the binlog reader), so the + // source-side drop goes through it. + if !mgtr.migrationContext.Noop { + if err := mgtr.retryOperation(mgtr.eventsStreamer.DropSourceOldTable); err != nil { + return err + } + } + if mgtr.migrationContext.Checkpoint { + if err := mgtr.retryOperation(mgtr.applier.DropCheckpointTable); err != nil { + return err + } + } + return nil + } + + if mgtr.migrationContext.Noop { + return nil + } + + // --ok-to-drop-table not set: log the artifacts left behind and the exact + // commands to drop them. + mgtr.migrationContext.Log.Infof("Am not dropping move-tables artifacts without `--ok-to-drop-table`. The following are left behind:") + mgtr.migrationContext.Log.Infof("- source rollback handle %s.%s. To drop it, issue:", sql.EscapeName(sourceDatabaseName), sql.EscapeName(delTableName)) + mgtr.migrationContext.Log.Infof("-- drop table %s.%s", sql.EscapeName(sourceDatabaseName), sql.EscapeName(delTableName)) + if mgtr.migrationContext.Checkpoint { + mgtr.migrationContext.Log.Infof("- target checkpoint table %s.%s. To drop it, issue:", sql.EscapeName(targetDatabaseName), sql.EscapeName(checkpointTableName)) + mgtr.migrationContext.Log.Infof("-- drop table %s.%s", sql.EscapeName(targetDatabaseName), sql.EscapeName(checkpointTableName)) + } + return nil +} + +// logMoveTablesRollbackHint prints a clear rollback hint after a failed +// move-tables run in which the source RENAME already happened. The source +// `__del` table is intentionally left in place as the rollback handle +// and the operator rolls the source back by renaming +// `__del` to the original table name. We do NOT drop `__del` on a failure path. +func (mgtr *Migrator) logMoveTablesRollbackHint() { + sourceDatabaseName := mgtr.migrationContext.DatabaseName + originalTableName := mgtr.migrationContext.OriginalTableName + delTableName := mgtr.migrationContext.GetOldTableName() + mgtr.migrationContext.Log.Infof("move-tables run failed after the source rename; leaving %s.%s in place as the rollback handle.", + sql.EscapeName(sourceDatabaseName), sql.EscapeName(delTableName)) + mgtr.migrationContext.Log.Infof("To roll back the source table, issue:") + mgtr.migrationContext.Log.Infof("-- rename table %s.%s to %s.%s", + sql.EscapeName(sourceDatabaseName), sql.EscapeName(delTableName), + sql.EscapeName(sourceDatabaseName), sql.EscapeName(originalTableName)) +} + func (mgtr *Migrator) teardown() { atomic.StoreInt64(&mgtr.finishedMigrating, 1) diff --git a/go/logic/migrator_move_tables_cleanup_test.go b/go/logic/migrator_move_tables_cleanup_test.go new file mode 100644 index 000000000..31fb3679d --- /dev/null +++ b/go/logic/migrator_move_tables_cleanup_test.go @@ -0,0 +1,71 @@ +package logic + +import ( + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/github/gh-ost/go/base" +) + +// capturingLogger records Infof messages so tests can assert on the operator +// commands emitted by the move-tables cleanup path. It embeds base.Logger so it +// only needs to override the one method the tests care about. +type capturingLogger struct { + base.Logger + infofs []string +} + +func (l *capturingLogger) Infof(format string, args ...interface{}) { + l.infofs = append(l.infofs, fmt.Sprintf(format, args...)) +} + +func (l *capturingLogger) has(substr string) bool { + for _, line := range l.infofs { + if strings.Contains(line, substr) { + return true + } + } + return false +} + +func newCleanupTestMigrator() (*Migrator, *capturingLogger) { + logger := &capturingLogger{Logger: base.NewDefaultLogger()} + mc := base.NewMigrationContext() + mc.Log = logger + mc.DatabaseName = "source_db" + mc.OriginalTableName = "t" + mc.MoveTables.TableNames = []string{"t"} + mc.MoveTables.TargetDatabase = "target_db" + return NewMigrator(mc, "test"), logger +} + +// TestMoveTablesFinalCleanup_EmitsOperatorCommands verifies that without +// --ok-to-drop-table, cleanup drops nothing and instead logs the exact +// `drop table` commands for the source rollback handle and the target +// checkpoint table. The applier/streamer are nil, so any real drop would panic. +func TestMoveTablesFinalCleanup_EmitsOperatorCommands(t *testing.T) { + m, logger := newCleanupTestMigrator() + m.migrationContext.OkToDropTable = false + m.migrationContext.Checkpoint = true + + require.NoError(t, m.moveTablesFinalCleanup()) + + require.True(t, logger.has("-- drop table `source_db`.`_t_del`"), + "must emit the command to drop the source rollback handle") + require.True(t, logger.has("-- drop table `target_db`.`_t_ghk`"), + "must emit the command to drop the target checkpoint table") +} + +// TestLogMoveTablesRollbackHint_EmitsRenameCommand verifies the failure-path +// hint emits the exact rename command to roll the source table back. +func TestLogMoveTablesRollbackHint_EmitsRenameCommand(t *testing.T) { + m, logger := newCleanupTestMigrator() + + m.logMoveTablesRollbackHint() + + require.True(t, logger.has("-- rename table `source_db`.`_t_del` to `source_db`.`t`"), + "must emit the rename command to roll the source table back") +} diff --git a/go/logic/streamer.go b/go/logic/streamer.go index 7d3d00120..e8682d903 100644 --- a/go/logic/streamer.go +++ b/go/logic/streamer.go @@ -15,6 +15,7 @@ import ( "github.com/github/gh-ost/go/base" "github.com/github/gh-ost/go/binlog" "github.com/github/gh-ost/go/mysql" + "github.com/github/gh-ost/go/sql" "github.com/openark/golib/sqlutils" ) @@ -279,6 +280,32 @@ func (es *EventsStreamer) Close() (err error) { return err } +// DropSourceOldTable drops the source "__del" table in move-tables mode. The +// __del table is the post-cutover rollback handle on the source cluster; it is +// only dropped after a successful run when --ok-to-drop-table is set. +// The applier's dropTable targets the move-tables target cluster, +// so the source-side drop is owned by the streamer: its `db` +// handle uses InspectorConnectionConfig (the source) and stays open in both the +// normal and the cutover-resume paths (Close() only closes the binlog reader, +// not `db`). +func (es *EventsStreamer) DropSourceOldTable() error { + databaseName := es.migrationContext.DatabaseName + tableName := es.migrationContext.GetOldTableName() + query := fmt.Sprintf(`drop /* gh-ost */ table if exists %s.%s`, + sql.EscapeName(databaseName), + sql.EscapeName(tableName), + ) + es.migrationContext.Log.Infof("Dropping source table %s.%s", + sql.EscapeName(databaseName), + sql.EscapeName(tableName), + ) + if _, err := sqlutils.ExecNoPrepare(es.db, query); err != nil { + return err + } + es.migrationContext.Log.Infof("Source table dropped") + return nil +} + func (es *EventsStreamer) Teardown() { es.db.Close() } From 6c9e25bd6e4fd9be20760cabc8e8e6e30e61ad37 Mon Sep 17 00:00:00 2001 From: Chris Kirkland Date: Fri, 19 Jun 2026 14:28:11 -0600 Subject: [PATCH 12/23] tests: add move-table integration coverage Add local integration scenarios for single-table copying and concurrent writes. Refs: #1714 --- .github/workflows/move-tables-tests.yml | 46 ++ localtests/move-tables-test.sh | 507 ++++++++++++++++++ .../{ => single-concurrent-writes}/create.sql | 0 .../single-concurrent-writes/on_test.sh | 6 + .../single-concurrent-writes/tables.txt | 1 + localtests/move-tables/single/create.sql | 34 ++ localtests/move-tables/single/tables.txt | 1 + localtests/test.sh | 2 +- script/docker-gh-ost-move-tables-tests | 126 +++++ script/move-tables/insert-source-primary-loop | 3 +- script/move-tables/setup | 4 +- 11 files changed, 726 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/move-tables-tests.yml create mode 100755 localtests/move-tables-test.sh rename localtests/move-tables/{ => single-concurrent-writes}/create.sql (100%) create mode 100755 localtests/move-tables/single-concurrent-writes/on_test.sh create mode 100644 localtests/move-tables/single-concurrent-writes/tables.txt create mode 100644 localtests/move-tables/single/create.sql create mode 100644 localtests/move-tables/single/tables.txt create mode 100755 script/docker-gh-ost-move-tables-tests diff --git a/.github/workflows/move-tables-tests.yml b/.github/workflows/move-tables-tests.yml new file mode 100644 index 000000000..e028dbb5a --- /dev/null +++ b/.github/workflows/move-tables-tests.yml @@ -0,0 +1,46 @@ +name: move-tables tests +permissions: + contents: read + +on: [pull_request] + +jobs: + docker-tests: + runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + image: + # - 'mysql/mysql-server:5.7.41' # metadata locks not supported by default? (https://github.com/github/gh-ost/actions/runs/27841216224/job/82401716601?pr=1714) + - 'mysql:8.0.41' + - 'mysql:8.4.3' + - 'percona/percona-server:8.0.41-32' + env: + TEST_MYSQL_IMAGE: ${{ matrix.image }} + + steps: + - uses: actions/checkout@v4 + + - name: Setup environment + run: script/docker-gh-ost-move-tables-tests up + + - name: Run tests + run: script/docker-gh-ost-move-tables-tests run + + - name: Set artifact name + if: failure() + run: | + ARTIFACT_NAME=$(echo "${{ matrix.image }}" | tr '/:' '-') + echo "ARTIFACT_NAME=test-move-tables-logs-${ARTIFACT_NAME}" >> $GITHUB_ENV + + - name: Upload test logs on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: ${{ env.ARTIFACT_NAME }} + path: /tmp/gh-ost-test.* + retention-days: 7 + + - name: Teardown environment + if: always() + run: script/docker-gh-ost-move-tables-tests down diff --git a/localtests/move-tables-test.sh b/localtests/move-tables-test.sh new file mode 100755 index 000000000..bb9b52daf --- /dev/null +++ b/localtests/move-tables-test.sh @@ -0,0 +1,507 @@ +#!/bin/bash + +# Local integration tests. To be used by CI. +# See https://github.com/github/gh-ost/tree/doc/local-tests.md +# + +# Usage: localtests/test/sh [filter] +# By default, runs all move-tables tests. Given filter, will only run tests matching given regep + +repo_root=$(git rev-parse --show-toplevel) +script_path="$repo_root/script/move-tables" +tests_path=$(dirname $0)/move-tables +test_logfile=/tmp/gh-ost-test.log +default_ghost_binary=/tmp/gh-ost-test +ghost_binary="" +database=test +docker=false +gtid=false +storage_engine=innodb +exec_command_file=/tmp/gh-ost-test.bash +orig_structure_output_file=/tmp/gh-ost-test.orig.structure.sql +ghost_structure_output_file=/tmp/gh-ost-test.ghost.structure.sql +orig_content_output_file=/tmp/gh-ost-test.orig.content.csv +ghost_content_output_file=/tmp/gh-ost-test.ghost.content.csv +postpone_cutover_flag_file=/tmp/gh-ost-test.ghost.postpone.flag + +source_master_host= +source_master_port= +source_replica_host= +source_replica_port= +target_master_host= +target_master_port= +target_replica_host= +target_replica_port= +original_sql_mode= +current_gtid_mode= +test_timeout=120 +test_failure_log_tail_lines=50 + +OPTIND=1 +while getopts "b:s:dg" OPTION; do + case $OPTION in + b) + ghost_binary="$OPTARG" + ;; + s) + storage_engine="$OPTARG" + ;; + d) + docker=true + ;; + g) + gtid=true + ;; + esac +done +shift $((OPTIND - 1)) + +test_pattern="${1:-.}" + +mysql-exec() { + cluster=$1 + role=$2 + shift 2 + + if [[ $TEST_MYSQL_IMAGE =~ "mysql:8.4" ]]; then + $script_path/mysql-$cluster-$role --ssl-mode=required "$@" + else + $script_path/mysql-$cluster-$role "$@" + fi +} + +verify_master_and_replica() { + cluster=$1 + echo "Verifying $cluster cluster..." + + if [ "$(mysql-exec $cluster primary -e "select 1" -ss)" != "1" ]; then + echo "Cannot verify $cluster primary" + exit 1 + fi + read master_host master_port <<<$(mysql-exec $cluster primary -e "select @@hostname, @@port" -ss) + [ "$master_host" == "$(hostname)" ] && master_host="127.0.0.1" + echo "# master verified at $master_host:$master_port" + if ! mysql-exec $cluster primary -e "set global event_scheduler := 1"; then + echo "Cannot enable event_scheduler on master" + exit 1 + fi + original_sql_mode="$(mysql-exec $cluster primary -e "select @@global.sql_mode" -s -s)" + echo "sql_mode on master is ${original_sql_mode}" + + current_gtid_mode=$(mysql-exec $cluster primary -s -s -e "select @@global.gtid_mode" 2>/dev/null || echo unsupported) + current_enforce_gtid_consistency=$(mysql-exec $cluster primary -s -s -e "select @@global.enforce_gtid_consistency" 2>/dev/null || echo unsupported) + current_master_server_uuid=$(mysql-exec $cluster primary -s -s -e "select @@global.server_uuid" 2>/dev/null || echo unsupported) + current_replica_server_uuid=$(mysql-exec $cluster replica -s -s -e "select @@global.server_uuid" 2>/dev/null || echo unsupported) + echo "gtid_mode on master is ${current_gtid_mode} with enforce_gtid_consistency=${current_enforce_gtid_consistency}" + echo "server_uuid on master is ${current_master_server_uuid}, replica is ${current_replica_server_uuid}" + + echo "Gracefully sleeping for 3 seconds while replica is setting up..." + sleep 3 + + if [ "$(mysql-exec $cluster replica -e "select 1" -ss)" != "1" ]; then + echo "Cannot verify gh-ost-test-mysql-replica" + exit 1 + fi + if [ "$(mysql-exec $cluster replica -e "select @@global.binlog_format" -ss)" != "ROW" ]; then + echo "Expecting test replica to have binlog_format=ROW" + exit 1 + fi + read replica_host replica_port <<<$(mysql-exec $cluster replica -e "select @@hostname, @@port" -ss) + [ "$replica_host" == "$(hostname)" ] && replica_host="127.0.0.1" + echo "# replica verified at $replica_host:$replica_port" + + if [ "$docker" = true ]; then + master_host="0.0.0.0" + if [ "$cluster" == "source" ]; then + master_port="3307" + elif [ "$cluster" == "target" ]; then + master_port="3309" + fi + echo "# using docker master at $master_host:$master_port" + replica_host="0.0.0.0" + if [ "$cluster" == "source" ]; then + replica_port="3308" + elif [ "$cluster" == "target" ]; then + replica_port="3310" + fi + echo "# using docker replica at $replica_host:$replica_port" + fi + + if [ "$cluster" == "source" ]; then + source_master_host=$master_host + source_master_port=$master_port + source_replica_host=$replica_host + source_replica_port=$replica_port + elif [ "$cluster" == "target" ]; then + target_master_host=$master_host + target_master_port=$master_port + target_replica_host=$replica_host + target_replica_port=$replica_port + fi +} + +echo_dot() { + echo -n "." +} + +start_replication() { + cluster=$1 + + echo "Starting replication for $cluster..." + mysql_version="$(mysql-exec $cluster replica -e "select @@version")" + if [[ $mysql_version =~ "8.4" ]]; then + seconds_behind_source="Seconds_Behind_Source" + replica_terminology="replica" + else + seconds_behind_source="Seconds_Behind_Master" + replica_terminology="slave" + fi + mysql-exec $cluster replica -e "stop $replica_terminology; start $replica_terminology;" + + num_attempts=0 + while mysql-exec $cluster replica -e "show $replica_terminology status\G" | grep $seconds_behind_source | grep -q NULL; do + ((num_attempts = num_attempts + 1)) + if [ $num_attempts -gt 10 ]; then + echo + echo "ERROR replication failure" + exit 1 + fi + echo_dot + sleep 1 + done +} + +build_ghost_command() { + # Build gh-ost command with all standard options + # + # expected $1 to be a comma-separated list of tables to move + cmd="GOTRACEBACK=crash $ghost_binary \ + --move-tables=$1 \ + --user=root \ + --password=opensesame \ + --host=$source_replica_host \ + --port=$source_replica_port \ + --database=$database \ + --target-user=root \ + --target-password=opensesame \ + --target-host=$target_master_host \ + --target-port=$target_master_port \ + --target-database=$database \ + --serve-socket-file=/tmp/gh-ost.test.sock \ + --initially-drop-socket-file \ + --default-retries=3 \ + --chunk-size=10 \ + --verbose \ + --debug \ + --stack \ + --checkpoint \ + --postpone-cut-over-flag-file=$postpone_cutover_flag_file \ + --execute ${extra_args[@]}" +} + +print_log_excerpt() { + echo "=== Last $test_failure_log_tail_lines lines of $test_logfile ===" + tail -n $test_failure_log_tail_lines $test_logfile + echo "=== End log excerpt ===" +} + +validate_expected_failure() { + # Check if test expected to fail and validate error message + # Expects: tests_path, test_name, execution_result, test_logfile + if [ -f $tests_path/$test_name/expect_failure ]; then + if [ $execution_result -eq 0 ]; then + echo + echo "ERROR $test_name execution was expected to exit on error but did not." + print_log_excerpt + return 1 + fi + if [ -s $tests_path/$test_name/expect_failure ]; then + # 'expect_failure' file has content. We expect to find this content in the log. + expected_error_message="$(cat $tests_path/$test_name/expect_failure)" + if grep -q "$expected_error_message" $test_logfile; then + return 0 + fi + echo + echo "ERROR $test_name execution was expected to exit with error message '${expected_error_message}' but did not." + print_log_excerpt + return 1 + fi + # 'expect_failure' file has no content. We generally agree that the failure is correct + return 0 + fi + + if [ $execution_result -ne 0 ]; then + echo + echo "ERROR $test_name execution failure. cat $test_logfile:" + cat $test_logfile + return 1 + fi + return 0 +} + +cleanup() { + # reset test database + mysql-exec source primary --default-character-set=utf8mb4 $database -e "drop database if exists $database; create database $database;" + mysql-exec target primary --default-character-set=utf8mb4 $database -e "drop database if exists $database; create database $database;" +} + +test_single() { + local test_name + test_name="$1" + + # Read the list of tables to migrate from the test's tables.txt + if [ ! -f $tests_path/$test_name/tables.txt ]; then + echo "🐛 ERROR: $tests_path/$test_name/tables.txt not found" + return 1 + fi + echo "----" + cat $tests_path/$test_name/tables.txt + echo "----" + tables_to_migrate=() + while IFS='' read -r line; do + tables_to_migrate+=("$line") + done < <(cat $tests_path/$test_name/tables.txt) + + if [ -f $tests_path/$test_name/ignore_versions ]; then + ignore_versions=$(cat $tests_path/$test_name/ignore_versions) + mysql_version=$(mysql-exec source primary -s -s -e "select @@version") + mysql_version_comment=$(mysql-exec source primary -s -s -e "select @@version_comment") + if echo "$mysql_version" | egrep -q "^${ignore_versions}"; then + echo -n "Skipping: $test_name" + return 0 + elif echo "$mysql_version_comment" | egrep -i -q "^${ignore_versions}"; then + echo -n "Skipping: $test_name" + return 0 + fi + fi + + echo -n "Testing: $test_name (${#tables_to_migrate[@]} table(s))" + + echo_dot + start_replication source + start_replication target + echo_dot + + if [ -f $tests_path/$test_name/gtid_mode ]; then + target_gtid_mode=$(cat $tests_path/$test_name/gtid_mode) + if [ "$current_gtid_mode" != "$target_gtid_mode" ]; then + echo "gtid_mode is ${current_gtid_mode}, expected ${target_gtid_mode}" + exit 1 + fi + fi + + if [ -f $tests_path/$test_name/sql_mode ]; then + mysql-exec source primary --default-character-set=utf8mb4 $database -e "set @@global.sql_mode='$(cat $tests_path/$test_name/sql_mode)'" + mysql-exec source replica --default-character-set=utf8mb4 $database -e "set @@global.sql_mode='$(cat $tests_path/$test_name/sql_mode)'" + mysql-exec target primary --default-character-set=utf8mb4 $database -e "set @@global.sql_mode='$(cat $tests_path/$test_name/sql_mode)'" + mysql-exec target replica --default-character-set=utf8mb4 $database -e "set @@global.sql_mode='$(cat $tests_path/$test_name/sql_mode)'" + fi + + mysql-exec source primary --default-character-set=utf8mb4 $database <$tests_path/$test_name/create.sql + test_create_result=$? + + if [ $test_create_result -ne 0 ]; then + echo + echo "ERROR $test_name create failure. cat $tests_path/$test_name/create.sql:" + cat $tests_path/$test_name/create.sql + return 1 + fi + + extra_args="" + if [ -f $tests_path/$test_name/extra_args ]; then + extra_args=$(cat $tests_path/$test_name/extra_args) + fi + if [ "$gtid" = true ]; then + extra_args+=" --gtid" + fi + + # graceful sleep for replica to catch up + echo_dot + sleep 1 + + # Check for custom test script + if [ -f $tests_path/$test_name/test.sh ]; then + # Run the custom test script in a subshell with timeout monitoring + # The subshell inherits all functions and variables from the current shell + (source $tests_path/$test_name/test.sh) & + test_pid=$! + + # Monitor the test with timeout + timeout_counter=0 + while kill -0 $test_pid 2>/dev/null; do + if [ $timeout_counter -ge $test_timeout ]; then + kill -TERM $test_pid 2>/dev/null + sleep 1 + kill -KILL $test_pid 2>/dev/null + wait $test_pid 2>/dev/null + echo + echo "ERROR $test_name execution timed out" + print_log_excerpt + return 1 + fi + sleep 1 + ((timeout_counter++)) + done + + # Get the exit code + wait $test_pid 2>/dev/null + execution_result=$? + return $execution_result + fi + + # kick off the on_test script for the test. this enables arbitrary custom logic + # concurrent with the gh-ost process. this enables additional scenarios like + # streaming of writes prior to the write cutover. + # + # IMPORTANT: The on-test script is executed in the background and will be killed as soon + # as the gh-ost process terminates. + if [ -f $tests_path/$test_name/on_test.sh ]; then + $tests_path/$test_name/on_test.sh &> /dev/null & + on_test_pid=$! + fi + + # queue up removal of the postpone cutover flag, otherwise gh-ost hangs on the cutover + ( + sleep 1; + echo "⏩ Sending unpostpone cutover" + rm $postpone_cutover_flag_file &> /dev/null; + ) & + + # Build and execute gh-ost command + move_tables_arg=$(IFS=, ; echo "${tables_to_migrate[*]}") + build_ghost_command "$move_tables_arg" + echo_dot + echo $cmd >$exec_command_file + echo_dot + timeout $test_timeout bash $exec_command_file >$test_logfile 2>&1 + + execution_result=$? + + if [ -n "$on_test_pid" ]; then + kill -KILL $on_test_pid &>/dev/null + fi + + # Check for timeout (exit code 124) + if [ $execution_result -eq 124 ]; then + echo + echo "ERROR $test_name execution timed out" + print_log_excerpt + return 1 + fi + + if [ -f $tests_path/$test_name/sql_mode ]; then + mysql-exec source primary --default-character-set=utf8mb4 $database -e "set @@global.sql_mode='${original_sql_mode}'" + mysql-exec source replica --default-character-set=utf8mb4 $database -e "set @@global.sql_mode='${original_sql_mode}'" + mysql-exec target primary --default-character-set=utf8mb4 $database -e "set @@global.sql_mode='${original_sql_mode}'" + mysql-exec target replica --default-character-set=utf8mb4 $database -e "set @@global.sql_mode='${original_sql_mode}'" + fi + + # Validate expected failure or success + if ! validate_expected_failure; then + return 1 + fi + + # If this was an expected failure test, we're done (no need to validate structure/checksums) + if [ -f $tests_path/$test_name/expect_failure ]; then + return 0 + fi + + # Test succeeded - now validate structure/checksums and contents + for table_name in "${tables_to_migrate[@]}"; do + echo "⚙️ Validating table: $table_name" + + # Validate that the structure of the table matches on the source and target clusters (accounting for table rename on source) + mysql-exec source replica --default-character-set=utf8mb4 $database -e "show create table _${table_name}_del\G" -ss | sed -e "s/_${table_name}_del/${table_name}/g" >$orig_structure_output_file + sed -i "s/_${table_name}_del/${table_name}/g" $orig_structure_output_file + mysql-exec target replica --default-character-set=utf8mb4 $database -e "show create table ${table_name}\G" -ss >$ghost_structure_output_file + + if ! diff $orig_structure_output_file $ghost_structure_output_file > /dev/null 2>&1 ; then + echo "ERROR $test_name: structure mismatch on table $table_name" + echo "---" + diff $orig_structure_output_file $ghost_structure_output_file + + echo "diff $orig_structure_output_file $ghost_structure_output_file" + + return 1 + fi + echo "✅ Table $table_name: structures match" + + echo_dot + + # validate contents match + mysql-exec source replica --default-character-set=utf8mb4 $database -e "select * from _${table_name}_del" -ss >$orig_content_output_file + mysql-exec target replica --default-character-set=utf8mb4 $database -e "select * from ${table_name}" -ss >$ghost_content_output_file + orig_content_checksum=$(cat $orig_content_output_file | md5sum) + ghost_content_checksum=$(cat $ghost_content_output_file | md5sum) + + if [ "$orig_content_checksum" != "$ghost_content_checksum" ]; then + mysql-exec source replica --default-character-set=utf8mb4 $database -e "select * from _${table_name}_del" -ss >$orig_content_output_file + mysql-exec target replica --default-character-set=utf8mb4 $database -e "select * from ${table_name}" -ss >$ghost_content_output_file + echo "ERROR $test_name: checksum mismatch on table $table_name" + echo "---" + diff $orig_content_output_file $ghost_content_output_file + + echo "diff $orig_content_output_file $ghost_content_output_file" + + return 1 + fi + echo "✅ Table $table_name: content checksums match" + + echo_dot + echo_dot + done +} + +build_binary() { + echo "Building" + rm -f $default_ghost_binary + [ "$ghost_binary" == "" ] && ghost_binary="$default_ghost_binary" + if [ -f "$ghost_binary" ]; then + echo "Using binary: $ghost_binary" + return 0 + fi + + go build -o $ghost_binary go/cmd/gh-ost/main.go + + if [ $? -ne 0 ]; then + echo "Build failure" + exit 1 + fi +} + +test_all() { + build_binary + test_dirs=$(find "$tests_path" -mindepth 1 -maxdepth 1 ! -path . -type d | grep "$test_pattern" | sort) + while read -r test_dir; do + test_name=$(basename "$test_dir") + local test_start_time=$(date +%s) + if ! test_single "$test_name"; then + local test_end_time=$(date +%s) + local test_duration=$((test_end_time - test_start_time)) + echo "+ FAIL (${test_duration}s)" + return 1 + else + local test_end_time=$(date +%s) + local test_duration=$((test_end_time - test_start_time)) + echo + echo "+ pass (${test_duration}s)" + fi + + cleanup + + for cluster in source target; do + mysql_version="$(mysql-exec $cluster replica -e "select @@version")" + replica_terminology="slave" + if [[ $mysql_version =~ "8.4" ]]; then + replica_terminology="replica" + fi + mysql-exec $cluster replica -e "start $replica_terminology" + done + done <<<"$test_dirs" + + echo "✅ All tests completed." +} + +verify_master_and_replica source +verify_master_and_replica target +test_all diff --git a/localtests/move-tables/create.sql b/localtests/move-tables/single-concurrent-writes/create.sql similarity index 100% rename from localtests/move-tables/create.sql rename to localtests/move-tables/single-concurrent-writes/create.sql diff --git a/localtests/move-tables/single-concurrent-writes/on_test.sh b/localtests/move-tables/single-concurrent-writes/on_test.sh new file mode 100755 index 000000000..71120eb95 --- /dev/null +++ b/localtests/move-tables/single-concurrent-writes/on_test.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +# insert data into the source primary, starting at ID 100 in batches of 10. kill +# the process after 5 seconds +DATABASE=test script/move-tables/insert-source-primary-loop 100 0.1 10 & +sleep 5 && kill $! diff --git a/localtests/move-tables/single-concurrent-writes/tables.txt b/localtests/move-tables/single-concurrent-writes/tables.txt new file mode 100644 index 000000000..11fc5eef8 --- /dev/null +++ b/localtests/move-tables/single-concurrent-writes/tables.txt @@ -0,0 +1 @@ +gh_ost_test diff --git a/localtests/move-tables/single/create.sql b/localtests/move-tables/single/create.sql new file mode 100644 index 000000000..46e919003 --- /dev/null +++ b/localtests/move-tables/single/create.sql @@ -0,0 +1,34 @@ +drop table if exists gh_ost_test; +create table gh_ost_test ( + id bigint(20) NOT NULL AUTO_INCREMENT, + column1 int(11) NOT NULL, + column2 smallint(5) unsigned NOT NULL, + column3 mediumint(8) unsigned NOT NULL, + column4 tinyint(3) unsigned NOT NULL, + column5 int(11) NOT NULL, + column6 int(11) NOT NULL, + PRIMARY KEY (id), + KEY c12_ix (column1, column2) +) auto_increment=1; + +insert into gh_ost_test values + (NULL, 1001, 100, 500000, 10, 1700000001, 1700000002), + (NULL, 1002, 200, 600000, 20, 1700000003, 1700000004), + (NULL, 1003, 300, 700000, 30, 1700000005, 1700000006), + (NULL, 1004, 400, 800000, 40, 1700000007, 1700000008), + (NULL, 1005, 500, 900000, 50, 1700000009, 1700000010), + (NULL, 1006, 600, 1000000, 60, 1700000011, 1700000012), + (NULL, 1007, 700, 1100000, 70, 1700000013, 1700000014), + (NULL, 1008, 800, 1200000, 80, 1700000015, 1700000016), + (NULL, 1009, 900, 1300000, 90, 1700000017, 1700000018), + (NULL, 1010, 1000, 1400000, 100, 1700000019, 1700000020), + (NULL, 1011, 1100, 1500000, 110, 1700000021, 1700000022), + (NULL, 1012, 1200, 1600000, 120, 1700000023, 1700000024), + (NULL, 1013, 1300, 1700000, 130, 1700000025, 1700000026), + (NULL, 1014, 1400, 1800000, 140, 1700000027, 1700000028), + (NULL, 1015, 1500, 1900000, 150, 1700000029, 1700000030), + (NULL, 1016, 1600, 2000000, 160, 1700000031, 1700000032), + (NULL, 1017, 1700, 2100000, 170, 1700000033, 1700000034), + (NULL, 1018, 1800, 2200000, 180, 1700000035, 1700000036), + (NULL, 1019, 1900, 2300000, 190, 1700000037, 1700000038), + (NULL, 1020, 2000, 2400000, 200, 1700000039, 1700000040); \ No newline at end of file diff --git a/localtests/move-tables/single/tables.txt b/localtests/move-tables/single/tables.txt new file mode 100644 index 000000000..11fc5eef8 --- /dev/null +++ b/localtests/move-tables/single/tables.txt @@ -0,0 +1 @@ +gh_ost_test diff --git a/localtests/test.sh b/localtests/test.sh index dfc9bee6a..dd9ae722d 100755 --- a/localtests/test.sh +++ b/localtests/test.sh @@ -522,7 +522,7 @@ build_binary() { test_all() { build_binary - test_dirs=$(find "$tests_path" -mindepth 1 -maxdepth 1 ! -path . -type d | grep "$test_pattern" | sort) + test_dirs=$(find "$tests_path" -mindepth 1 -maxdepth 1 ! -path . -type d | grep "$test_pattern" | grep -v move-tables | sort) # Read the test list on FD 3, not stdin: the mysql wrappers may run # `docker exec -i`, which attaches and drains stdin. On stdin (FD 0) the # first such call inside the loop would swallow the remaining test-dir diff --git a/script/docker-gh-ost-move-tables-tests b/script/docker-gh-ost-move-tables-tests new file mode 100755 index 000000000..4e7f09c6a --- /dev/null +++ b/script/docker-gh-ost-move-tables-tests @@ -0,0 +1,126 @@ +#!/bin/bash + +# This script starts four MySQL docker containers in a primary-replica setup +# for two distinct clusters which can be used to run move-tables tests +# in localtests/ . +# Set the environment var TEST_MYSQL_IMAGE to change the docker image. +# +# Usage: +# docker-gh-ost-move-tables-tests up [-t] start the containers +# docker-gh-ost-move-tables-tests down remove the containers +# docker-gh-ost-move-tables-tests run [-t] run move-tables tests on the containers + +set -e + +GH_OST_ROOT=$(git rev-parse --show-toplevel) +if [[ ":$PATH:" != *":$GH_OST_ROOT:"* ]]; then + export PATH="${PATH}:${GH_OST_ROOT}/script" +fi +SCRIPT_PATH="${GH_OST_ROOT}/script/move-tables" + +poll_mysql() { + cluster=$1 + role=$2 + + echo "polling $role for $cluster..." + + CTR=0 + while ! mysql-exec "$cluster" "$role" -e "select 1;" >/dev/null 2>&1; do + sleep 1 + CTR=$((CTR + 1)) + if [ $CTR -gt 30 ]; then + echo " ❌ MySQL $cluster/$role failed to start" + return 1 + fi + done + echo " ✔ MySQL $cluster/$role OK" + return 0 +} + +mysql-exec() { + cluster=$1 + role=$2 + shift 2 + + if [[ $TEST_MYSQL_IMAGE =~ "mysql:8.4" ]]; then + $SCRIPT_PATH/mysql-$cluster-$role --ssl-mode=required "$@" + else + $SCRIPT_PATH/mysql-$cluster-$role "$@" + fi +} + +setup() { + [ -z "$TEST_MYSQL_IMAGE" ] && TEST_MYSQL_IMAGE="mysql:8.0.41" + + echo "Starting MySQL $TEST_MYSQL_IMAGE containers..." + compose_file="$GH_OST_ROOT/localtests/docker-compose-move-tables.yml" + MYSQL_SHA2_RSA_KEYS_FLAG="" + MYSQL_PASSWORD_HASHING_ALGORITHM="mysql_native_password" + if [[ $TEST_MYSQL_IMAGE =~ "mysql:8.4" ]]; then + MYSQL_PASSWORD_HASHING_ALGORITHM="caching_sha2_password" + MYSQL_SHA2_RSA_KEYS_FLAG="--caching-sha2-password-auto-generate-rsa-keys=ON" + fi + (TEST_MYSQL_IMAGE="$TEST_MYSQL_IMAGE" MYSQL_SHA2_RSA_KEYS_FLAG="$MYSQL_SHA2_RSA_KEYS_FLAG" envsubst <"$compose_file") >"$compose_file.tmp" + + docker compose -f "$compose_file.tmp" up -d --wait + + for cluster in "source" "target"; do + echo "-----------------------------------" + echo "--- Setting up $cluster cluster ---" + echo "-----------------------------------" + + echo "Waiting for MySQL..." + poll_mysql "$cluster" "primary" || exit 1 + poll_mysql "$cluster" "replica" || exit 1 + + echo -n "Setting up replication..." + mysql-exec "$cluster" "primary" -e "create user if not exists 'repl'@'%' identified with $MYSQL_PASSWORD_HASHING_ALGORITHM by 'repl';" + mysql-exec "$cluster" "primary" -e "grant replication slave on *.* to 'repl'@'%'; flush privileges;" + mysql-exec "$cluster" "primary" -e "create user if not exists 'gh-ost'@'%' identified with $MYSQL_PASSWORD_HASHING_ALGORITHM by 'gh-ost';" + mysql-exec "$cluster" "primary" -e "grant all on *.* to 'gh-ost'@'%';" + + primary_port=3307 + if [ "$cluster" = "target" ]; then + primary_port=3309 + fi + + sleep 1 + if [[ $TEST_MYSQL_IMAGE =~ "mysql:8.4" ]]; then + mysql-exec "$cluster" "replica" -e "change replication source to source_host='mysql-$cluster-primary', source_port=$primary_port, source_user='repl', source_password='repl', source_auto_position=1, source_ssl=1;" + mysql-exec "$cluster" "replica" -e "start replica;" + else + mysql-exec "$cluster" "replica" -e "change master to master_host='mysql-$cluster-primary', master_port=$primary_port, master_user='repl', master_password='repl', master_auto_position=1;" + mysql-exec "$cluster" "replica" -e "start slave;" + fi + echo "OK" + done +} + +teardown() { + echo "Tearing down..." + compose_file="$GH_OST_ROOT/localtests/docker-compose-move-tables.yml" + docker compose -f "$compose_file.tmp" down --volumes --remove-orphans +} + +main() { + local cmd="$1" + if [[ "$cmd" == "up" ]]; then + setup + elif [[ "$cmd" == "down" ]]; then + teardown + elif [[ "$cmd" == "run" ]]; then + shift 1 + + if [ "$1" == "--rm" ]; then + # run teardown, setup,a nd run for convenience + echo "⚙️ Bootstrapping environment from scratch..." + teardown + setup + shift 1 + fi + + "$GH_OST_ROOT/localtests/move-tables-test.sh" -d "$@" + fi +} + +main "$@" diff --git a/script/move-tables/insert-source-primary-loop b/script/move-tables/insert-source-primary-loop index d2a0afddc..490442ec4 100755 --- a/script/move-tables/insert-source-primary-loop +++ b/script/move-tables/insert-source-primary-loop @@ -13,6 +13,7 @@ start_i="${1:-100000}" delay="${2:-0.2}" rows_per_batch="${3:-1}" i="$start_i" +DATABASE="${DATABASE:-gh_ost_test_db} echo "Starting continuous inserts on source primary. Press Ctrl+C to stop." echo "start_column1=$start_i sleep_seconds=$delay rows_per_batch=$rows_per_batch" @@ -33,7 +34,7 @@ while true; do values+="$row" done - script/move-tables/mysql-source-primary -D gh_ost_test_db -e " + script/move-tables/mysql-source-primary -D $DATABASE -e " INSERT INTO gh_ost_test (column1, column2, column3, column4, column5, column6) VALUES $values; " diff --git a/script/move-tables/setup b/script/move-tables/setup index 813e7203e..f3ff5e0f4 100755 --- a/script/move-tables/setup +++ b/script/move-tables/setup @@ -11,7 +11,7 @@ set -e GH_OST_ROOT=$(git rev-parse --show-toplevel) SCRIPT_PATH="${GH_OST_ROOT}/script/move-tables" -DATABASE_NAME="gh_ost_test_db" +DATABASE_NAME="test" poll_mysql() { CTR=0 @@ -127,7 +127,7 @@ setup() { echo "OK" echo -n "Seeding data in source cluster..." - exec-mysql-source-primary -D $DATABASE_NAME < "$GH_OST_ROOT/localtests/move-tables/create.sql" + exec-mysql-source-primary -D $DATABASE_NAME < "$GH_OST_ROOT/localtests/move-tables/single/create.sql" echo "OK" } From 0589dc102f3433b4d4e37d9c8958d973fa5d9d4c Mon Sep 17 00:00:00 2001 From: Daniel Joos Date: Mon, 22 Jun 2026 15:12:00 +0200 Subject: [PATCH 13/23] move-tables: restore drain GTID on resume Restore the move-table drain GTID when resuming an interrupted migration. Refs: #1720 --- go/logic/migrator.go | 1 + 1 file changed, 1 insertion(+) diff --git a/go/logic/migrator.go b/go/logic/migrator.go index 73a55df64..fe04ad029 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -988,6 +988,7 @@ func (mgtr *Migrator) resumeMoveTablesCutOverFromCheckpoint(chk *Checkpoint) err } atomic.StoreInt64(&mgtr.migrationContext.CutOverCompleteFlag, 1) mgtr.migrationContext.Log.Debugf("T4: CutOverCompleteFlag set") + mgtr.migrationContext.MoveTables.DrainGTID = chk.MoveTablesCutOverDrainGTID if err := mgtr.hooksExecutor.OnSuccess(false); err != nil { return fmt.Errorf("on-success hook failed: %w", err) } From fc933034cb7f503ca1428f63e5bef8d0cbcc44da Mon Sep 17 00:00:00 2001 From: Zach Sierakowski Date: Mon, 22 Jun 2026 14:49:58 +0000 Subject: [PATCH 14/23] move-tables: require source-primary connections Introduce source-primary connection configuration and guard primary-required move-table operations. Refs: #1718 --- go/base/context.go | 29 +- go/cmd/gh-ost/main.go | 1 + go/logic/applier.go | 13 +- go/logic/applier_test.go | 4 +- go/logic/migrator.go | 268 +++++++++++++++--- go/logic/migrator_move_tables_cleanup_test.go | 12 + go/logic/migrator_move_tables_cutover_test.go | 142 +++++++++- go/logic/streamer.go | 27 -- localtests/move-tables-test.sh | 1 + script/move-tables/README.md | 7 +- script/move-tables/reset | 17 ++ script/move-tables/setup | 28 ++ 12 files changed, 462 insertions(+), 87 deletions(-) create mode 100755 script/move-tables/reset diff --git a/go/base/context.go b/go/base/context.go index 33c0b4794..32a9f849e 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -283,14 +283,31 @@ type MigrationContext struct { // move tables: MoveTables struct { - TableNames []string // List of table names to be moved. - TargetHost string // Target hostname for the move. This must be a primary/writable host. - TargetPort int // Target MySQL port for the move. - TargetUser string // Target username for the move. If not specified, it will default to the source user. - TargetPass string // Target password for the move. If not specified, it will default to the source password. - TargetDatabase string // Target database name for the move. If not specified, it will default to the source database name. + TableNames []string // List of table names to be moved. + TargetHost string // Target hostname for the move. This must be a primary/writable host. + TargetPort int // Target MySQL port for the move. + TargetUser string // Target username for the move. If not specified, it will default to the source user. + TargetPass string // Target password for the move. If not specified, it will default to the source password. + TargetDatabase string // Target database name for the move. If not specified, it will default to the source database name. + + // AllowOnSourcePrimary opts in to running the move-tables read path (schema + // inspection, the full row copy, binlog streaming) directly against the + // source cluster's primary. By default gh-ost stops early when --host is the + // primary, since reading the whole table copy from the primary is the load + // move-tables is meant to avoid; the operator should point --host at a replica. + AllowOnSourcePrimary bool + ConnectionConfig *mysql.ConnectionConfig + // SourcePrimaryConnectionConfig is the detected source-cluster primary. All + // source reads (schema inspection, row copy, binlog streaming) go through the + // inspector config (InspectorConnectionConfig), which may point at a read + // replica to take load off the primary. The cutover RENAME + drain-GTID + // capture and the source `__del` DROP must run on a writable primary, so they + // use this dedicated config. When the source --host is itself the primary (no + // replica topology), detection returns the inspector key and the two coincide. + SourcePrimaryConnectionConfig *mysql.ConnectionConfig + DrainGTID mysql.BinlogCoordinates // Source @@gtid_executed captured immediately after the source RENAME TABLE; the applier drains until it reaches this coordinate (move-tables only). } diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index ba285f54e..a6525fbe1 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -196,6 +196,7 @@ func main() { flag.StringVar(&migrationContext.MoveTables.TargetUser, "target-user", "", "Target MySQL username for --move-tables mode. If not provided, uses the same user as the source connection") flag.StringVar(&migrationContext.MoveTables.TargetPass, "target-password", "", "Target MySQL password for --move-tables mode. If not provided, uses the same password as the source connection") flag.StringVar(&migrationContext.MoveTables.TargetDatabase, "target-database", "", "Target MySQL database name for --move-tables mode. If not provided, uses the same database name as the source connection") + flag.BoolVar(&migrationContext.MoveTables.AllowOnSourcePrimary, "allow-on-source-primary", false, "allow --move-tables to read (schema, row copy, binlog) from the source cluster's primary. By default gh-ost stops if --host is the primary; prefer pointing --host at a replica to spare the primary the copy load.") flag.CommandLine.SetOutput(os.Stdout) flag.Parse() diff --git a/go/logic/applier.go b/go/logic/applier.go index 94c437cb6..5c0cceb6a 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -198,6 +198,13 @@ func (apl *Applier) InitDBConnections() (err error) { if _, err := base.ValidateConnection(apl.moveTablesTargetDB, apl.moveTablesConnectionConfig, apl.migrationContext, apl.name); err != nil { return err } + // Fail fast if the move-tables target is not a writable primary. All target + // work (table create, row-copy INSERT, checkpoint writes, checkpoint DROP) + // requires a writable host; catching read_only here turns a confusing + // mid-run write failure into a clear startup error. + if err := assertConnectionWritable(apl.moveTablesTargetDB, apl.moveTablesConnectionConfig.Key, "target"); err != nil { + return err + } } apl.migrationContext.Log.Infof("Applier initiated on %+v, version %+v", apl.connectionConfig.ImpliedKey, apl.migrationContext.ApplierMySQLVersion) return nil @@ -1152,7 +1159,7 @@ func (apl *Applier) ReadLastCheckpoint() (*Checkpoint, error) { chk.LastTrxCoords = fileCoords } if apl.migrationContext.IsMoveTablesMode() && drainGTIDStr != "" { - drainGTID, err := mysql.NewGTIDBinlogCoordinates(drainGTIDStr) + drainGTID, err := mysql.NewGTIDBinlogCoordinates(mysql.FlavorFor(apl.migrationContext.InspectorMySQLVersion), drainGTIDStr) if err != nil { return nil, err } @@ -1176,7 +1183,7 @@ func (apl *Applier) ReadMoveTablesCutOverCheckpoint() (*Checkpoint, error) { chk.Timestamp = time.Unix(timestamp, 0) if coordStr != "" { if apl.migrationContext.UseGTIDs { - coords, err := mysql.NewGTIDBinlogCoordinates(coordStr) + coords, err := mysql.NewGTIDBinlogCoordinates(mysql.FlavorFor(apl.migrationContext.InspectorMySQLVersion), coordStr) if err != nil { return nil, err } @@ -1190,7 +1197,7 @@ func (apl *Applier) ReadMoveTablesCutOverCheckpoint() (*Checkpoint, error) { } } if drainGTIDStr != "" { - drainGTID, err := mysql.NewGTIDBinlogCoordinates(drainGTIDStr) + drainGTID, err := mysql.NewGTIDBinlogCoordinates(mysql.FlavorFor(apl.migrationContext.InspectorMySQLVersion), drainGTIDStr) if err != nil { return nil, err } diff --git a/go/logic/applier_test.go b/go/logic/applier_test.go index 1e1183692..05f0dc44e 100644 --- a/go/logic/applier_test.go +++ b/go/logic/applier_test.go @@ -1279,9 +1279,9 @@ func (suite *ApplierTestSuite) TestWriteCheckpointMoveTables() { err = applier.ReadMigrationRangeValues(inspector.db) suite.Require().NoError(err) - coords, err := mysql.NewGTIDBinlogCoordinates("00000000-0000-0000-0000-000000000001:1-10") + coords, err := mysql.NewGTIDBinlogCoordinates(mysql.MySQLFlavor, "00000000-0000-0000-0000-000000000001:1-10") suite.Require().NoError(err) - drainGTID, err := mysql.NewGTIDBinlogCoordinates("00000000-0000-0000-0000-000000000001:1-20") + drainGTID, err := mysql.NewGTIDBinlogCoordinates(mysql.MySQLFlavor, "00000000-0000-0000-0000-000000000001:1-20") suite.Require().NoError(err) chk := &Checkpoint{ diff --git a/go/logic/migrator.go b/go/logic/migrator.go index fe04ad029..b904d8fa3 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -7,6 +7,7 @@ package logic import ( "context" + gosql "database/sql" "errors" "fmt" "io" @@ -93,6 +94,12 @@ type Migrator struct { migrationContext *base.MigrationContext statusWriter io.Writer + // sourcePrimaryDB is the writable source-cluster primary handle used only for + // move-tables cutover writes (the RENAME + drain-GTID capture) and the source + // `__del` DROP. Source reads use the inspector/streamer connections, which may + // point at a read replica. nil outside move-tables mode. + sourcePrimaryDB *gosql.DB + firstThrottlingCollected chan bool ghostTableMigrated chan bool rowCopyComplete chan error @@ -908,6 +915,14 @@ func (mgtr *Migrator) drainMoveTablesCutOver(drainGTID mysql.BinlogCoordinates) defer cancel() ticker := time.NewTicker(moveTablesCutOverDrainPollInterval) defer ticker.Stop() + // fallbackStreak counts consecutive polls where the streamer-frontier fallback + // held. Requiring two consecutive observations closes the sub-µs window between + // `eventStruct := <-applyEventsQueue` in executeWriteFuncs and the in-flight + // increment inside onApplyEventStruct: across a full poll interval a popped-but- + // unapplied event is either applied (advancing applierCoords, so the normal path + // handles it) or still in flight (applyInFlight>0 resets the streak). + fallbackStreak := 0 + const fallbackStreakRequired = 2 for { if err := mgtr.checkAbort(); err != nil { return err @@ -941,9 +956,17 @@ func (mgtr *Migrator) drainMoveTablesCutOver(drainGTID mysql.BinlogCoordinates) } // Fallback for non-DML tail: GTID can advance due to unrelated/non-row events, // so applier may stop moving while streamer has already crossed drain. + // Debounced across consecutive polls so the receive-vs-in-flight window + // cannot trigger a premature completion (see fallbackStreak above). if applyInFlight == 0 && moveTablesDrainProvenByStreamerProgress(drainGTID, streamerCoords, applyBacklog, streamerBacklog) { - mgtr.migrationContext.Log.Infof("T3: drain complete via streamer frontier (non-DML tail after T2)") - return nil + fallbackStreak++ + if fallbackStreak >= fallbackStreakRequired { + mgtr.migrationContext.Log.Infof("T3: drain complete via streamer frontier (non-DML tail after T2)") + return nil + } + mgtr.migrationContext.Log.Debugf("T3: streamer frontier reached, debouncing (%d/%d)", fallbackStreak, fallbackStreakRequired) + } else { + fallbackStreak = 0 } if drainReached { mgtr.migrationContext.Log.Debugf("T3: drain GTID reached but backlog remains (apply=%d, streamer=%d, in_flight=%d)", applyBacklog, streamerBacklog, applyInFlight) @@ -1059,6 +1082,14 @@ func (mgtr *Migrator) MoveTables() (err error) { if err := mgtr.initiateStreaming(); err != nil { return err } + // The cutover-resume path skips initiateInspector, so set up the + // source-primary connection here. It is needed by finalCleanup to drop + // the source `__del` rollback handle on a writable primary; the streamer + // connection may be a read replica. Uses the streamer's source version + // for replica-status terminology. + if err := mgtr.setupMoveTablesSourcePrimary(mgtr.eventsStreamer.dbVersion); err != nil { + return err + } if err := mgtr.applier.prepareQueries(); err != nil { return err } @@ -1275,44 +1306,80 @@ func (mgtr *Migrator) moveTablesCutOver() (err error) { return fmt.Errorf("on-before-cut-over hook failed: %w", err) } - // ----- T1 + T2: RENAME then capture @@gtid_executed on the same connection ----- - // Pin both operations to a single *sql.Conn so MySQL's within-session - // ordering guarantee makes it impossible for T2 to observe a state that - // pre-dates T1's commit. Using mgtr.inspector.db directly would let the - // pool schedule T1 and T2 on different underlying TCP connections (or, with - // a proxy, different servers), breaking the happens-before relationship. + // ----- T1 + T2: RENAME then capture @@gtid_executed in ONE round trip ----- + // A single multi-statement query (RENAME ...; SELECT @@global.gtid_executed) + // collapses the two operations into one server round trip, so there is no + // client-side or pool-scheduling gap between them: database/sql runs both + // statements on the same pooled connection, and the server executes them in + // order. The captured GTID is therefore the source executed set immediately + // after — and causally after — the rename commit, and a client-side crash can + // no longer land between the rename and the capture. The source-primary DSN + // sets multiStatements=true (see setupMoveTablesSourcePrimary). + // + // We use the dedicated source-primary handle (NOT mgtr.inspector.db): the + // inspector may be a read replica, while the RENAME must run on a writable + // primary. // - // No retry on the RENAME: it is not idempotent — a partial success leaves - // the table already renamed and a retry would fail. The operator re-runs - // the whole hook chain on failure. + // No retry on the RENAME: it is not idempotent — a partial success leaves the + // table already renamed and a retry would fail. The operator re-runs the whole + // hook chain on failure. cutOverCtx := mgtr.migrationContext.GetContext() - pinnedConn, err := mgtr.inspector.db.Conn(cutOverCtx) - if err != nil { - return fmt.Errorf("failed to pin connection for T1/T2: %w", err) + if mgtr.sourcePrimaryDB == nil { + return errors.New("source primary connection not initialized; cannot perform move-tables cutover") } - defer pinnedConn.Close() sourceDB := mgtr.migrationContext.DatabaseName sourceTable := mgtr.migrationContext.OriginalTableName delTable := mgtr.migrationContext.GetOldTableName() - renameQuery := fmt.Sprintf("RENAME TABLE %s.%s TO %s.%s", + renameAndCaptureQuery := fmt.Sprintf("rename /* gh-ost */ table %s.%s to %s.%s;\nselect @@global.gtid_executed", sql.EscapeName(sourceDB), sql.EscapeName(sourceTable), sql.EscapeName(sourceDB), sql.EscapeName(delTable)) - mgtr.migrationContext.Log.Infof("T1: renaming source table: %s", renameQuery) - if _, err := pinnedConn.ExecContext(cutOverCtx, renameQuery); err != nil { - return fmt.Errorf("RENAME failed: %w", err) - } - // The source `__del` table now exists and is the rollback handle. Mark the - // rename as done so any later failure emits the rollback hint (and never - // drops `__del`). - atomic.StoreInt64(&mgtr.migrationContext.MoveTablesSourceRenamedFlag, 1) + mgtr.migrationContext.Log.Infof("T1+T2: renaming source table and capturing drain GTID: %s", renameAndCaptureQuery) - // ----- T2: capture @@gtid_executed on the SAME connection as T1 ----- // @@GLOBAL scope is explicit so the intent is unambiguous in the SQL itself. // Design: https://github.com/github/gh-ost-tablemove-poc/blob/9dc6df75c4c88ff473906a497836c7518f5614ec/design/coop_cutover.md#32-correctness-verification-for-p4 - var drainGTIDStr string - if err := pinnedConn.QueryRowContext(cutOverCtx, "select @@global.gtid_executed").Scan(&drainGTIDStr); err != nil { - return fmt.Errorf("drain GTID capture failed: %w", err) + drainGTIDStr, err := func() (string, error) { + rows, err := mgtr.sourcePrimaryDB.QueryContext(cutOverCtx, renameAndCaptureQuery) + if err != nil { + // The error surfaces from the first statement (RENAME); the table was + // NOT renamed, so do not set the rollback flag. + return "", err + } + defer rows.Close() + // QueryContext returned without error, meaning the RENAME committed on the + // source primary. The source `__del` table now exists and is the rollback + // handle; mark the rename as done so any later failure emits the rollback + // hint (and never drops `__del`). + atomic.StoreInt64(&mgtr.migrationContext.MoveTablesSourceRenamedFlag, 1) + + // The RENAME produces no row-bearing result set. Advance to the SELECT's + // result set if the driver left us on the column-less RENAME result. + cols, err := rows.Columns() + if err != nil { + return "", err + } + if len(cols) == 0 { + if !rows.NextResultSet() { + if err := rows.Err(); err != nil { + return "", err + } + return "", errors.New("expected result set for @@global.gtid_executed after RENAME") + } + } + if !rows.Next() { + if err := rows.Err(); err != nil { + return "", err + } + return "", errors.New("no row returned for @@global.gtid_executed") + } + var gtid string + if err := rows.Scan(>id); err != nil { + return "", err + } + return gtid, nil + }() + if err != nil { + return fmt.Errorf("source RENAME + drain GTID capture failed: %w", err) } drainGTID, err := mysql.NewGTIDBinlogCoordinates(mysql.FlavorFor(mgtr.migrationContext.InspectorMySQLVersion), drainGTIDStr) if err != nil { @@ -1739,6 +1806,126 @@ func (mgtr *Migrator) initiateServer() (err error) { return nil } +// assertConnectionWritable fails fast if the given connection points at a +// read_only server. Move-tables writes (target table create + INSERTs, +// checkpoint management, the source RENAME + `__del` DROP) all require writable +// primaries; catching this at startup turns a confusing mid-run failure into a +// clear message. super_read_only implies read_only, so a single check covers both. +func assertConnectionWritable(db *gosql.DB, key mysql.InstanceKey, role string) error { + var readOnly bool + if err := db.QueryRow(`select /* gh-ost */ @@global.read_only`).Scan(&readOnly); err != nil { + return fmt.Errorf("failed to check read_only on move-tables %s %+v: %w", role, key, err) + } + if readOnly { + return fmt.Errorf("move-tables %s %+v is read_only; it must be a writable primary", role, key) + } + return nil +} + +// resolveSourcePrimaryConnectionConfig determines the writable source-cluster +// primary for move-tables cutover writes. It reuses the standard gh-ost master +// detection (walking SHOW SLAVE STATUS from the inspector connection), so when +// the source --host is a replica we find its primary, and when --host is itself +// the primary detection returns the inspector config unchanged (graceful +// fallback, no replica required). --assume-master-host forces the primary, +// mirroring the standard non-move-tables path. +func (mgtr *Migrator) resolveSourcePrimaryConnectionConfig(dbVersion string) (*mysql.ConnectionConfig, error) { + if mgtr.migrationContext.AssumeMasterHostname != "" { + key, err := mysql.ParseInstanceKey(mgtr.migrationContext.AssumeMasterHostname) + if err != nil { + return nil, err + } + cfg := mgtr.migrationContext.InspectorConnectionConfig.DuplicateCredentials(*key) + if mgtr.migrationContext.CliMasterUser != "" { + cfg.User = mgtr.migrationContext.CliMasterUser + } + if mgtr.migrationContext.CliMasterPassword != "" { + cfg.Password = mgtr.migrationContext.CliMasterPassword + } + if err := cfg.RegisterTLSConfig(); err != nil { + return nil, err + } + return cfg, nil + } + visitedKeys := mysql.NewInstanceKeyMap() + return mysql.GetMasterConnectionConfigSafe(dbVersion, mgtr.migrationContext.InspectorConnectionConfig, visitedKeys, mgtr.migrationContext.AllowedMasterMaster) +} + +// setupMoveTablesSourcePrimary resolves the source primary connection config, +// opens its DB handle, and asserts it is writable. It is called from +// initiateInspector (normal path) and from the cutover-resume path (which skips +// the inspector); both supply the source MySQL version used for replica-status +// terminology. Safe to call once per run. +func (mgtr *Migrator) setupMoveTablesSourcePrimary(dbVersion string) error { + cfg, err := mgtr.resolveSourcePrimaryConnectionConfig(dbVersion) + if err != nil { + return fmt.Errorf("failed to resolve move-tables source primary: %w", err) + } + mgtr.migrationContext.MoveTables.SourcePrimaryConnectionConfig = cfg + + uri := cfg.GetDBUri(mgtr.migrationContext.DatabaseName) + "&multiStatements=true" + db, _, err := mysql.GetDB(mgtr.migrationContext.Uuid, uri) + if err != nil { + return err + } + // Assign before the writability check so teardown() reclaims the pool even if + // the gate rejects a read_only host. + mgtr.sourcePrimaryDB = db + if err := assertConnectionWritable(db, cfg.Key, "source primary"); err != nil { + return err + } + mgtr.migrationContext.Log.Infof("Move-tables source primary is %+v; source reads use %+v", + cfg.Key, mgtr.migrationContext.InspectorConnectionConfig.Key) + return nil +} + +// validateMoveTablesSourceReadHost stops a move-tables run early when the source +// --host is the cluster primary. The read path (schema inspection, the full row +// copy, and binlog streaming) all run on --host; pointing it at the primary puts +// the copy load on the primary, which is exactly what move-tables aims to avoid. +// We detect this by comparing the resolved source primary against the inspector +// key — when --host is the primary, master detection returns the inspector +// config unchanged, so the keys match. The operator can repoint --host at a +// replica or explicitly opt in with --allow-on-source-primary. +func (mgtr *Migrator) validateMoveTablesSourceReadHost() error { + if mgtr.migrationContext.MoveTables.AllowOnSourcePrimary { + return nil + } + spc := mgtr.migrationContext.MoveTables.SourcePrimaryConnectionConfig + if spc == nil { + return nil + } + if !spc.Key.Equals(&mgtr.migrationContext.InspectorConnectionConfig.Key) { + return nil + } + return fmt.Errorf("move-tables source --host %+v is the cluster primary; reading the full table copy from the primary is the load move-tables is meant to avoid. Point --host at a replica so reads come off the primary, or pass --allow-on-source-primary to proceed against the primary anyway", spc.Key) +} + +// dropSourceOldTable drops the source `__del` rollback handle on the source +// primary. The inspector/streamer source connections may be a read replica, so +// the drop cannot go through them; it must use the writable source-primary handle. +func (mgtr *Migrator) dropSourceOldTable() error { + if mgtr.sourcePrimaryDB == nil { + return errors.New("source primary connection not initialized; cannot drop source __del table") + } + databaseName := mgtr.migrationContext.DatabaseName + tableName := mgtr.migrationContext.GetOldTableName() + query := fmt.Sprintf(`drop /* gh-ost */ table if exists %s.%s`, + sql.EscapeName(databaseName), + sql.EscapeName(tableName), + ) + mgtr.migrationContext.Log.Infof("Dropping source table %s.%s on primary %+v", + sql.EscapeName(databaseName), + sql.EscapeName(tableName), + mgtr.migrationContext.MoveTables.SourcePrimaryConnectionConfig.Key, + ) + if _, err := mgtr.sourcePrimaryDB.Exec(query); err != nil { + return err + } + mgtr.migrationContext.Log.Infof("Source table dropped") + return nil +} + // initiateInspector connects, validates and inspects the "inspector" server. // The "inspector" server is typically a replica; it is where we issue some // queries such as: @@ -1761,6 +1948,17 @@ func (mgtr *Migrator) initiateInspector() (err error) { // Let's get master connection config if mgtr.migrationContext.IsMoveTablesMode() { mgtr.migrationContext.ApplierConnectionConfig = mgtr.migrationContext.MoveTables.ConnectionConfig + // The source --host (inspector) is used for all reads and may be a read + // replica. Detect and connect the source-cluster primary, which the cutover + // RENAME, drain-GTID capture, and source `__del` DROP run against. + if err := mgtr.setupMoveTablesSourcePrimary(mgtr.inspector.dbVersion); err != nil { + return err + } + // Guard the read path: if --host turned out to be the primary itself, stop + // early rather than silently copying the whole table off the primary. + if err := mgtr.validateMoveTablesSourceReadHost(); err != nil { + return err + } } else if mgtr.migrationContext.AssumeMasterHostname == "" { // No forced master host; detect master if mgtr.migrationContext.ApplierConnectionConfig, err = mgtr.inspector.getMasterConnectionConfig(); err != nil { @@ -2687,12 +2885,11 @@ func (mgtr *Migrator) moveTablesFinalCleanup() error { if mgtr.migrationContext.OkToDropTable { // The source `__del` rollback handle only exists after a real cutover, - // never in Noop runs. The streamer owns the live source connection in - // both the normal and cutover-resume paths (its `db` handle uses the - // source config and Close() above only closed the binlog reader), so the - // source-side drop goes through it. + // never in Noop runs. It must be dropped on the source primary: the + // inspector/streamer source connections may point at a read replica, so the + // drop goes through the dedicated source-primary handle. if !mgtr.migrationContext.Noop { - if err := mgtr.retryOperation(mgtr.eventsStreamer.DropSourceOldTable); err != nil { + if err := mgtr.retryOperation(mgtr.dropSourceOldTable); err != nil { return err } } @@ -2759,4 +2956,9 @@ func (mgtr *Migrator) teardown() { mgtr.migrationContext.Log.Infof("Tearing down throttler") mgtr.throttler.Teardown() } + + if mgtr.sourcePrimaryDB != nil { + mgtr.migrationContext.Log.Infof("Tearing down source primary connection") + mgtr.sourcePrimaryDB.Close() + } } diff --git a/go/logic/migrator_move_tables_cleanup_test.go b/go/logic/migrator_move_tables_cleanup_test.go index 31fb3679d..eccfe96b9 100644 --- a/go/logic/migrator_move_tables_cleanup_test.go +++ b/go/logic/migrator_move_tables_cleanup_test.go @@ -69,3 +69,15 @@ func TestLogMoveTablesRollbackHint_EmitsRenameCommand(t *testing.T) { require.True(t, logger.has("-- rename table `source_db`.`_t_del` to `source_db`.`t`"), "must emit the rename command to roll the source table back") } + +// TestMoveTablesDropSourceOldTable_NilSourcePrimaryErrors verifies the source +// `__del` drop fails cleanly (rather than panicking) when the source-primary +// connection was never initialized. The drop must never silently no-op. +func TestMoveTablesDropSourceOldTable_NilSourcePrimaryErrors(t *testing.T) { + m, _ := newCleanupTestMigrator() + + err := m.dropSourceOldTable() + + require.Error(t, err) + require.Contains(t, err.Error(), "source primary connection not initialized") +} diff --git a/go/logic/migrator_move_tables_cutover_test.go b/go/logic/migrator_move_tables_cutover_test.go index a62ba80f5..74a399d08 100644 --- a/go/logic/migrator_move_tables_cutover_test.go +++ b/go/logic/migrator_move_tables_cutover_test.go @@ -24,10 +24,10 @@ import ( // ----------------------------------------------------------------------------- // Pure unit tests - no MySQL. These exercise the orchestration branches that -// run BEFORE T1's RENAME, so they do not require a real inspector.db. Per the -// Option A decision in commit 3's plan, the "RENAME was not attempted" check -// is a proxy assertion: m.inspector is nil, so if T1 were reached the test -// would panic instead of silently passing. +// run BEFORE T1's RENAME, so they do not require a real source-primary DB. The +// "RENAME was not attempted" check is a proxy assertion: m.sourcePrimaryDB is +// nil, so if T1 were reached the test would fail with the source-primary-not- +// initialized error instead of the earlier error it asserts on. // ----------------------------------------------------------------------------- // TestMoveTablesCutOver_NoopShortCircuits maps to the Noop semantics decision @@ -55,8 +55,9 @@ func TestMoveTablesCutOver_NoopShortCircuits(t *testing.T) { // TestMoveTablesCutOver_OnBeforeCutOverHookAbortsBeforeRename maps to T0 in // coop_cutover.md section 1.3 ("non-zero return code aborts cutover"). The "aborts -// BEFORE source DDL" assertion is enforced as a proxy: m.inspector is nil, so -// if T1 RENAME executed via mgtr.inspector.db, this test would panic. +// BEFORE source DDL" assertion is enforced as a proxy: m.sourcePrimaryDB is nil, +// so if T1 RENAME executed it would fail with the source-primary-not-initialized +// error rather than the asserted hook error. func TestMoveTablesCutOver_OnBeforeCutOverHookAbortsBeforeRename(t *testing.T) { var calls []string boom := errors.New("hook says no") @@ -161,7 +162,7 @@ func TestResumeMoveTablesCutOverFromCheckpointAlreadyDrained(t *testing.T) { m := NewMigrator(ctx, "test") m.applier = NewApplier(ctx) - drainGTID, err := mysql.NewGTIDBinlogCoordinates("11111111-1111-1111-1111-111111111111:1-10") + drainGTID, err := mysql.NewGTIDBinlogCoordinates(mysql.MySQLFlavor, "11111111-1111-1111-1111-111111111111:1-10") require.NoError(t, err) chk := &Checkpoint{ @@ -184,6 +185,67 @@ func TestResumeMoveTablesCutOverFromCheckpointAlreadyDrained(t *testing.T) { } } +// TestResolveSourcePrimaryConnectionConfig_AssumeMasterHostnameOverride verifies +// that --assume-master-host forces the move-tables source primary to the given +// host, and that --master-user/--master-password override the inherited source +// credentials. No DB is required: the override branch builds the config purely +// from the inspector config. +func TestResolveSourcePrimaryConnectionConfig_AssumeMasterHostnameOverride(t *testing.T) { + mc := base.NewMigrationContext() + mc.InspectorConnectionConfig.User = "src_user" + mc.InspectorConnectionConfig.Password = "src_pass" + mc.AssumeMasterHostname = "10.0.0.5:3307" + mc.CliMasterUser = "master_user" + mc.CliMasterPassword = "master_pass" + m := NewMigrator(mc, "test") + + cfg, err := m.resolveSourcePrimaryConnectionConfig("8.0.42") + require.NoError(t, err) + require.Equal(t, "10.0.0.5", cfg.Key.Hostname) + require.Equal(t, 3307, cfg.Key.Port) + require.Equal(t, "master_user", cfg.User, "--master-user must override source credentials") + require.Equal(t, "master_pass", cfg.Password, "--master-password must override source credentials") + + // Without explicit master credentials, the forced primary inherits the source + // (inspector) credentials. + mc.CliMasterUser = "" + mc.CliMasterPassword = "" + cfg, err = m.resolveSourcePrimaryConnectionConfig("8.0.42") + require.NoError(t, err) + require.Equal(t, "src_user", cfg.User) + require.Equal(t, "src_pass", cfg.Password) +} + +// TestValidateMoveTablesSourceReadHost verifies the read-path guard: a replica +// source passes, a source that resolves to the primary is blocked with a hint to +// use a replica or --allow-on-source-primary, and the opt-in flag bypasses it. +func TestValidateMoveTablesSourceReadHost(t *testing.T) { + newMigrator := func(srcKey, primaryKey mysql.InstanceKey, allow bool) *Migrator { + mc := base.NewMigrationContext() + mc.MoveTables.TableNames = []string{"t"} + mc.InspectorConnectionConfig.Key = srcKey + mc.MoveTables.SourcePrimaryConnectionConfig = &mysql.ConnectionConfig{Key: primaryKey} + mc.MoveTables.AllowOnSourcePrimary = allow + return NewMigrator(mc, "test") + } + replica := mysql.InstanceKey{Hostname: "replica.example.com", Port: 3306} + primary := mysql.InstanceKey{Hostname: "primary.example.com", Port: 3306} + + t.Run("source is a replica: passes", func(t *testing.T) { + require.NoError(t, newMigrator(replica, primary, false).validateMoveTablesSourceReadHost()) + }) + + t.Run("source is the primary: blocked", func(t *testing.T) { + err := newMigrator(primary, primary, false).validateMoveTablesSourceReadHost() + require.Error(t, err) + require.Contains(t, err.Error(), "--allow-on-source-primary") + }) + + t.Run("source is the primary but opted in: passes", func(t *testing.T) { + require.NoError(t, newMigrator(primary, primary, true).validateMoveTablesSourceReadHost()) + }) +} + // ----------------------------------------------------------------------------- // Integration tests - real MySQL via testcontainers, exercise T1/T2/T3. // @@ -251,8 +313,10 @@ func (s *MoveTablesCutOverSuite) containingDrainGTID() *mysql.GTIDBinlogCoordina } // buildMigrator wires a Migrator with the test container's *sql.DB pinned to -// inspector.db and a fresh Applier. initialCoords may be nil for the drain- -// timeout case. +// inspector.db and a fresh Applier. The cutover RENAME + drain-GTID capture run +// on sourcePrimaryDB (a dedicated handle with multiStatements enabled, since +// T1/T2 are issued as a single multi-statement round trip). initialCoords may be +// nil for the drain-timeout case. func (s *MoveTablesCutOverSuite) buildMigrator(fakeHooks base.Hooks, initialCoords mysql.BinlogCoordinates) (*Migrator, *base.MigrationContext) { ctx := context.Background() connectionConfig, err := getTestConnectionConfig(ctx, s.mysqlContainer) @@ -261,11 +325,17 @@ func (s *MoveTablesCutOverSuite) buildMigrator(fakeHooks base.Hooks, initialCoor mc := newTestMigrationContext() mc.ApplierConnectionConfig = connectionConfig mc.InspectorConnectionConfig = connectionConfig + mc.MoveTables.SourcePrimaryConnectionConfig = connectionConfig mc.SetConnectionConfig("innodb") mc.Hooks = fakeHooks m := NewMigrator(mc, "test") m.inspector = &Inspector{db: s.db, migrationContext: mc} + // The source primary handle needs multiStatements enabled for the consolidated + // T1+T2 (RENAME; SELECT @@global.gtid_executed) round trip. + sourcePrimaryDB, _, err := mysql.GetDB(mc.Uuid, connectionConfig.GetDBUri(testMysqlDatabase)+"&multiStatements=true") + s.Require().NoError(err) + m.sourcePrimaryDB = sourcePrimaryDB m.applier = NewApplier(mc) if initialCoords != nil { m.applier.CurrentCoordinatesMutex.Lock() @@ -275,6 +345,58 @@ func (s *MoveTablesCutOverSuite) buildMigrator(fakeHooks base.Hooks, initialCoor return m, mc } +// TestDropSourceOldTableUsesSourcePrimary verifies the source `__del` rollback +// handle is dropped through the dedicated source-primary connection. In +// production the inspector/streamer source connections may be a read replica, so +// the drop must not route through them. +func (s *MoveTablesCutOverSuite) TestDropSourceOldTableUsesSourcePrimary() { + ctx := context.Background() + _, err := s.db.ExecContext(ctx, fmt.Sprintf("CREATE TABLE %s (id INT PRIMARY KEY)", getTestOldTableName())) + s.Require().NoError(err) + + var calls []string + fakeHooks := &recordingHooks{name: "fake", calls: &calls} + m, _ := s.buildMigrator(fakeHooks, s.containingDrainGTID()) + + s.Require().NoError(m.dropSourceOldTable()) + + var name string + err = s.db.QueryRow(fmt.Sprintf("SHOW TABLES IN %s LIKE '_%s_del'", + testMysqlDatabase, testMysqlTableName)).Scan(&name) + s.Require().ErrorIs(err, gosql.ErrNoRows, "source __del handle must be dropped via the source primary") +} + +// TestResolveSourcePrimaryFallsBackToInspectorWhenNoReplica verifies the +// graceful fallback: when the source --host has no upstream primary (the +// standalone test container), master detection returns the inspector connection +// config, so source reads and cutover writes share the one available host. +func (s *MoveTablesCutOverSuite) TestResolveSourcePrimaryFallsBackToInspectorWhenNoReplica() { + var calls []string + m, mc := s.buildMigrator(&recordingHooks{name: "fake", calls: &calls}, nil) + + cfg, err := m.resolveSourcePrimaryConnectionConfig("8.0.42") + s.Require().NoError(err) + s.Require().Equal(mc.InspectorConnectionConfig.Key.Hostname, cfg.Key.Hostname) + s.Require().Equal(mc.InspectorConnectionConfig.Key.Port, cfg.Key.Port) +} + +// TestAssertConnectionWritableRejectsReadOnly verifies the startup writability +// gate: a writable primary passes, and a read_only server is rejected with a +// clear error. The same helper guards both the source primary and the target. +func (s *MoveTablesCutOverSuite) TestAssertConnectionWritableRejectsReadOnly() { + key := mysql.InstanceKey{Hostname: "test-host", Port: 3306} + s.Require().NoError(assertConnectionWritable(s.db, key, "source primary"), + "a writable primary must pass the gate") + + _, err := s.db.Exec("SET GLOBAL read_only = ON") + s.Require().NoError(err) + defer func() { _, _ = s.db.Exec("SET GLOBAL read_only = OFF") }() + + err = assertConnectionWritable(s.db, key, "source primary") + s.Require().Error(err) + s.Require().Contains(err.Error(), "read_only") +} + // TestHappyPath drives the full T0-T6 protocol against the test container. // Asserts hook ordering (T0 then T5), T4 flag set, and the source-side rename. // Maps to acceptance criterion #8209 "RENAME executes; drain completes; @@ -322,7 +444,7 @@ func (s *MoveTablesCutOverSuite) TestRenameFailurePropagates() { err := m.moveTablesCutOver() s.Require().Error(err) - s.Require().Contains(err.Error(), "RENAME failed") + s.Require().Contains(err.Error(), "source RENAME + drain GTID capture failed") s.Require().Equal(int64(0), atomic.LoadInt64(&mc.CutOverCompleteFlag), "post-state: RENAME failure must leave CutOverCompleteFlag unset") diff --git a/go/logic/streamer.go b/go/logic/streamer.go index e8682d903..7d3d00120 100644 --- a/go/logic/streamer.go +++ b/go/logic/streamer.go @@ -15,7 +15,6 @@ import ( "github.com/github/gh-ost/go/base" "github.com/github/gh-ost/go/binlog" "github.com/github/gh-ost/go/mysql" - "github.com/github/gh-ost/go/sql" "github.com/openark/golib/sqlutils" ) @@ -280,32 +279,6 @@ func (es *EventsStreamer) Close() (err error) { return err } -// DropSourceOldTable drops the source "__del" table in move-tables mode. The -// __del table is the post-cutover rollback handle on the source cluster; it is -// only dropped after a successful run when --ok-to-drop-table is set. -// The applier's dropTable targets the move-tables target cluster, -// so the source-side drop is owned by the streamer: its `db` -// handle uses InspectorConnectionConfig (the source) and stays open in both the -// normal and the cutover-resume paths (Close() only closes the binlog reader, -// not `db`). -func (es *EventsStreamer) DropSourceOldTable() error { - databaseName := es.migrationContext.DatabaseName - tableName := es.migrationContext.GetOldTableName() - query := fmt.Sprintf(`drop /* gh-ost */ table if exists %s.%s`, - sql.EscapeName(databaseName), - sql.EscapeName(tableName), - ) - es.migrationContext.Log.Infof("Dropping source table %s.%s", - sql.EscapeName(databaseName), - sql.EscapeName(tableName), - ) - if _, err := sqlutils.ExecNoPrepare(es.db, query); err != nil { - return err - } - es.migrationContext.Log.Infof("Source table dropped") - return nil -} - func (es *EventsStreamer) Teardown() { es.db.Close() } diff --git a/localtests/move-tables-test.sh b/localtests/move-tables-test.sh index bb9b52daf..e74db7545 100755 --- a/localtests/move-tables-test.sh +++ b/localtests/move-tables-test.sh @@ -181,6 +181,7 @@ build_ghost_command() { --password=opensesame \ --host=$source_replica_host \ --port=$source_replica_port \ + --assume-master-host=${source_master_host}:${source_master_port} \ --database=$database \ --target-user=root \ --target-password=opensesame \ diff --git a/script/move-tables/README.md b/script/move-tables/README.md index 398ec7517..07d1037a7 100644 --- a/script/move-tables/README.md +++ b/script/move-tables/README.md @@ -24,14 +24,9 @@ script/build --cli Run gh-ost to move tables: ```bash -./script/build --cli; ./bin/gh-ost --move-tables=gh_ost_test --host=localhost --port=3307 --user root --password opensesame --database=gh_ost_test_db --target-host=localhost --target-port=3309 --target-user root --target-password opensesame --target-database=gh_ost_test_db --postpone-cut-over-flag-file=/tmp/ghost-move-tables.postpone.flag --execute --verbose --checkpoint --checkpoint-seconds 10 +./script/build --cli; ./bin/gh-ost --move-tables=gh_ost_test --host=localhost --port=3308 --user root --password opensesame --database=gh_ost_test_db --target-host=localhost --target-port=3309 --target-user root --target-password opensesame --target-database=gh_ost_test_db --postpone-cut-over-flag-file=/tmp/ghost-move-tables.postpone.flag --execute --verbose --checkpoint --checkpoint-seconds 10 --initially-drop-socket-file ``` -Note: replicas in this local topology are configured with `read_only=ON` and -`super_read_only=ON`. If you point `--host` at `mysql-source-replica` (3308), -the cutover `RENAME TABLE` step will fail by design. Use source primary (3307) -as the inspected host when you want cutover to rename on source. - Start continuous inserts against the source. ```bash script/move-tables/insert-source-primary-loop diff --git a/script/move-tables/reset b/script/move-tables/reset new file mode 100755 index 000000000..679e7aa78 --- /dev/null +++ b/script/move-tables/reset @@ -0,0 +1,17 @@ +#!/bin/bash + +set -euo pipefail + +GH_OST_ROOT=$(git rev-parse --show-toplevel) +SCRIPT_PATH="${GH_OST_ROOT}/script/move-tables" +DATABASE_NAME="${GH_OST_TEST_DB:-gh_ost_test_db}" + +# Reset source table state regardless of whether cutover renamed it. +${SCRIPT_PATH}/mysql-source-primary -D "${DATABASE_NAME}" -e "DROP TABLE IF EXISTS _gh_ost_test_del, gh_ost_test;" + +# Recreate and seed source table data, same fixture as setup uses. +${SCRIPT_PATH}/mysql-source-primary -D "${DATABASE_NAME}" < "${GH_OST_ROOT}/localtests/move-tables/create.sql" + +${SCRIPT_PATH}/mysql-target-primary -D "${DATABASE_NAME}" -e "DROP TABLE IF EXISTS gh_ost_test, _gh_ost_test_ghk;" + +echo "Reset source and target tables in ${DATABASE_NAME}" \ No newline at end of file diff --git a/script/move-tables/setup b/script/move-tables/setup index f3ff5e0f4..9bb7902cd 100755 --- a/script/move-tables/setup +++ b/script/move-tables/setup @@ -60,9 +60,37 @@ exec-mysql-target-replica() { fi } +# gh-ost runs on the host, outside the docker network, but source replication +# advertises the primary by its compose service name (mysql-source-primary:3307), +# which the host cannot resolve. Map the service names to loopback so gh-ost's +# auto-detected source primary resolves; published ports are 1:1 so the port +# already matches. Idempotent (guarded by a marker) and best-effort: a failure +# here is non-fatal, since operators can add the entries manually or pass +# --assume-master-host=mysql-source-primary:3307. +ensure_host_aliases() { + local marker="# gh-ost move-tables local aliases" + local entry="127.0.0.1 mysql-source-primary mysql-source-replica mysql-target-primary mysql-target-replica $marker" + if grep -qF "$marker" /etc/hosts 2>/dev/null; then + return 0 + fi + echo -n "Adding move-tables host aliases to /etc/hosts (may prompt for sudo)..." + if printf '%s\n' "$entry" | sudo tee -a /etc/hosts >/dev/null 2>&1; then + echo "OK" + else + echo "SKIPPED" + echo " Could not write /etc/hosts. gh-ost auto-detects the source primary as" + echo " 'mysql-source-primary:3307', which the host must resolve. Either add this line" + echo " to /etc/hosts manually:" + echo " $entry" + echo " or run gh-ost with --assume-master-host=mysql-source-primary:3307" + fi +} + setup() { [ -z "$TEST_MYSQL_IMAGE" ] && TEST_MYSQL_IMAGE="mysql:8.0.41" + ensure_host_aliases + echo "Starting MySQL $TEST_MYSQL_IMAGE containers (2 clusters)..." compose_file="$GH_OST_ROOT/localtests/docker-compose-move-tables.yml" MYSQL_SHA2_RSA_KEYS_FLAG="" From cb1cf02a9d19660e59b45f8f416bc8b143b112a5 Mon Sep 17 00:00:00 2001 From: Zach Sierakowski Date: Mon, 22 Jun 2026 17:09:56 +0000 Subject: [PATCH 15/23] move-tables: report target lag accurately Update status reporting and lag measurement for move-table migrations. Refs: #1721 --- go/base/context.go | 106 +++++++++++++----- go/binlog/binlog_entry.go | 7 +- go/binlog/gomysql_reader.go | 1 + go/logic/hooks.go | 12 +- go/logic/migrator.go | 59 ++++++++-- go/logic/progress_snapshot.go | 2 + script/move-tables/README.md | 6 +- script/move-tables/insert-source-primary-loop | 2 +- script/move-tables/reset | 4 +- 9 files changed, 159 insertions(+), 40 deletions(-) diff --git a/go/base/context.go b/go/base/context.go index 32a9f849e..7fa50c428 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -177,31 +177,38 @@ type MigrationContext struct { CutOverType CutOver ReplicaServerId uint - Hostname string - AssumeMasterHostname string - ApplierTimeZone string - ApplierWaitTimeout int64 - TableEngine string - RowsEstimate int64 - RowsDeltaEstimate int64 - UsedRowsEstimateMethod RowsEstimateMethod - HasSuperPrivilege bool - OriginalBinlogFormat string - OriginalBinlogRowImage string - InspectorConnectionConfig *mysql.ConnectionConfig - InspectorMySQLVersion string - ApplierConnectionConfig *mysql.ConnectionConfig - ApplierMySQLVersion string - StartTime time.Time - RowCopyStartTime time.Time - RowCopyEndTime time.Time - LockTablesStartTime time.Time - RenameTablesStartTime time.Time - RenameTablesEndTime time.Time - pointOfInterestTime time.Time - pointOfInterestTimeMutex *sync.Mutex - lastHeartbeatOnChangelogTime time.Time - lastHeartbeatOnChangelogMutex *sync.Mutex + Hostname string + AssumeMasterHostname string + ApplierTimeZone string + ApplierWaitTimeout int64 + TableEngine string + RowsEstimate int64 + RowsDeltaEstimate int64 + UsedRowsEstimateMethod RowsEstimateMethod + HasSuperPrivilege bool + OriginalBinlogFormat string + OriginalBinlogRowImage string + InspectorConnectionConfig *mysql.ConnectionConfig + InspectorMySQLVersion string + ApplierConnectionConfig *mysql.ConnectionConfig + ApplierMySQLVersion string + StartTime time.Time + RowCopyStartTime time.Time + RowCopyEndTime time.Time + LockTablesStartTime time.Time + RenameTablesStartTime time.Time + RenameTablesEndTime time.Time + pointOfInterestTime time.Time + pointOfInterestTimeMutex *sync.Mutex + lastHeartbeatOnChangelogTime time.Time + lastHeartbeatOnChangelogMutex *sync.Mutex + // lastAppliedBinlogEventTime is the binlog-header timestamp of the last event + // applied to the target in move-tables mode. lastBinlogEventStreamedTime is the + // wall-clock time the streamer last delivered an event for the moved table. + // Together they drive the move-tables writer-lag metric (see GetBinlogWriterLag). + lastAppliedBinlogEventTime time.Time + lastBinlogEventStreamedTime time.Time + binlogWriterLagMutex *sync.Mutex CurrentLag int64 currentProgress uint64 etaNanoseonds int64 @@ -364,6 +371,7 @@ func NewMigrationContext() *MigrationContext { configMutex: &sync.Mutex{}, pointOfInterestTimeMutex: &sync.Mutex{}, lastHeartbeatOnChangelogMutex: &sync.Mutex{}, + binlogWriterLagMutex: &sync.Mutex{}, ColumnRenameMap: make(map[string]string), PanicAbort: make(chan error), ctx: ctx, @@ -743,6 +751,54 @@ func (mctx *MigrationContext) GetLastHeartbeatOnChangelogTime() time.Time { return mctx.lastHeartbeatOnChangelogTime } +// UpdateLastAppliedBinlogEventTime records the binlog-header timestamp of the +// last event successfully applied to the target. Used by move-tables mode to +// derive writer lag. +func (mctx *MigrationContext) UpdateLastAppliedBinlogEventTime(t time.Time) { + mctx.binlogWriterLagMutex.Lock() + defer mctx.binlogWriterLagMutex.Unlock() + + mctx.lastAppliedBinlogEventTime = t +} + +// MarkBinlogEventStreamed records that the streamer just delivered an event for +// the moved table. Used to distinguish "falling behind" from "source is idle". +func (mctx *MigrationContext) MarkBinlogEventStreamed() { + mctx.binlogWriterLagMutex.Lock() + defer mctx.binlogWriterLagMutex.Unlock() + + mctx.lastBinlogEventStreamedTime = time.Now() +} + +// BumpBinlogWriterLagIfIdle treats prolonged streamer silence as "caught up": +// if no event has been streamed for the moved table within idleThreshold, the +// last-applied timestamp is advanced to now so writer lag does not climb forever +// while the source is quiet. +func (mctx *MigrationContext) BumpBinlogWriterLagIfIdle(idleThreshold time.Duration) { + mctx.binlogWriterLagMutex.Lock() + defer mctx.binlogWriterLagMutex.Unlock() + + if mctx.lastBinlogEventStreamedTime.IsZero() || time.Since(mctx.lastBinlogEventStreamedTime) >= idleThreshold { + mctx.lastAppliedBinlogEventTime = time.Now() + } +} + +// GetBinlogWriterLag returns now - last applied event timestamp, the move-tables +// writer lag. It returns 0 before any event has been applied. +func (mctx *MigrationContext) GetBinlogWriterLag() time.Duration { + mctx.binlogWriterLagMutex.Lock() + defer mctx.binlogWriterLagMutex.Unlock() + + if mctx.lastAppliedBinlogEventTime.IsZero() { + return 0 + } + lag := time.Since(mctx.lastAppliedBinlogEventTime) + if lag < 0 { + return 0 + } + return lag +} + func (mctx *MigrationContext) SetHeartbeatIntervalMilliseconds(heartbeatIntervalMilliseconds int64) { if heartbeatIntervalMilliseconds < 100 { heartbeatIntervalMilliseconds = 100 diff --git a/go/binlog/binlog_entry.go b/go/binlog/binlog_entry.go index 7620281d2..c1216054c 100644 --- a/go/binlog/binlog_entry.go +++ b/go/binlog/binlog_entry.go @@ -7,6 +7,7 @@ package binlog import ( "fmt" + "time" "github.com/github/gh-ost/go/mysql" ) @@ -14,7 +15,11 @@ import ( // BinlogEntry describes an entry in the binary log type BinlogEntry struct { Coordinates mysql.BinlogCoordinates - DmlEvent *BinlogDMLEvent + // Timestamp is the wall-clock time recorded in the binlog event header of the + // event that produced this entry. It is used in move-tables mode to measure + // writer lag (now - last applied event timestamp). + Timestamp time.Time + DmlEvent *BinlogDMLEvent } // NewBinlogEntryAt creates an empty, ready to go BinlogEntry object diff --git a/go/binlog/gomysql_reader.go b/go/binlog/gomysql_reader.go index e139726fd..ee50b0d1a 100644 --- a/go/binlog/gomysql_reader.go +++ b/go/binlog/gomysql_reader.go @@ -122,6 +122,7 @@ func (gmr *GoMySQLReader) handleRowsEvent(ev *replication.BinlogEvent, rowsEvent continue } binlogEntry := NewBinlogEntryAt(currentCoords) + binlogEntry.Timestamp = time.Unix(int64(ev.Header.Timestamp), 0) binlogEntry.DmlEvent = NewBinlogDMLEvent( string(rowsEvent.Table.Schema), string(rowsEvent.Table.Table), diff --git a/go/logic/hooks.go b/go/logic/hooks.go index ceec4b6b9..2a48a24be 100644 --- a/go/logic/hooks.go +++ b/go/logic/hooks.go @@ -235,7 +235,17 @@ func (he *HooksExecutor) applyEnvironmentVariables(extraVariables ...string) []s env = append(env, fmt.Sprintf("GH_OST_EXECUTING_HOST=%s", he.migrationContext.Hostname)) env = append(env, fmt.Sprintf("GH_OST_TARGET_HOST=%s", he.migrationContext.GetTargetHostname())) env = append(env, fmt.Sprintf("GH_OST_INSPECTED_LAG=%f", he.migrationContext.GetCurrentLagDuration().Seconds())) - env = append(env, fmt.Sprintf("GH_OST_HEARTBEAT_LAG=%f", he.migrationContext.TimeSinceLastHeartbeatOnChangelog().Seconds())) + // In move-tables mode there is no changelog heartbeat; writer lag (now - last + // applied binlog event timestamp) replaces the heartbeat-derived lag. Re-point + // GH_OST_HEARTBEAT_LAG at it so existing hooks keep seeing a meaningful value, + // and also expose it explicitly as GH_OST_BINLOG_WRITER_LAG_SECONDS. + heartbeatLagSeconds := he.migrationContext.TimeSinceLastHeartbeatOnChangelog().Seconds() + binlogWriterLagSeconds := he.migrationContext.GetBinlogWriterLag().Seconds() + if he.migrationContext.IsMoveTablesMode() { + heartbeatLagSeconds = binlogWriterLagSeconds + } + env = append(env, fmt.Sprintf("GH_OST_HEARTBEAT_LAG=%f", heartbeatLagSeconds)) + env = append(env, fmt.Sprintf("GH_OST_BINLOG_WRITER_LAG_SECONDS=%f", binlogWriterLagSeconds)) env = append(env, fmt.Sprintf("GH_OST_PROGRESS=%f", he.migrationContext.GetProgressPct())) env = append(env, fmt.Sprintf("GH_OST_ETA_SECONDS=%d", he.migrationContext.GetETASeconds())) env = append(env, fmt.Sprintf("GH_OST_HOOKS_HINT=%s", he.migrationContext.HooksHintMessage)) diff --git a/go/logic/migrator.go b/go/logic/migrator.go index b904d8fa3..c5a277241 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -56,9 +56,10 @@ type lockProcessedStruct struct { } type applyEventStruct struct { - writeFunc *tableWriteFunc - dmlEvent *binlog.BinlogDMLEvent - coords mysql.BinlogCoordinates + writeFunc *tableWriteFunc + dmlEvent *binlog.BinlogDMLEvent + coords mysql.BinlogCoordinates + eventTimestamp time.Time } func newApplyEventStructByFunc(writeFunc *tableWriteFunc) *applyEventStruct { @@ -67,7 +68,7 @@ func newApplyEventStructByFunc(writeFunc *tableWriteFunc) *applyEventStruct { } func newApplyEventStructByDML(dmlEntry *binlog.BinlogEntry) *applyEventStruct { - result := &applyEventStruct{dmlEvent: dmlEntry.DmlEvent, coords: dmlEntry.Coordinates} + result := &applyEventStruct{dmlEvent: dmlEntry.DmlEvent, coords: dmlEntry.Coordinates, eventTimestamp: dmlEntry.Timestamp} return result } @@ -2255,15 +2256,28 @@ func (mgtr *Migrator) printStatus(rule PrintStatusRule, snap migrationProgressSn if !mgtr.shouldPrintStatus(rule, snap.elapsedSeconds, snap.etaDuration) { return } + // Lag reporting differs by mode. Standard mode reports the inspected replica + // "Lag" (changelog heartbeat) plus "HeartbeatLag". Move-tables mode has no + // changelog heartbeat, and the source-side replication "Lag" is meaningless + // (writes go to the target, so migration-induced replica lag appears on target + // replicas and is handled separately by throttling). The "Lag" field is + // therefore dropped and writer lag (now - last applied binlog event timestamp) + // is reported as "WriterLag" instead. + lagStatus := fmt.Sprintf("Lag: %.2fs, HeartbeatLag: %.2fs", + snap.replicationLagSeconds, + snap.heartbeatLagSeconds, + ) + if mgtr.migrationContext.IsMoveTablesMode() { + lagStatus = fmt.Sprintf("WriterLag: %.2fs", snap.writerLagSeconds) + } - status := fmt.Sprintf("Copy: %d/%d %.1f%%; Applied: %d; Backlog: %d/%d; Time: %+v(total), %+v(copy); streamer: %+v; Lag: %.2fs, HeartbeatLag: %.2fs, State: %s; ETA: %s", + status := fmt.Sprintf("Copy: %d/%d %.1f%%; Applied: %d; Backlog: %d/%d; Time: %+v(total), %+v(copy); streamer: %+v; %s, State: %s; ETA: %s", snap.totalRowsCopied, snap.rowsEstimate, snap.progressPct, snap.dmlApplied, snap.applyEventsBacklog, snap.applyEventsCapacity, base.PrettifyDurationOutput(snap.elapsedTime), base.PrettifyDurationOutput(snap.elapsedRowCopyTime), snap.streamerBinlogPosition, - snap.replicationLagSeconds, - snap.heartbeatLagSeconds, + lagStatus, snap.state, snap.eta, ) @@ -2328,6 +2342,24 @@ func (mgtr *Migrator) initiateStreaming() error { mgtr.migrationContext.SetRecentBinlogCoordinates(mgtr.eventsStreamer.GetCurrentBinlogCoordinates()) } }() + + // In move-tables mode there is no changelog heartbeat. Writer lag is derived + // from binlog-header timestamps of applied events; when the streamer has been + // silent for the heartbeat interval, treat that silence as "caught up" and bump + // the last-applied timestamp to now so lag does not climb forever while idle. + if mgtr.migrationContext.IsMoveTablesMode() { + go func() { + interval := time.Duration(mgtr.migrationContext.HeartbeatIntervalMilliseconds) * time.Millisecond + ticker := time.NewTicker(interval) + defer ticker.Stop() + for range ticker.C { + if atomic.LoadInt64(&mgtr.finishedMigrating) > 0 { + return + } + mgtr.migrationContext.BumpBinlogWriterLagIfIdle(interval) + } + }() + } return nil } @@ -2344,6 +2376,11 @@ func (mgtr *Migrator) addDMLEventsListener() error { mgtr.migrationContext.DatabaseName, originalTableName, func(dmlEntry *binlog.BinlogEntry) error { + // Record that the streamer just delivered an event for the moved table, + // so the idle-bump rule can tell "falling behind" from "source is quiet". + if mgtr.migrationContext.IsMoveTablesMode() { + mgtr.migrationContext.MarkBinlogEventStreamed() + } // Use helper to prevent deadlock if buffer fills and executeWriteFuncs exits // This is critical because this callback blocks the event streamer return base.SendWithContext(mgtr.migrationContext.GetContext(), mgtr.applyEventsQueue, newApplyEventStructByDML(dmlEntry)) @@ -2578,6 +2615,7 @@ func (mgtr *Migrator) onApplyEventStruct(eventStruct *applyEventStruct) error { if eventStruct.dmlEvent != nil { dmlEvents := [](*binlog.BinlogDMLEvent){} dmlEvents = append(dmlEvents, eventStruct.dmlEvent) + lastEventTimestamp := eventStruct.eventTimestamp var nonDmlStructToApply *applyEventStruct availableEvents := len(mgtr.applyEventsQueue) @@ -2595,6 +2633,7 @@ func (mgtr *Migrator) onApplyEventStruct(eventStruct *applyEventStruct) error { break } dmlEvents = append(dmlEvents, additionalStruct.dmlEvent) + lastEventTimestamp = additionalStruct.eventTimestamp } // Create a task to apply the DML event; this will be execute by executeWriteFuncs() var applyEventFunc tableWriteFunc = func() error { @@ -2608,6 +2647,12 @@ func (mgtr *Migrator) onApplyEventStruct(eventStruct *applyEventStruct) error { mgtr.applier.CurrentCoordinates = eventStruct.coords mgtr.applier.CurrentCoordinatesMutex.Unlock() + // In move-tables mode there is no changelog heartbeat; writer lag is derived + // from the binlog-header timestamp of the last event we just applied. + if mgtr.migrationContext.IsMoveTablesMode() && !lastEventTimestamp.IsZero() { + mgtr.migrationContext.UpdateLastAppliedBinlogEventTime(lastEventTimestamp) + } + if nonDmlStructToApply != nil { // We pulled DML events from the queue, and then we hit a non-DML event. Wait! // We need to handle it! diff --git a/go/logic/progress_snapshot.go b/go/logic/progress_snapshot.go index 375532cca..f5aa5af91 100644 --- a/go/logic/progress_snapshot.go +++ b/go/logic/progress_snapshot.go @@ -28,6 +28,7 @@ type migrationProgressSnapshot struct { streamerBinlogPosition string replicationLagSeconds float64 heartbeatLagSeconds float64 + writerLagSeconds float64 } func (mgtr *Migrator) migrationProgressSnapshot() migrationProgressSnapshot { @@ -63,6 +64,7 @@ func (mgtr *Migrator) migrationProgressSnapshot() migrationProgressSnapshot { streamerBinlogPosition: streamerBinlogPosition, replicationLagSeconds: mgtr.migrationContext.GetCurrentLagDuration().Seconds(), heartbeatLagSeconds: mgtr.migrationContext.TimeSinceLastHeartbeatOnChangelog().Seconds(), + writerLagSeconds: mgtr.migrationContext.GetBinlogWriterLag().Seconds(), } } diff --git a/script/move-tables/README.md b/script/move-tables/README.md index 07d1037a7..c582eefc2 100644 --- a/script/move-tables/README.md +++ b/script/move-tables/README.md @@ -7,12 +7,12 @@ script/move-tables/setup Verify data is present in the source cluster. ```bash -script/move-tables/mysql-source-primary -D gh_ost_test_db -e "SELECT * FROM gh_ost_test;" +script/move-tables/mysql-source-primary -D test -e "SELECT * FROM gh_ost_test;" ``` Verify the empty database is present in the target cluster. ```bash -script/move-tables/mysql-target-primary -D gh_ost_test_db -e "SHOW TABLES;" +script/move-tables/mysql-target-primary -D test -e "SHOW TABLES;" ``` ### Testing `gh-ost` @@ -24,7 +24,7 @@ script/build --cli Run gh-ost to move tables: ```bash -./script/build --cli; ./bin/gh-ost --move-tables=gh_ost_test --host=localhost --port=3308 --user root --password opensesame --database=gh_ost_test_db --target-host=localhost --target-port=3309 --target-user root --target-password opensesame --target-database=gh_ost_test_db --postpone-cut-over-flag-file=/tmp/ghost-move-tables.postpone.flag --execute --verbose --checkpoint --checkpoint-seconds 10 --initially-drop-socket-file +./script/build --cli; ./bin/gh-ost --move-tables=gh_ost_test --host=localhost --port=3308 --user root --password opensesame --database=test --target-host=localhost --target-port=3309 --target-user root --target-password opensesame --target-database=test --postpone-cut-over-flag-file=/tmp/ghost-move-tables.postpone.flag --execute --verbose --checkpoint --checkpoint-seconds 10 --initially-drop-socket-file ``` Start continuous inserts against the source. diff --git a/script/move-tables/insert-source-primary-loop b/script/move-tables/insert-source-primary-loop index 490442ec4..1147e55f1 100755 --- a/script/move-tables/insert-source-primary-loop +++ b/script/move-tables/insert-source-primary-loop @@ -13,7 +13,7 @@ start_i="${1:-100000}" delay="${2:-0.2}" rows_per_batch="${3:-1}" i="$start_i" -DATABASE="${DATABASE:-gh_ost_test_db} +DATABASE="${DATABASE:-test}" echo "Starting continuous inserts on source primary. Press Ctrl+C to stop." echo "start_column1=$start_i sleep_seconds=$delay rows_per_batch=$rows_per_batch" diff --git a/script/move-tables/reset b/script/move-tables/reset index 679e7aa78..a9a354869 100755 --- a/script/move-tables/reset +++ b/script/move-tables/reset @@ -4,13 +4,13 @@ set -euo pipefail GH_OST_ROOT=$(git rev-parse --show-toplevel) SCRIPT_PATH="${GH_OST_ROOT}/script/move-tables" -DATABASE_NAME="${GH_OST_TEST_DB:-gh_ost_test_db}" +DATABASE_NAME="${GH_OST_TEST_DB:-test}" # Reset source table state regardless of whether cutover renamed it. ${SCRIPT_PATH}/mysql-source-primary -D "${DATABASE_NAME}" -e "DROP TABLE IF EXISTS _gh_ost_test_del, gh_ost_test;" # Recreate and seed source table data, same fixture as setup uses. -${SCRIPT_PATH}/mysql-source-primary -D "${DATABASE_NAME}" < "${GH_OST_ROOT}/localtests/move-tables/create.sql" +${SCRIPT_PATH}/mysql-source-primary -D "${DATABASE_NAME}" < "${GH_OST_ROOT}/localtests/move-tables/single/create.sql" ${SCRIPT_PATH}/mysql-target-primary -D "${DATABASE_NAME}" -e "DROP TABLE IF EXISTS gh_ost_test, _gh_ost_test_ghk;" From 450e8d7daea6973a07fc272f2695bb4411d40887 Mon Sep 17 00:00:00 2001 From: Chris Kirkland Date: Tue, 23 Jun 2026 14:05:38 -0600 Subject: [PATCH 16/23] tests: cover move-table cutover recovery Add failpoint-driven integration coverage for copy, drain, hook, and cutover recovery. Refs: #1723 --- .gitignore | 4 + go.mod | 1 + go.sum | 2 + go/base/context.go | 35 +- go/cmd/gh-ost/main.go | 7 +- go/logic/migrator.go | 7 + localtests/move-tables-test.sh | 107 ++-- .../create.sql | 34 ++ .../tables.txt | 1 + .../test.sh | 121 ++++ .../create.sql | 34 ++ .../hooks/gh-ost-on-success | 4 + .../tables.txt | 1 + .../test.sh | 149 +++++ .../resume-panic-on-row-copy/create.sql | 34 ++ .../resume-panic-on-row-copy/tables.txt | 1 + .../resume-panic-on-row-copy/test.sh | 117 ++++ .../single-concurrent-writes/on_test.sh | 2 +- .../move-tables/single-with-hooks/create.sql | 34 ++ .../hooks/gh-ost-on-before-cut-over | 14 + .../hooks/gh-ost-on-row-copy-complete | 14 + .../single-with-hooks/hooks/gh-ost-on-success | 16 + .../single-with-hooks/hooks/util.sh | 42 ++ .../move-tables/single-with-hooks/tables.txt | 1 + .../move-tables/single-with-hooks/test.sh | 67 +++ .../github.com/pingcap/failpoint/.codecov.yml | 39 ++ .../github.com/pingcap/failpoint/.gitignore | 27 + .../pingcap/failpoint/CONTRIBUTING.md | 94 +++ vendor/github.com/pingcap/failpoint/LICENSE | 201 +++++++ .../pingcap/failpoint/MAINTAINERS.md | 12 + vendor/github.com/pingcap/failpoint/Makefile | 78 +++ vendor/github.com/pingcap/failpoint/README.md | 557 ++++++++++++++++++ .../github.com/pingcap/failpoint/failpoint.go | 150 +++++ .../pingcap/failpoint/failpoints.go | 324 ++++++++++ vendor/github.com/pingcap/failpoint/http.go | 110 ++++ vendor/github.com/pingcap/failpoint/marker.go | 100 ++++ vendor/github.com/pingcap/failpoint/terms.go | 376 ++++++++++++ vendor/modules.txt | 3 + 38 files changed, 2882 insertions(+), 38 deletions(-) create mode 100644 localtests/move-tables/resume-panic-before-drain-complete/create.sql create mode 100644 localtests/move-tables/resume-panic-before-drain-complete/tables.txt create mode 100644 localtests/move-tables/resume-panic-before-drain-complete/test.sh create mode 100644 localtests/move-tables/resume-panic-before-on-success-hook/create.sql create mode 100755 localtests/move-tables/resume-panic-before-on-success-hook/hooks/gh-ost-on-success create mode 100644 localtests/move-tables/resume-panic-before-on-success-hook/tables.txt create mode 100644 localtests/move-tables/resume-panic-before-on-success-hook/test.sh create mode 100644 localtests/move-tables/resume-panic-on-row-copy/create.sql create mode 100644 localtests/move-tables/resume-panic-on-row-copy/tables.txt create mode 100644 localtests/move-tables/resume-panic-on-row-copy/test.sh create mode 100644 localtests/move-tables/single-with-hooks/create.sql create mode 100755 localtests/move-tables/single-with-hooks/hooks/gh-ost-on-before-cut-over create mode 100755 localtests/move-tables/single-with-hooks/hooks/gh-ost-on-row-copy-complete create mode 100755 localtests/move-tables/single-with-hooks/hooks/gh-ost-on-success create mode 100755 localtests/move-tables/single-with-hooks/hooks/util.sh create mode 100644 localtests/move-tables/single-with-hooks/tables.txt create mode 100644 localtests/move-tables/single-with-hooks/test.sh create mode 100644 vendor/github.com/pingcap/failpoint/.codecov.yml create mode 100644 vendor/github.com/pingcap/failpoint/.gitignore create mode 100644 vendor/github.com/pingcap/failpoint/CONTRIBUTING.md create mode 100644 vendor/github.com/pingcap/failpoint/LICENSE create mode 100644 vendor/github.com/pingcap/failpoint/MAINTAINERS.md create mode 100644 vendor/github.com/pingcap/failpoint/Makefile create mode 100644 vendor/github.com/pingcap/failpoint/README.md create mode 100644 vendor/github.com/pingcap/failpoint/failpoint.go create mode 100644 vendor/github.com/pingcap/failpoint/failpoints.go create mode 100644 vendor/github.com/pingcap/failpoint/http.go create mode 100644 vendor/github.com/pingcap/failpoint/marker.go create mode 100644 vendor/github.com/pingcap/failpoint/terms.go diff --git a/.gitignore b/.gitignore index 5b4676a24..bdc5b754d 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,7 @@ /.vendor/ .idea/ *.tmp + +tools/ +*__failpoint_binding__.go +*.go__failpoint_stash__ diff --git a/go.mod b/go.mod index af30097c3..52121f925 100644 --- a/go.mod +++ b/go.mod @@ -56,6 +56,7 @@ require ( github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect github.com/pingcap/errors v0.11.5-0.20260310054046-9c8b3586e4b2 // indirect + github.com/pingcap/failpoint v0.0.0-20260521055755-e7642935314f // indirect github.com/pingcap/log v1.1.1-0.20260227082333-572e590d08f1 // indirect github.com/pingcap/tidb/pkg/parser v0.0.0-20260504140133-511dba1dbe17 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect diff --git a/go.sum b/go.sum index 709d499e2..c4df33263 100644 --- a/go.sum +++ b/go.sum @@ -101,6 +101,8 @@ github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJw github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= github.com/pingcap/errors v0.11.5-0.20260310054046-9c8b3586e4b2 h1:cLgCk5mwDG9lDH+dPK8TmEliTjyGJwwKN0qevWAl8IY= github.com/pingcap/errors v0.11.5-0.20260310054046-9c8b3586e4b2/go.mod h1:ktAJCA9lxrHHjVyVl2pKJFvzBnq2eZbb+CUOjBRPlXo= +github.com/pingcap/failpoint v0.0.0-20260521055755-e7642935314f h1:cDo4qNgaQc2POMWTXjNrMA7yySdIF/d1AaW8kOA7qOs= +github.com/pingcap/failpoint v0.0.0-20260521055755-e7642935314f/go.mod h1:jimwlLpI/XtwQdlZML15HS+j4rirvwZM0GLY07wwgOo= github.com/pingcap/log v1.1.1-0.20260227082333-572e590d08f1 h1:A2bEfgSb7hLwR9mxDszgGKweF+xY9YoTDG+8RjdFjDE= github.com/pingcap/log v1.1.1-0.20260227082333-572e590d08f1/go.mod h1:pxfz2oJfAuhwrb3/rcLqD//GS/5gRP4gD022iP3cEO0= github.com/pingcap/tidb/pkg/parser v0.0.0-20260504140133-511dba1dbe17 h1:cfAVPis6GP6lxQgm1WGaNGi4rVXTB4KDvYf96LjqRCM= diff --git a/go/base/context.go b/go/base/context.go index 7fa50c428..97f45cd88 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -22,9 +22,10 @@ import ( "github.com/github/gh-ost/go/metrics" "github.com/github/gh-ost/go/mysql" "github.com/github/gh-ost/go/sql" - "github.com/openark/golib/log" "github.com/go-ini/ini" + "github.com/openark/golib/log" + "github.com/pingcap/failpoint" ) // RowsEstimateMethod is the type of row number estimation @@ -318,6 +319,8 @@ type MigrationContext struct { DrainGTID mysql.BinlogCoordinates // Source @@gtid_executed captured immediately after the source RENAME TABLE; the applier drains until it reaches this coordinate (move-tables only). } + UnsafeFailPointsEnabled bool + Log Logger } @@ -1212,3 +1215,33 @@ func SendWithContext[T any](ctx context.Context, ch chan<- T, val T) error { return ctx.Err() } } + +type failPointOpts struct { + wait time.Duration +} + +type FailPointOpt func(*failPointOpts) + +// WithFailPointWait sets the time for a fail point to wait before exiting. +func WithFailPointWait(wait time.Duration) FailPointOpt { + return func(opts *failPointOpts) { + opts.wait = wait + } +} + +func (mctx *MigrationContext) NewFailPoint(name string, opts ...FailPointOpt) { + if mctx.UnsafeFailPointsEnabled { + var fpo failPointOpts + for _, opt := range opts { + opt(&fpo) + } + + failpoint.Inject(name, func(_ failpoint.Value) { + mctx.Log.Debugf("[TEST] Encountered fail point: '%s'", name) + if fpo.wait > 0 { + time.Sleep(fpo.wait) + } + panic(fmt.Sprintf("[TEST] Encountered fail point: '%s'", name)) + }) + } +} diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index a6525fbe1..bc1455323 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -198,6 +198,9 @@ func main() { flag.StringVar(&migrationContext.MoveTables.TargetDatabase, "target-database", "", "Target MySQL database name for --move-tables mode. If not provided, uses the same database name as the source connection") flag.BoolVar(&migrationContext.MoveTables.AllowOnSourcePrimary, "allow-on-source-primary", false, "allow --move-tables to read (schema, row copy, binlog) from the source cluster's primary. By default gh-ost stops if --host is the primary; prefer pointing --host at a replica to spare the primary the copy load.") + // unsafe fail points, for integration testing purposes + flag.BoolVar(&migrationContext.UnsafeFailPointsEnabled, "unsafe-fail-points-enabled", false, "UNSAFE: Enable fail points for integration testing purposes. Do not use in production.") + flag.CommandLine.SetOutput(os.Stdout) flag.Parse() cutOverLockTimeoutUserSpecified := false @@ -346,7 +349,9 @@ func main() { if *storageEngine == "rocksdb" { migrationContext.Log.Warning("RocksDB storage engine support is experimental") } - if migrationContext.CheckpointIntervalSeconds < 10 { + // ignore low checkpoint intervals in unsafe mode as frequent checkpoints are required to reliably + // reduce test duration + if migrationContext.CheckpointIntervalSeconds < 10 && !migrationContext.UnsafeFailPointsEnabled { migrationContext.Log.Fatalf("--checkpoint-seconds should be >=10") } if migrationContext.CountTableRows && migrationContext.PanicOnWarnings { diff --git a/go/logic/migrator.go b/go/logic/migrator.go index c5a277241..e32d3a792 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -1393,6 +1393,9 @@ func (mgtr *Migrator) moveTablesCutOver() (err error) { } } + mgtr.migrationContext.NewFailPoint("move-tables-panic-before-drain-completion", base.WithFailPointWait(2*time.Second)) + + // ------ T3: draining applier to drain GTID ----------- if err := mgtr.drainMoveTablesCutOver(drainGTID); err != nil { return err } @@ -1410,6 +1413,8 @@ func (mgtr *Migrator) moveTablesCutOver() (err error) { atomic.StoreInt64(&mgtr.migrationContext.CutOverCompleteFlag, 1) mgtr.migrationContext.Log.Debugf("T4: CutOverCompleteFlag set") + mgtr.migrationContext.NewFailPoint("move-tables-panic-before-on-success-hook", base.WithFailPointWait(2*time.Second)) + // ----- T5: on-success hook ----- // Hook unlocks user_rw@target via db-user-management and flips the // write_cutover? feature flag. Standard env vars only — GH_OST_DRAIN_GTID + @@ -2594,6 +2599,8 @@ func (mgtr *Migrator) iterateChunks() error { } return terminateRowIteration(err) } + + mgtr.migrationContext.NewFailPoint("move-tables-panic-after-row-copy", base.WithFailPointWait(2*time.Second)) } } diff --git a/localtests/move-tables-test.sh b/localtests/move-tables-test.sh index e74db7545..a3b2ada46 100755 --- a/localtests/move-tables-test.sh +++ b/localtests/move-tables-test.sh @@ -36,6 +36,7 @@ original_sql_mode= current_gtid_mode= test_timeout=120 test_failure_log_tail_lines=50 +tables_to_migrate=() OPTIND=1 while getopts "b:s:dg" OPTION; do @@ -175,8 +176,13 @@ build_ghost_command() { # Build gh-ost command with all standard options # # expected $1 to be a comma-separated list of tables to move + + # build comma-separated list of tables to move + move_tables_arg=$(IFS=, ; echo "${tables_to_migrate[*]}") + + # NOTE(chriskirkland): fully qualified package name + failpoint name cmd="GOTRACEBACK=crash $ghost_binary \ - --move-tables=$1 \ + --move-tables=$move_tables_arg \ --user=root \ --password=opensesame \ --host=$source_replica_host \ @@ -197,7 +203,13 @@ build_ghost_command() { --stack \ --checkpoint \ --postpone-cut-over-flag-file=$postpone_cutover_flag_file \ + --checkpoint-seconds=1 \ + --unsafe-fail-points-enabled \ --execute ${extra_args[@]}" + + if [ -n "$GO_FAILPOINTS" ]; then + cmd="GO_FAILPOINTS=\"$GO_FAILPOINTS\" $cmd" + fi } print_log_excerpt() { @@ -348,46 +360,47 @@ test_single() { wait $test_pid 2>/dev/null execution_result=$? return $execution_result - fi - # kick off the on_test script for the test. this enables arbitrary custom logic - # concurrent with the gh-ost process. this enables additional scenarios like - # streaming of writes prior to the write cutover. - # - # IMPORTANT: The on-test script is executed in the background and will be killed as soon - # as the gh-ost process terminates. - if [ -f $tests_path/$test_name/on_test.sh ]; then - $tests_path/$test_name/on_test.sh &> /dev/null & - on_test_pid=$! - fi + else - # queue up removal of the postpone cutover flag, otherwise gh-ost hangs on the cutover - ( - sleep 1; - echo "⏩ Sending unpostpone cutover" - rm $postpone_cutover_flag_file &> /dev/null; - ) & + # kick off the on_test script for the test. this enables arbitrary custom logic + # concurrent with the gh-ost process. this enables additional scenarios like + # streaming of writes prior to the write cutover. + # + # IMPORTANT: The on-test script is executed in the background and will be killed as soon + # as the gh-ost process terminates. + if [ -f $tests_path/$test_name/on_test.sh ]; then + $tests_path/$test_name/on_test.sh &> /dev/null & + on_test_pid=$! + fi - # Build and execute gh-ost command - move_tables_arg=$(IFS=, ; echo "${tables_to_migrate[*]}") - build_ghost_command "$move_tables_arg" - echo_dot - echo $cmd >$exec_command_file - echo_dot - timeout $test_timeout bash $exec_command_file >$test_logfile 2>&1 + # queue up removal of the postpone cutover flag, otherwise gh-ost hangs on the cutover + ( + sleep 1; + echo "⏩ Sending unpostpone cutover" + rm $postpone_cutover_flag_file &> /dev/null; + ) & - execution_result=$? + # Build and execute gh-ost command + build_ghost_command + echo_dot + echo $cmd >$exec_command_file + echo_dot + timeout $test_timeout bash $exec_command_file >$test_logfile 2>&1 - if [ -n "$on_test_pid" ]; then - kill -KILL $on_test_pid &>/dev/null - fi + execution_result=$? - # Check for timeout (exit code 124) - if [ $execution_result -eq 124 ]; then - echo - echo "ERROR $test_name execution timed out" - print_log_excerpt - return 1 + if [ -n "$on_test_pid" ]; then + kill -KILL $on_test_pid &>/dev/null + fi + + # Check for timeout (exit code 124) + if [ $execution_result -eq 124 ]; then + echo + echo "ERROR $test_name execution timed out" + print_log_excerpt + return 1 + fi fi if [ -f $tests_path/$test_name/sql_mode ]; then @@ -453,7 +466,29 @@ test_single() { done } +enable_failpoint() { + mkdir -p $repo_root/tools/bin + if [ ! -f $repo_root/tools/bin/failpoint-ctl ]; then + echo "⚙️ Installing failpoint" + GOBIN=$repo_root/tools/bin go install github.com/pingcap/failpoint/failpoint-ctl@v0.0.0-20220801062533-2eaa32854a6c + fi + + echo "⚙️ Enabling failpoint" + $repo_root/tools/bin/failpoint-ctl enable go + + echo "✅ Successfully enabled failpoint" +} + +disable_failpoint() { + echo "⚙️ Disabling failpoint" + $repo_root/tools/bin/failpoint-ctl disable go + + echo "✅ Successfully disabled failpoint" +} + build_binary() { + enable_failpoint + echo "Building" rm -f $default_ghost_binary [ "$ghost_binary" == "" ] && ghost_binary="$default_ghost_binary" @@ -468,6 +503,8 @@ build_binary() { echo "Build failure" exit 1 fi + + disable_failpoint } test_all() { diff --git a/localtests/move-tables/resume-panic-before-drain-complete/create.sql b/localtests/move-tables/resume-panic-before-drain-complete/create.sql new file mode 100644 index 000000000..46e919003 --- /dev/null +++ b/localtests/move-tables/resume-panic-before-drain-complete/create.sql @@ -0,0 +1,34 @@ +drop table if exists gh_ost_test; +create table gh_ost_test ( + id bigint(20) NOT NULL AUTO_INCREMENT, + column1 int(11) NOT NULL, + column2 smallint(5) unsigned NOT NULL, + column3 mediumint(8) unsigned NOT NULL, + column4 tinyint(3) unsigned NOT NULL, + column5 int(11) NOT NULL, + column6 int(11) NOT NULL, + PRIMARY KEY (id), + KEY c12_ix (column1, column2) +) auto_increment=1; + +insert into gh_ost_test values + (NULL, 1001, 100, 500000, 10, 1700000001, 1700000002), + (NULL, 1002, 200, 600000, 20, 1700000003, 1700000004), + (NULL, 1003, 300, 700000, 30, 1700000005, 1700000006), + (NULL, 1004, 400, 800000, 40, 1700000007, 1700000008), + (NULL, 1005, 500, 900000, 50, 1700000009, 1700000010), + (NULL, 1006, 600, 1000000, 60, 1700000011, 1700000012), + (NULL, 1007, 700, 1100000, 70, 1700000013, 1700000014), + (NULL, 1008, 800, 1200000, 80, 1700000015, 1700000016), + (NULL, 1009, 900, 1300000, 90, 1700000017, 1700000018), + (NULL, 1010, 1000, 1400000, 100, 1700000019, 1700000020), + (NULL, 1011, 1100, 1500000, 110, 1700000021, 1700000022), + (NULL, 1012, 1200, 1600000, 120, 1700000023, 1700000024), + (NULL, 1013, 1300, 1700000, 130, 1700000025, 1700000026), + (NULL, 1014, 1400, 1800000, 140, 1700000027, 1700000028), + (NULL, 1015, 1500, 1900000, 150, 1700000029, 1700000030), + (NULL, 1016, 1600, 2000000, 160, 1700000031, 1700000032), + (NULL, 1017, 1700, 2100000, 170, 1700000033, 1700000034), + (NULL, 1018, 1800, 2200000, 180, 1700000035, 1700000036), + (NULL, 1019, 1900, 2300000, 190, 1700000037, 1700000038), + (NULL, 1020, 2000, 2400000, 200, 1700000039, 1700000040); \ No newline at end of file diff --git a/localtests/move-tables/resume-panic-before-drain-complete/tables.txt b/localtests/move-tables/resume-panic-before-drain-complete/tables.txt new file mode 100644 index 000000000..11fc5eef8 --- /dev/null +++ b/localtests/move-tables/resume-panic-before-drain-complete/tables.txt @@ -0,0 +1 @@ +gh_ost_test diff --git a/localtests/move-tables/resume-panic-before-drain-complete/test.sh b/localtests/move-tables/resume-panic-before-drain-complete/test.sh new file mode 100644 index 000000000..39392d53a --- /dev/null +++ b/localtests/move-tables/resume-panic-before-drain-complete/test.sh @@ -0,0 +1,121 @@ + +#!/bin/bash +# Custom test: +# - panic after RENAME (T1) and prior to drain completion (T3), prior to cutover completion +# - validate RENAME and source writes are not possible +# - resume and complete the migration + +set -x + +database=test +table_name=gh_ost_test + +# Build gh-ost command from scratch using framework function (required to inject failpoints) +rm $ghost_binary +build_binary + +###################################################################################################### +### Run #1: Should panic after RENAME (T1) and before drain completion (T3) +###################################################################################################### + +echo "⚙️ Starting migration with failpoint (run #1)..." + +# Build the gh-ost command using the framework function +GO_FAILPOINTS="github.com/github/gh-ost/go/base/move-tables-panic-before-drain-completion=return(true)" build_ghost_command + +# queue up removal of the postpone cutover flag, otherwise gh-ost hangs on the cutover +( + sleep 2; + echo "⏩ Sending unpostpone cutover" + rm $postpone_cutover_flag_file &> /dev/null; +) & + +# Run the gh-ost command, expecting panic on the failpoint the first time +echo_dot +echo > $test_logfile +bash -c "$cmd" >>$test_logfile 2>&1 +ghost_result=$? + +if [ $ghost_result -eq 0 ]; then + echo "ERROR: gh-ost should have failed but did not." + return 1 +fi + +echo -e "\n\n\n\n\n" + +###################################################################################################### +### Intermediate validation +###################################################################################################### + +echo "⚙️ Validating checkpointed state on unexpected exit..." + +# Table was renamed on source +mysql-exec source primary $database -sNe "SELECT 1 FROM ${table_name} LIMIT 1;" +if [ $? -eq 0 ]; then + echo "ERROR: Table '${table_name}' exists on source but show have been renamed." + return 1 +fi + +mysql-exec source primary $database -sNe "SELECT 1 FROM _${table_name}_del LIMIT 1;" +if [ $? -gt 0 ]; then + echo "ERROR: Renamed table '_${table_name}_del' does not exist on source." + return 1 +fi + +# Table not writeable on source +mysql-exec source primary $database -e "INSERT INTO ${table_name} VALUES (NULL, 1021, 2001, 2400001, 201, 1700000041, 1700000041);" +if [ $? -eq 0 ]; then + echo "ERROR: Table '${table_name}' was writeable on source but should not be!." + return 1 +fi + +# Table still exists on target +mysql-exec target primary $database -sNe "SELECT 1 FROM ${table_name} LIMIT 1;" +if [ $? -gt 0 ]; then + echo "ERROR: Table '${table_name}' does not exist on target." + return 1 +fi + +# validate last checkpoint (cutover started and drain GTID are set) +cutover_started=$(mysql-exec target primary $database -Ne "SELECT gh_ost_move_tables_cutover_started FROM _${table_name}_ghk ORDER BY gh_ost_chk_id DESC LIMIT 1;") +if [ "$cutover_started" != 1 ]; then + echo "ERROR: Expected cutover started to be set in last checkpoint." + return 1 +fi + +drain_gtid=$(mysql-exec target primary $database -Ne "SELECT gh_ost_move_tables_drain_gtid FROM _${table_name}_ghk ORDER BY gh_ost_chk_id DESC LIMIT 1;") +if [ "$drain_gtid" == "" ]; then + echo "ERROR: Expected drain GTID to be set in last checkpoint." + return 1 +fi + +echo "✅ Validated checkpointed state on unexpected exit..." + +echo -e "\n\n\n\n\n" + +###################################################################################################### +### Run #2: Resume and complete the migration +###################################################################################################### + +echo "⚙️ Resuming migration (run #2)..." + +# resume migration +build_ghost_command +cmd="$cmd --resume" + +# queue up removal of the postpone cutover flag, otherwise gh-ost hangs on the cutover +( + sleep 2; + echo "⏩ Sending unpostpone cutover" + rm $postpone_cutover_flag_file &> /dev/null; +) & + +bash -c "$cmd" >>$test_logfile 2>&1 +ghost_result=$? + +if [ $ghost_result -ne 0 ]; then + echo "ERROR: gh-ost should have succeeded but did not. ($ghost_result)" + return 1 +fi + +echo -e "\n\n\n\n\n" diff --git a/localtests/move-tables/resume-panic-before-on-success-hook/create.sql b/localtests/move-tables/resume-panic-before-on-success-hook/create.sql new file mode 100644 index 000000000..46e919003 --- /dev/null +++ b/localtests/move-tables/resume-panic-before-on-success-hook/create.sql @@ -0,0 +1,34 @@ +drop table if exists gh_ost_test; +create table gh_ost_test ( + id bigint(20) NOT NULL AUTO_INCREMENT, + column1 int(11) NOT NULL, + column2 smallint(5) unsigned NOT NULL, + column3 mediumint(8) unsigned NOT NULL, + column4 tinyint(3) unsigned NOT NULL, + column5 int(11) NOT NULL, + column6 int(11) NOT NULL, + PRIMARY KEY (id), + KEY c12_ix (column1, column2) +) auto_increment=1; + +insert into gh_ost_test values + (NULL, 1001, 100, 500000, 10, 1700000001, 1700000002), + (NULL, 1002, 200, 600000, 20, 1700000003, 1700000004), + (NULL, 1003, 300, 700000, 30, 1700000005, 1700000006), + (NULL, 1004, 400, 800000, 40, 1700000007, 1700000008), + (NULL, 1005, 500, 900000, 50, 1700000009, 1700000010), + (NULL, 1006, 600, 1000000, 60, 1700000011, 1700000012), + (NULL, 1007, 700, 1100000, 70, 1700000013, 1700000014), + (NULL, 1008, 800, 1200000, 80, 1700000015, 1700000016), + (NULL, 1009, 900, 1300000, 90, 1700000017, 1700000018), + (NULL, 1010, 1000, 1400000, 100, 1700000019, 1700000020), + (NULL, 1011, 1100, 1500000, 110, 1700000021, 1700000022), + (NULL, 1012, 1200, 1600000, 120, 1700000023, 1700000024), + (NULL, 1013, 1300, 1700000, 130, 1700000025, 1700000026), + (NULL, 1014, 1400, 1800000, 140, 1700000027, 1700000028), + (NULL, 1015, 1500, 1900000, 150, 1700000029, 1700000030), + (NULL, 1016, 1600, 2000000, 160, 1700000031, 1700000032), + (NULL, 1017, 1700, 2100000, 170, 1700000033, 1700000034), + (NULL, 1018, 1800, 2200000, 180, 1700000035, 1700000036), + (NULL, 1019, 1900, 2300000, 190, 1700000037, 1700000038), + (NULL, 1020, 2000, 2400000, 200, 1700000039, 1700000040); \ No newline at end of file diff --git a/localtests/move-tables/resume-panic-before-on-success-hook/hooks/gh-ost-on-success b/localtests/move-tables/resume-panic-before-on-success-hook/hooks/gh-ost-on-success new file mode 100755 index 000000000..a2c0b6926 --- /dev/null +++ b/localtests/move-tables/resume-panic-before-on-success-hook/hooks/gh-ost-on-success @@ -0,0 +1,4 @@ +#!/bin/bash + +# touch file to mark completion of on-success hook +touch /tmp/gh-ost-hooks/on-success \ No newline at end of file diff --git a/localtests/move-tables/resume-panic-before-on-success-hook/tables.txt b/localtests/move-tables/resume-panic-before-on-success-hook/tables.txt new file mode 100644 index 000000000..11fc5eef8 --- /dev/null +++ b/localtests/move-tables/resume-panic-before-on-success-hook/tables.txt @@ -0,0 +1 @@ +gh_ost_test diff --git a/localtests/move-tables/resume-panic-before-on-success-hook/test.sh b/localtests/move-tables/resume-panic-before-on-success-hook/test.sh new file mode 100644 index 000000000..14c68bd67 --- /dev/null +++ b/localtests/move-tables/resume-panic-before-on-success-hook/test.sh @@ -0,0 +1,149 @@ + +#!/bin/bash +# Custom test: +# - panic after drain (T4) and prior to on-success (T5), prior to cutover completion +# - validate RENAME and source writes are not possible +# - validate contents of source and target are the same +# - resume and complete the migration + +database=test +table_name=gh_ost_test + +# Build gh-ost command from scratch using framework function (required to inject failpoints) +rm $ghost_binary +build_binary + +# ensure hook files are executable +chmod +x $tests_path/$test_name/hooks/* + +# clean up any existing test hook files +rm -rf /tmp/gh-ost-hooks/ +mkdir -p /tmp/gh-ost-hooks/ + +###################################################################################################### +### Run #1: Should panic after drain (T4) and before on-success (T5) +###################################################################################################### + + +echo "⚙️ Starting migration with failpoint (run #1)..." + +# Build the gh-ost command using the framework function +GO_FAILPOINTS="github.com/github/gh-ost/go/base/move-tables-panic-before-on-success-hook=return(true)" build_ghost_command +cmd="$cmd --hooks-path=$tests_path/$test_name/hooks" + +# queue up removal of the postpone cutover flag, otherwise gh-ost hangs on the cutover +( + sleep 2; + echo "⏩ Sending unpostpone cutover" + rm $postpone_cutover_flag_file &> /dev/null; +) & + +# drive some concurrent writes to the table to exercise queue drain (T3/T4) +( + DATABASE=test script/move-tables/insert-source-primary-loop 100 0.1 10 &>/dev/null & + writes_pid=$! + sleep 3 + kill $writes_pid +) & + +# Run the gh-ost command, expecting panic on the failpoint the first time +echo_dot +echo > $test_logfile +bash -c "$cmd" >>$test_logfile 2>&1 +ghost_result=$? + +if [ $ghost_result -eq 0 ]; then + echo "ERROR: gh-ost should have failed but did not." + return 1 +fi + +echo -e "\n\n\n\n\n" + +###################################################################################################### +### Intermediate validation +###################################################################################################### + +echo "⚙️ Validating checkpointed state on unexpected exit..." + +# Table was renamed on source +mysql-exec source primary $database -sNe "SELECT 1 FROM ${table_name} LIMIT 1;" +if [ $? -eq 0 ]; then + echo "ERROR: Table '${table_name}' exists on source but show have been renamed." + return 1 +fi + +mysql-exec source primary $database -sNe "SELECT 1 FROM _${table_name}_del LIMIT 1;" +if [ $? -gt 0 ]; then + echo "ERROR: Renamed table '_${table_name}_del' does not exist on source." + return 1 +fi + +# Table not writeable on source +mysql-exec source primary $database -sNe "INSERT INTO ${table_name} VALUES (NULL, 1021, 2001, 2400001, 201, 1700000041, 1700000041);" +if [ $? -eq 0 ]; then + echo "ERROR: Table '${table_name}' was writeable on source but should not be!." + return 1 +fi + +# Table still exists on target +mysql-exec target primary $database -sNe "SELECT 1 FROM ${table_name} LIMIT 1;" +if [ $? -gt 0 ]; then + echo "ERROR: Table '${table_name}' does not exist on target." + return 1 +fi + +# contents of table on source and target are the same +source_contents_file=/tmp/gh-ost-test.resume-move-tables-panic-before-on-success-hook-source_contents.txt +target_contents_file=/tmp/gh-ost-test.resume-move-tables-panic-before-on-success-hook-target_contents.txt +mysql-exec source primary $database -sNe "SELECT * FROM _${table_name}_del;" > $source_contents_file +mysql-exec target primary $database -sNe "SELECT * FROM ${table_name};" > $target_contents_file + +if ! diff $source_contents_file $target_contents_file; then + echo "ERROR: Contents of table '${table_name}' are not the same on source and target." + echo "---- DIFF -----" + diff --side-by-side $source_contents_file $target_contents_file + echo "---------------" + return 1 +fi + +# validate on-success hook was not called +if [ -f /tmp/gh-ost-hooks/on-success ]; then + echo "ERROR: on-success hook was called when it should not have been." + return 1 +fi + +echo "✅ Validated checkpointed state on unexpected exit..." + +echo -e "\n\n\n\n\n" + +###################################################################################################### +### Run #2: Resume and complete the migration +###################################################################################################### + +echo "⚙️ Resuming migration (run #2)..." + +# resume migration +build_ghost_command +cmd="$cmd --resume --hooks-path=$tests_path/$test_name/hooks" + +# queue up removal of the postpone cutover flag, otherwise gh-ost hangs on the cutover +( + sleep 2; + echo "⏩ Sending unpostpone cutover" + rm $postpone_cutover_flag_file &> /dev/null; +) & + +bash -c "$cmd" >>$test_logfile 2>&1 +ghost_result=$? + +if [ $ghost_result -ne 0 ]; then + echo "ERROR: gh-ost should have succeeded but did not. ($ghost_result)" + return 1 +fi + +# validate on-success hook was was called +if [ ! -f /tmp/gh-ost-hooks/on-success ]; then + echo "ERROR: on-success hook was not called when it should have been." +fi + +echo -e "\n\n\n\n\n" diff --git a/localtests/move-tables/resume-panic-on-row-copy/create.sql b/localtests/move-tables/resume-panic-on-row-copy/create.sql new file mode 100644 index 000000000..46e919003 --- /dev/null +++ b/localtests/move-tables/resume-panic-on-row-copy/create.sql @@ -0,0 +1,34 @@ +drop table if exists gh_ost_test; +create table gh_ost_test ( + id bigint(20) NOT NULL AUTO_INCREMENT, + column1 int(11) NOT NULL, + column2 smallint(5) unsigned NOT NULL, + column3 mediumint(8) unsigned NOT NULL, + column4 tinyint(3) unsigned NOT NULL, + column5 int(11) NOT NULL, + column6 int(11) NOT NULL, + PRIMARY KEY (id), + KEY c12_ix (column1, column2) +) auto_increment=1; + +insert into gh_ost_test values + (NULL, 1001, 100, 500000, 10, 1700000001, 1700000002), + (NULL, 1002, 200, 600000, 20, 1700000003, 1700000004), + (NULL, 1003, 300, 700000, 30, 1700000005, 1700000006), + (NULL, 1004, 400, 800000, 40, 1700000007, 1700000008), + (NULL, 1005, 500, 900000, 50, 1700000009, 1700000010), + (NULL, 1006, 600, 1000000, 60, 1700000011, 1700000012), + (NULL, 1007, 700, 1100000, 70, 1700000013, 1700000014), + (NULL, 1008, 800, 1200000, 80, 1700000015, 1700000016), + (NULL, 1009, 900, 1300000, 90, 1700000017, 1700000018), + (NULL, 1010, 1000, 1400000, 100, 1700000019, 1700000020), + (NULL, 1011, 1100, 1500000, 110, 1700000021, 1700000022), + (NULL, 1012, 1200, 1600000, 120, 1700000023, 1700000024), + (NULL, 1013, 1300, 1700000, 130, 1700000025, 1700000026), + (NULL, 1014, 1400, 1800000, 140, 1700000027, 1700000028), + (NULL, 1015, 1500, 1900000, 150, 1700000029, 1700000030), + (NULL, 1016, 1600, 2000000, 160, 1700000031, 1700000032), + (NULL, 1017, 1700, 2100000, 170, 1700000033, 1700000034), + (NULL, 1018, 1800, 2200000, 180, 1700000035, 1700000036), + (NULL, 1019, 1900, 2300000, 190, 1700000037, 1700000038), + (NULL, 1020, 2000, 2400000, 200, 1700000039, 1700000040); \ No newline at end of file diff --git a/localtests/move-tables/resume-panic-on-row-copy/tables.txt b/localtests/move-tables/resume-panic-on-row-copy/tables.txt new file mode 100644 index 000000000..11fc5eef8 --- /dev/null +++ b/localtests/move-tables/resume-panic-on-row-copy/tables.txt @@ -0,0 +1 @@ +gh_ost_test diff --git a/localtests/move-tables/resume-panic-on-row-copy/test.sh b/localtests/move-tables/resume-panic-on-row-copy/test.sh new file mode 100644 index 000000000..03df29fca --- /dev/null +++ b/localtests/move-tables/resume-panic-on-row-copy/test.sh @@ -0,0 +1,117 @@ + +#!/bin/bash +# Custom test: +# - panic during row copy stage, prior to cutover +# - resume and complete the migration + +database=test +table_name=gh_ost_test + +# Build gh-ost command from scratch using framework function (required to inject failpoints) +rm $ghost_binary +build_binary + +###################################################################################################### +### Run #1: Should panic after first row copy and migration will not complete +###################################################################################################### + +echo "⚙️ Starting migration with failpoint (run #1)..." + +# Build the gh-ost command using the framework function +GO_FAILPOINTS="github.com/github/gh-ost/go/base/move-tables-panic-after-row-copy=return(true)" build_ghost_command + +# Run the gh-ost command, expecting panic on the failpoint the first time +echo_dot +echo > $test_logfile +bash -c "$cmd" >>$test_logfile 2>&1 +ghost_result=$? + +if [ $ghost_result -eq 0 ]; then + echo "ERROR: gh-ost should have failed but did not." + return 1 +fi + +echo -e "\n\n\n\n\n" + +###################################################################################################### +### Intermediate validation +###################################################################################################### + +echo "⚙️ Validating checkpointed state on unexpected exit..." + +# checkpoint table exists on target and is non-empty +mysql-exec target primary $database -sNe "SELECT 1 FROM _${table_name}_ghk LIMIT 1;" +if [ $? -gt 0 ]; then + echo "ERROR: Checkpoint table is empty or does not exist." + return 1 +fi + +# original table still exists on source +mysql-exec source replica $database -sNe "SELECT 1 FROM ${table_name} LIMIT 1;" +if [ $? -gt 0 ]; then + echo "ERROR: Table '${table_name}' does not exist on the source cluster." + return 1 +fi + +# original table exists on the target +mysql-exec target replica $database -sNe "SELECT 1 FROM ${table_name} LIMIT 1;" +if [ $? -gt 0 ]; then + echo "ERROR: Table '${table_name}' does not exist on the target cluster." + return 1 +fi + +# validate we processed a single row-copy chunk (10 rows) and there are 20 total to process +rows_copied=$(mysql-exec target primary $database -Ne "SELECT gh_ost_rows_copied FROM _${table_name}_ghk ORDER BY gh_ost_chk_id DESC LIMIT 1;") +if [ $rows_copied -ne 10 ]; then + echo "ERROR: Expected last checkpoint to show 10 rows copied." + return 1 +fi + +echo "✅ Validating checkpointed state on unexpected exit..." + +echo -e "\n\n\n\n\n" + +###################################################################################################### +### Run #2: Resume and complete the migration +###################################################################################################### + +echo "⚙️ Resuming migration (run #2)..." + +# resume migration +build_ghost_command +cmd="$cmd --resume" + +# queue up removal of the postpone cutover flag, otherwise gh-ost hangs on the cutover +( + sleep 2; + echo "⏩ Sending unpostpone cutover" + rm $postpone_cutover_flag_file &> /dev/null; +) & + +bash -c "$cmd" >>$test_logfile 2>&1 +ghost_result=$? + +if [ $ghost_result -ne 0 ]; then + echo "ERROR: gh-ost should have succeeded but did not. ($ghost_result)" + return 1 +fi + +echo -e "\n\n\n\n\n" + +###################################################################################################### +### post-migration validation +###################################################################################################### + +echo "⚙️ Validating checkpointed state after resumed migration..." + +# validate we processed the rest of the 20 rows to copy +rows_copied=$(mysql-exec target primary $database -Ne "SELECT gh_ost_rows_copied FROM _${table_name}_ghk ORDER BY gh_ost_chk_id DESC LIMIT 1;") +if [ $rows_copied -ne 20 ]; then + echo "ERROR: Expected last checkpoint to show 20 rows copied." + return 1 +fi + +echo "✅ Validating checkpointed state on resumed migration." + +echo -e "\n\n\n\n\n" + diff --git a/localtests/move-tables/single-concurrent-writes/on_test.sh b/localtests/move-tables/single-concurrent-writes/on_test.sh index 71120eb95..46849a09b 100755 --- a/localtests/move-tables/single-concurrent-writes/on_test.sh +++ b/localtests/move-tables/single-concurrent-writes/on_test.sh @@ -2,5 +2,5 @@ # insert data into the source primary, starting at ID 100 in batches of 10. kill # the process after 5 seconds -DATABASE=test script/move-tables/insert-source-primary-loop 100 0.1 10 & +DATABASE=test script/move-tables/insert-source-primary-loop 100 0.01 100 & sleep 5 && kill $! diff --git a/localtests/move-tables/single-with-hooks/create.sql b/localtests/move-tables/single-with-hooks/create.sql new file mode 100644 index 000000000..46e919003 --- /dev/null +++ b/localtests/move-tables/single-with-hooks/create.sql @@ -0,0 +1,34 @@ +drop table if exists gh_ost_test; +create table gh_ost_test ( + id bigint(20) NOT NULL AUTO_INCREMENT, + column1 int(11) NOT NULL, + column2 smallint(5) unsigned NOT NULL, + column3 mediumint(8) unsigned NOT NULL, + column4 tinyint(3) unsigned NOT NULL, + column5 int(11) NOT NULL, + column6 int(11) NOT NULL, + PRIMARY KEY (id), + KEY c12_ix (column1, column2) +) auto_increment=1; + +insert into gh_ost_test values + (NULL, 1001, 100, 500000, 10, 1700000001, 1700000002), + (NULL, 1002, 200, 600000, 20, 1700000003, 1700000004), + (NULL, 1003, 300, 700000, 30, 1700000005, 1700000006), + (NULL, 1004, 400, 800000, 40, 1700000007, 1700000008), + (NULL, 1005, 500, 900000, 50, 1700000009, 1700000010), + (NULL, 1006, 600, 1000000, 60, 1700000011, 1700000012), + (NULL, 1007, 700, 1100000, 70, 1700000013, 1700000014), + (NULL, 1008, 800, 1200000, 80, 1700000015, 1700000016), + (NULL, 1009, 900, 1300000, 90, 1700000017, 1700000018), + (NULL, 1010, 1000, 1400000, 100, 1700000019, 1700000020), + (NULL, 1011, 1100, 1500000, 110, 1700000021, 1700000022), + (NULL, 1012, 1200, 1600000, 120, 1700000023, 1700000024), + (NULL, 1013, 1300, 1700000, 130, 1700000025, 1700000026), + (NULL, 1014, 1400, 1800000, 140, 1700000027, 1700000028), + (NULL, 1015, 1500, 1900000, 150, 1700000029, 1700000030), + (NULL, 1016, 1600, 2000000, 160, 1700000031, 1700000032), + (NULL, 1017, 1700, 2100000, 170, 1700000033, 1700000034), + (NULL, 1018, 1800, 2200000, 180, 1700000035, 1700000036), + (NULL, 1019, 1900, 2300000, 190, 1700000037, 1700000038), + (NULL, 1020, 2000, 2400000, 200, 1700000039, 1700000040); \ No newline at end of file diff --git a/localtests/move-tables/single-with-hooks/hooks/gh-ost-on-before-cut-over b/localtests/move-tables/single-with-hooks/hooks/gh-ost-on-before-cut-over new file mode 100755 index 000000000..5c7a5b872 --- /dev/null +++ b/localtests/move-tables/single-with-hooks/hooks/gh-ost-on-before-cut-over @@ -0,0 +1,14 @@ +#!/bin/bash + +repo_root=$(git rev-parse --show-toplevel) +source $repo_root/localtests/move-tables/single-with-hooks/hooks/util.sh + +# dump environment variables on dirty exit +trap '[[ $? -eq 0 ]] || dump_env' EXIT + +set -e + +assert_common_envs + +# touch file to mark completion of on-before-cut-over hook +touch /tmp/gh-ost-hooks/on-before-cut-over diff --git a/localtests/move-tables/single-with-hooks/hooks/gh-ost-on-row-copy-complete b/localtests/move-tables/single-with-hooks/hooks/gh-ost-on-row-copy-complete new file mode 100755 index 000000000..0b2ea7d4c --- /dev/null +++ b/localtests/move-tables/single-with-hooks/hooks/gh-ost-on-row-copy-complete @@ -0,0 +1,14 @@ +#!/bin/bash + +repo_root=$(git rev-parse --show-toplevel) +source $repo_root/localtests/move-tables/single-with-hooks/hooks/util.sh + +# dump environment variables on dirty exit +trap '[[ $? -eq 0 ]] || dump_env' EXIT + +set -e + +assert_common_envs + +# touch file to mark completion of on-row-copy-complete hook +touch /tmp/gh-ost-hooks/on-row-copy-complete \ No newline at end of file diff --git a/localtests/move-tables/single-with-hooks/hooks/gh-ost-on-success b/localtests/move-tables/single-with-hooks/hooks/gh-ost-on-success new file mode 100755 index 000000000..2397c8aaa --- /dev/null +++ b/localtests/move-tables/single-with-hooks/hooks/gh-ost-on-success @@ -0,0 +1,16 @@ +#!/bin/bash + +repo_root=$(git rev-parse --show-toplevel) +source $repo_root/localtests/move-tables/single-with-hooks/hooks/util.sh + +# dump environment variables on dirty exit +trap '[[ $? -eq 0 ]] || dump_env' EXIT + +set -e + +assert_common_envs + +assert_env_present "GH_OST_DRAIN_GTID" + +# touch file to mark completion of on-success hook +touch /tmp/gh-ost-hooks/on-success \ No newline at end of file diff --git a/localtests/move-tables/single-with-hooks/hooks/util.sh b/localtests/move-tables/single-with-hooks/hooks/util.sh new file mode 100755 index 000000000..d5820acfc --- /dev/null +++ b/localtests/move-tables/single-with-hooks/hooks/util.sh @@ -0,0 +1,42 @@ +#/bin/bash + +assert_env_equal() { + env_name=$1 + expected=$2 + + if [ "${!env_name}" != "${expected}" ]; then + echo "ERROR: Expected '${expected}' for ${env_name}, but got '${!env_name}'" + exit 1 + fi +} + +assert_env_present() { + env_name=$1 + + echo "checking '${env_name}=${!env_name}'" + if [[ -z "${!env_name}" ]]; then + echo "ERROR: Expected '${env_name}' to be set but not present" + exit 1 + fi +} + +assert_common_envs() { + assert_env_present "GH_OST_TARGET_HOST" + + assert_env_equal "GH_OST_TARGET_DATABASE_NAME" "test" + assert_env_equal "GH_OST_TABLE_NAME" "gh_ost_test" + assert_env_equal "GH_OST_OLD_TABLE_NAME" "_gh_ost_test_del" + assert_env_equal "GH_OST_TARGET_TABLE_NAME" "gh_ost_test" + assert_env_equal "GH_OST_MOVE_TABLES" "true" + assert_env_equal "GH_OST_REVERT" "false" +} + +dump_env() { + echo "-----------------------------------------------------" + echo "----------------- ENVIRONS --------------------------" + echo "-----------------------------------------------------" + env | grep "GH_OST_" + echo "-----------------------------------------------------" + echo "-----------------------------------------------------" + echo "-----------------------------------------------------" +} \ No newline at end of file diff --git a/localtests/move-tables/single-with-hooks/tables.txt b/localtests/move-tables/single-with-hooks/tables.txt new file mode 100644 index 000000000..11fc5eef8 --- /dev/null +++ b/localtests/move-tables/single-with-hooks/tables.txt @@ -0,0 +1 @@ +gh_ost_test diff --git a/localtests/move-tables/single-with-hooks/test.sh b/localtests/move-tables/single-with-hooks/test.sh new file mode 100644 index 000000000..73fbb7771 --- /dev/null +++ b/localtests/move-tables/single-with-hooks/test.sh @@ -0,0 +1,67 @@ + +#!/bin/bash +# Custom test: +# Executes migration with custom hooks (on-row-copy-complete, on-before-cut-over, on-success) +# which are executed at different stages of the migration and validate the environment variables +# expected to be available to the respective hooks. + +database=test +table_name=gh_ost_test + +# Build gh-ost command from scratch using framework function +build_binary + +###################################################################################################### +### Run gh-ost with custom hooks neabled +###################################################################################################### + +echo "⚙️ Running gh-ost with custom hooks..." + +# ensure hook files are executable +chmod +x $tests_path/$test_name/hooks/* + +# clean up any existing test hook files +rm -rf /tmp/gh-ost-hooks/ +mkdir -p /tmp/gh-ost-hooks/ + +# Build the gh-ost command using the framework function +build_ghost_command +cmd="$cmd --hooks-path=$tests_path/$test_name/hooks" + +# queue up removal of the postpone cutover flag, otherwise gh-ost hangs on the cutover +( + sleep 2; + echo "⏩ Sending unpostpone cutover" + rm $postpone_cutover_flag_file &> /dev/null; +) & + +# Run the gh-ost command +echo_dot +echo > $test_logfile +bash -c "$cmd" >>$test_logfile 2>&1 +ghost_result=$? + +if [ $ghost_result -ne 0 ]; then + echo "ERROR: gh-ost failed unexpectedly." + return 1 +fi + +echo "✅ gh-ost move-tables succeeded!" + +echo -e "\n\n\n\n\n" + + +###################################################################################################### +### Validate hook status +###################################################################################################### + +echo "⚙️ Validating hook status after execution..." + +for expected in on-row-copy-complete on-before-cut-over on-success; do + if [ ! -f "/tmp/gh-ost-hooks/$expected" ]; then + echo "ERROR: Expected test hook file '/tmp/gh-ost-hooks/$expected' was not found." + return 1 + fi +done + +echo "✅ Hook status validated successfully." diff --git a/vendor/github.com/pingcap/failpoint/.codecov.yml b/vendor/github.com/pingcap/failpoint/.codecov.yml new file mode 100644 index 000000000..402988545 --- /dev/null +++ b/vendor/github.com/pingcap/failpoint/.codecov.yml @@ -0,0 +1,39 @@ +codecov: + notify: + require_ci_to_pass: yes + +coverage: + precision: 4 + round: down + range: "65...90" + + status: + project: + default: + threshold: 20 #Allow the coverage to drop by threshold%, and posting a success status. + patch: + default: + target: 0% # trial operation + changes: no + +parsers: + gcov: + branch_detection: + conditional: yes + loop: yes + method: no + macro: no + +comment: + layout: "header, diff" + behavior: default + require_changes: no + +ignore: + - "LICENSES" + - "*_test.go" + - "marker.go" # This file only contains empty function stub + - "failpoint-ctl" # Ignore the `failpoint-ctl` command line tool + - ".git" + - "*.yml" + - "*.md" diff --git a/vendor/github.com/pingcap/failpoint/.gitignore b/vendor/github.com/pingcap/failpoint/.gitignore new file mode 100644 index 000000000..b1e5133d6 --- /dev/null +++ b/vendor/github.com/pingcap/failpoint/.gitignore @@ -0,0 +1,27 @@ +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, build with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +bin +coverage.out +.idea/ +*.iml +*.swp +*.txt +*.log +tags +profile.coverprofile +overalls.coverprofile +explain_test +*.fail.go +vendor +.DS_Store diff --git a/vendor/github.com/pingcap/failpoint/CONTRIBUTING.md b/vendor/github.com/pingcap/failpoint/CONTRIBUTING.md new file mode 100644 index 000000000..cfefb0ed5 --- /dev/null +++ b/vendor/github.com/pingcap/failpoint/CONTRIBUTING.md @@ -0,0 +1,94 @@ +# How to contribute + +This document outlines some of the conventions on development workflow, commit +message formatting, contact points and other resources to make it easier to get +your contribution accepted. + +## Getting started + +- Fork the repository on GitHub. +- Read the README.md for build instructions. +- Play with the project, submit bugs, submit patches! + +## Building Failpoint + +Developing Failpoint requires: + +* [Go 1.13](http://golang.org/doc/code.html) +* An internet connection to download the dependencies + +Simply run `make` to build the program. + +```sh +make +``` + +### Running tests + +This project contains unit tests and integration tests with coverage collection. +See [tests/README.md](./tests/README.md) for how to execute and add tests. + +### Updating dependencies + +Failpoint manages dependencies using [Go module](https://github.com/golang/go/wiki/Modules). +To add or update a dependency, either + +* Use the `go mod edit` command to change the dependency, or +* Edit `go.mod` and then run `make update` to update the checksum. + +## Contribution flow + +This is a rough outline of what a contributor's workflow looks like: + +- Create a topic branch from where you want to base your work. This is usually `master`. +- Make commits of logical units and add test case if the change fixes a bug or adds new functionality. +- Run tests and make sure all the tests are passed. +- Make sure your commit messages are in the proper format (see below). +- Push your changes to a topic branch in your fork of the repository. +- Submit a pull request. +- Your PR must receive LGTMs from two maintainers. + +Thanks for your contributions! + +### Code style + +The coding style suggested by the Golang community is used in `failpoint`. +See the [style doc](https://github.com/golang/go/wiki/CodeReviewComments) for details. + +Please follow this style to makeg `failpoint` easy to review, maintain and develop. + +### Format of the Commit Message + +We follow a rough convention for commit messages that is designed to answer two +questions: what changed and why. The subject line should feature the what and +the body of the commit should describe the why. + +``` +restore: add comment for variable declaration + +Improve documentation. +``` + +The format can be described more formally as follows: + +``` +: + + + +