Add MariaDB support for --move-tables mode

.github/workflows/move-tables-tests.yml

@@ -10,11 +10,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        image:
-          # - 'mysql/mysql-server:5.7.41' # metadata locks not supported by default? (https://github.com/github/gh-ost/actions/runs/27841216224/job/82401716601?pr=1714)
-          - 'mysql:8.0.41'
-          - 'mysql:8.4.3'
-          - 'percona/percona-server:8.0.41-32'
+        image: ['mysql:8.0.41','mysql:8.4.3','percona/percona-server:8.0.41-32','mariadb:10.5.29','mariadb:10.6.27','mariadb:10.11.18','mariadb:11.4.12','mariadb:11.8.8']
     env:
       TEST_MYSQL_IMAGE: ${{ matrix.image }}
 

go/logic/applier.go

@@ -1472,9 +1472,25 @@ func (apl *Applier) ReadMoveTablesCutOverCheckpoint() (*Checkpoint, error) {
 		return nil, err
 	}
 	chk.Timestamp = time.Unix(timestamp, 0)
+	sourceGTIDFlavor := ""
+	if apl.migrationContext.UseGTIDs && (coordStr != "" || drainGTIDStr != "") {
+		sourceVersion := apl.migrationContext.InspectorMySQLVersion
+		if sourceVersion == "" {
+			var err error
+			sourceVersion, err = mysql.GetDBVersion(
+				apl.migrationContext.Uuid,
+				apl.migrationContext.InspectorConnectionConfig.GetDBUri(apl.migrationContext.DatabaseName),
+			)
+			if err != nil {
+				return nil, err
+			}
+			apl.migrationContext.InspectorMySQLVersion = sourceVersion
+		}
+		sourceGTIDFlavor = mysql.FlavorFor(sourceVersion)
+	}
 	if coordStr != "" {
 		if apl.migrationContext.UseGTIDs {
-			coords, err := mysql.NewGTIDBinlogCoordinates(mysql.FlavorFor(apl.migrationContext.InspectorMySQLVersion), coordStr)
+			coords, err := mysql.NewGTIDBinlogCoordinates(sourceGTIDFlavor, coordStr)
 			if err != nil {
 				return nil, err
 			}
@@ -1488,7 +1504,7 @@ func (apl *Applier) ReadMoveTablesCutOverCheckpoint() (*Checkpoint, error) {
 		}
 	}
 	if drainGTIDStr != "" {
-		drainGTID, err := mysql.NewGTIDBinlogCoordinates(mysql.FlavorFor(apl.migrationContext.InspectorMySQLVersion), drainGTIDStr)
+		drainGTID, err := mysql.NewGTIDBinlogCoordinates(sourceGTIDFlavor, drainGTIDStr)
 		if err != nil {
 			return nil, err
 		}

go/logic/migrator.go

@@ -1448,12 +1448,14 @@ func (mgtr *Migrator) moveTablesCutOver() (err error) {
 			sql.EscapeName(sourceDB), sql.EscapeName(tableName),
 			sql.EscapeName(sourceDB), sql.EscapeName(delTable)))
 	}
-	renameAndCaptureQuery := fmt.Sprintf("rename /* gh-ost */ table %s;\nselect @@global.gtid_executed",
-		strings.Join(renameClauses, ", "))
+	drainGTIDVariable := moveTablesDrainGTIDVariable(mgtr.migrationContext.InspectorMySQLVersion)
+	renameAndCaptureQuery := fmt.Sprintf("rename /* gh-ost */ table %s;\nselect %s",
+		strings.Join(renameClauses, ", "), drainGTIDVariable)
 	mgtr.migrationContext.Log.Infof("T1+T2: renaming %d source table(s) and capturing drain GTID: %s",
 		len(renameClauses), renameAndCaptureQuery)
 
-	// @@GLOBAL scope is explicit so the intent is unambiguous in the SQL itself.
+	// MySQL exposes @@global.gtid_executed, while MariaDB exposes
+	// @@global.gtid_binlog_pos.
 	// Design: https://github.com/github/gh-ost-tablemove-poc/blob/9dc6df75c4c88ff473906a497836c7518f5614ec/design/coop_cutover.md#32-correctness-verification-for-p4
 	drainGTIDStr, err := func() (string, error) {
 		rows, err := mgtr.sourcePrimaryDB.QueryContext(cutOverCtx, renameAndCaptureQuery)
@@ -1480,14 +1482,14 @@ func (mgtr *Migrator) moveTablesCutOver() (err error) {
 				if err := rows.Err(); err != nil {
 					return "", err
 				}
-				return "", errors.New("expected result set for @@global.gtid_executed after RENAME")
+				return "", fmt.Errorf("expected result set for %s after RENAME", drainGTIDVariable)
 			}
 		}
 		if !rows.Next() {
 			if err := rows.Err(); err != nil {
 				return "", err
 			}
-			return "", errors.New("no row returned for @@global.gtid_executed")
+			return "", fmt.Errorf("no row returned for %s", drainGTIDVariable)
 		}
 		var gtid string
 		if err := rows.Scan(&gtid); err != nil {
@@ -1547,6 +1549,13 @@ func (mgtr *Migrator) moveTablesCutOver() (err error) {
 	return nil
 }
 
+func moveTablesDrainGTIDVariable(mysqlVersion string) string {
+	if mysql.IsMariaDB(mysqlVersion) {
+		return "@@global.gtid_binlog_pos"
+	}
+	return "@@global.gtid_executed"
+}
+
 // ExecOnFailureHook executes the onFailure hook, and this method is provided as the only external
 // hook access point
 func (mgtr *Migrator) ExecOnFailureHook() (err error) {
@@ -2680,15 +2689,18 @@ func (mgtr *Migrator) initiateApplier() error {
 		}
 	}
 
-	// ensure performance_schema.metadata_locks is available.
-	if err := mgtr.applier.StateMetadataLockInstrument(); err != nil {
-		mgtr.migrationContext.Log.Warning("unable to enable metadata lock instrument, see further error details")
-	}
-	if !mgtr.migrationContext.IsOpenMetadataLockInstruments {
-		if !mgtr.migrationContext.SkipMetadataLockCheck {
-			return mgtr.migrationContext.Log.Errorf("bailing out because metadata lock instrument not enabled. Use --skip-metadata-lock-check if you wish to proceed without. See https://github.com/github/gh-ost/pull/1536 for details")
+	if !mgtr.migrationContext.IsMoveTablesMode() {
+		// Standard cut-over uses the atomic magic-lock protocol and verifies its
+		// pending metadata lock before releasing the original-table lock.
+		if err := mgtr.applier.StateMetadataLockInstrument(); err != nil {
+			mgtr.migrationContext.Log.Warning("unable to enable metadata lock instrument, see further error details")
+		}
+		if !mgtr.migrationContext.IsOpenMetadataLockInstruments {
+			if !mgtr.migrationContext.SkipMetadataLockCheck {
+				return mgtr.migrationContext.Log.Errorf("bailing out because metadata lock instrument not enabled. Use --skip-metadata-lock-check if you wish to proceed without. See https://github.com/github/gh-ost/pull/1536 for details")
+			}
+			mgtr.migrationContext.Log.Warning("proceeding without metadata lock check. There is a small chance of data loss if another session accesses the ghost table during cut-over. See https://github.com/github/gh-ost/pull/1536 for details")
 		}
-		mgtr.migrationContext.Log.Warning("proceeding without metadata lock check. There is a small chance of data loss if another session accesses the ghost table during cut-over. See https://github.com/github/gh-ost/pull/1536 for details")
 	}
 
 	if !mgtr.migrationContext.IsMoveTablesMode() {

go/logic/migrator_move_tables_cutover_test.go

@@ -84,6 +84,23 @@ func TestMoveTablesCutOver_OnBeforeCutOverHookAbortsBeforeRename(t *testing.T) {
 		"post-state: only the failing T0 hook fires; no OnSuccess, no OnBeginPostponed")
 }
 
+func TestMoveTablesDrainGTIDVariable(t *testing.T) {
+	testCases := []struct {
+		name    string
+		version string
+		want    string
+	}{
+		{name: "MySQL", version: "8.0.41", want: "@@global.gtid_executed"},
+		{name: "MariaDB", version: "10.11.18-MariaDB", want: "@@global.gtid_binlog_pos"},
+	}
+
+	for _, testCase := range testCases {
+		t.Run(testCase.name, func(t *testing.T) {
+			require.Equal(t, testCase.want, moveTablesDrainGTIDVariable(testCase.version))
+		})
+	}
+}
+
 type onSuccessCheckHooks struct {
 	*recordingHooks
 	onSuccessCheck func() error

localtests/docker-compose-move-tables.yml

@@ -2,7 +2,7 @@ services:
   mysql-source-primary:
     image: $TEST_MYSQL_IMAGE
     container_name: mysql-source-primary
-    command: --server-id=1 --log-bin=mysql-bin --binlog-format=row --gtid-mode=ON --enforce-gtid-consistency=ON --character-set-server=utf8mb4 $MYSQL_NATIVE_PASSWORD_FLAG
+    command: --server-id=1 $MYSQL_PRIMARY_OPTIONS
     environment:
       MYSQL_ROOT_PASSWORD: opensesame
       MYSQL_ROOT_HOST: '%'
@@ -16,7 +16,7 @@ services:
   mysql-source-replica:
     image: $TEST_MYSQL_IMAGE
     container_name: mysql-source-replica
-    command: --server-id=2 --log-bin=mysql-bin --binlog-format=row --gtid-mode=ON --enforce-gtid-consistency=ON --log-slave-updates=ON --character-set-server=utf8mb4 $MYSQL_NATIVE_PASSWORD_FLAG
+    command: --server-id=2 $MYSQL_REPLICA_OPTIONS
     environment:
       MYSQL_ROOT_PASSWORD: opensesame
       MYSQL_ROOT_HOST: '%'
@@ -30,7 +30,7 @@ services:
   mysql-target-primary:
     image: $TEST_MYSQL_IMAGE
     container_name: mysql-target-primary
-    command: --server-id=3 --log-bin=mysql-bin --binlog-format=row --gtid-mode=ON --enforce-gtid-consistency=ON --character-set-server=utf8mb4 $MYSQL_NATIVE_PASSWORD_FLAG
+    command: --server-id=3 $MYSQL_PRIMARY_OPTIONS
     environment:
       MYSQL_ROOT_PASSWORD: opensesame
       MYSQL_ROOT_HOST: '%'
@@ -44,7 +44,7 @@ services:
   mysql-target-replica:
     image: $TEST_MYSQL_IMAGE
     container_name: mysql-target-replica
-    command: --server-id=4 --log-bin=mysql-bin --binlog-format=row --gtid-mode=ON --enforce-gtid-consistency=ON --log-slave-updates=ON --character-set-server=utf8mb4 $MYSQL_NATIVE_PASSWORD_FLAG
+    command: --server-id=4 $MYSQL_REPLICA_OPTIONS
     environment:
       MYSQL_ROOT_PASSWORD: opensesame
       MYSQL_ROOT_HOST: '%'

localtests/move-tables-test.sh

@@ -89,12 +89,18 @@ verify_master_and_replica() {
     original_sql_mode="$(mysql-exec $cluster primary -e "select @@global.sql_mode" -s -s)"
     echo "sql_mode on master is ${original_sql_mode}"
 
-    current_gtid_mode=$(mysql-exec $cluster primary -s -s -e "select @@global.gtid_mode" 2>/dev/null || echo unsupported)
-    current_enforce_gtid_consistency=$(mysql-exec $cluster primary -s -s -e "select @@global.enforce_gtid_consistency" 2>/dev/null || echo unsupported)
-    current_master_server_uuid=$(mysql-exec $cluster primary -s -s -e "select @@global.server_uuid" 2>/dev/null || echo unsupported)
-    current_replica_server_uuid=$(mysql-exec $cluster replica -s -s -e "select @@global.server_uuid" 2>/dev/null || echo unsupported)
-    echo "gtid_mode on master is ${current_gtid_mode} with enforce_gtid_consistency=${current_enforce_gtid_consistency}"
-    echo "server_uuid on master is ${current_master_server_uuid}, replica is ${current_replica_server_uuid}"
+    mysql_version=$(mysql-exec $cluster primary -s -s -e "select @@version")
+    if [[ $mysql_version == *MariaDB* ]]; then
+        current_gtid_mode="ON"
+        echo "MariaDB GTID replication is enabled"
+    else
+        current_gtid_mode=$(mysql-exec $cluster primary -s -s -e "select @@global.gtid_mode" 2>/dev/null || echo unsupported)
+        current_enforce_gtid_consistency=$(mysql-exec $cluster primary -s -s -e "select @@global.enforce_gtid_consistency" 2>/dev/null || echo unsupported)
+        current_master_server_uuid=$(mysql-exec $cluster primary -s -s -e "select @@global.server_uuid" 2>/dev/null || echo unsupported)
+        current_replica_server_uuid=$(mysql-exec $cluster replica -s -s -e "select @@global.server_uuid" 2>/dev/null || echo unsupported)
+        echo "gtid_mode on master is ${current_gtid_mode} with enforce_gtid_consistency=${current_enforce_gtid_consistency}"
+        echo "server_uuid on master is ${current_master_server_uuid}, replica is ${current_replica_server_uuid}"
+    fi
 
     echo "Gracefully sleeping for 3 seconds while replica is setting up..."
     sleep 3

localtests/move-tables/generated-columns/create.sql

@@ -2,9 +2,9 @@ drop table if exists gh_ost_test;
 create table gh_ost_test (
   id int auto_increment,
   a int not null,
-  virtual_sum int as (a + 10) virtual not null,
+  virtual_sum int as (a + 10) virtual,
   b int not null,
-  stored_sum int as (a + b) stored not null,
+  stored_sum int as (a + b) stored,
   json_value json default null,
   virtual_json_value varchar(16) as (
     coalesce(json_unquote(json_extract(json_value, '$.value')), 'direct')

script/docker-gh-ost-move-tables-tests

@@ -54,13 +54,20 @@ setup() {
 
     echo "Starting MySQL $TEST_MYSQL_IMAGE containers..."
     compose_file="$GH_OST_ROOT/localtests/docker-compose-move-tables.yml"
-    MYSQL_SHA2_RSA_KEYS_FLAG=""
+    MYSQL_EXTRA_SERVER_OPTIONS=""
     MYSQL_PASSWORD_HASHING_ALGORITHM="mysql_native_password"
     if [[ $TEST_MYSQL_IMAGE =~ "mysql:8.4" ]]; then
         MYSQL_PASSWORD_HASHING_ALGORITHM="caching_sha2_password"
-        MYSQL_SHA2_RSA_KEYS_FLAG="--caching-sha2-password-auto-generate-rsa-keys=ON"
+        MYSQL_EXTRA_SERVER_OPTIONS="--caching-sha2-password-auto-generate-rsa-keys=ON"
     fi
-    (TEST_MYSQL_IMAGE="$TEST_MYSQL_IMAGE" MYSQL_SHA2_RSA_KEYS_FLAG="$MYSQL_SHA2_RSA_KEYS_FLAG" envsubst <"$compose_file") >"$compose_file.tmp"
+    if [[ $TEST_MYSQL_IMAGE =~ "mariadb" ]]; then
+        MYSQL_PRIMARY_OPTIONS="--log-bin=mariadb-bin --binlog-format=row --gtid-strict-mode=ON --character-set-server=utf8mb4"
+        MYSQL_REPLICA_OPTIONS="$MYSQL_PRIMARY_OPTIONS --log-slave-updates=ON"
+    else
+        MYSQL_PRIMARY_OPTIONS="--log-bin=mysql-bin --binlog-format=row --gtid-mode=ON --enforce-gtid-consistency=ON --character-set-server=utf8mb4 $MYSQL_EXTRA_SERVER_OPTIONS"
+        MYSQL_REPLICA_OPTIONS="$MYSQL_PRIMARY_OPTIONS --log-slave-updates=ON"
+    fi
+    (TEST_MYSQL_IMAGE="$TEST_MYSQL_IMAGE" MYSQL_PRIMARY_OPTIONS="$MYSQL_PRIMARY_OPTIONS" MYSQL_REPLICA_OPTIONS="$MYSQL_REPLICA_OPTIONS" envsubst <"$compose_file") >"$compose_file.tmp"
 
     docker compose -f "$compose_file.tmp" up -d --wait
 
@@ -74,9 +81,14 @@ setup() {
         poll_mysql "$cluster" "replica" || exit 1
 
         echo -n "Setting up replication..."
-        mysql-exec "$cluster" "primary" -e "create user if not exists 'repl'@'%' identified with $MYSQL_PASSWORD_HASHING_ALGORITHM by 'repl';"
+        if [[ $TEST_MYSQL_IMAGE =~ "mariadb" ]]; then
+            mysql-exec "$cluster" "primary" -e "create user if not exists 'repl'@'%' identified by 'repl';"
+            mysql-exec "$cluster" "primary" -e "create user if not exists 'gh-ost'@'%' identified by 'gh-ost';"
+        else
+            mysql-exec "$cluster" "primary" -e "create user if not exists 'repl'@'%' identified with $MYSQL_PASSWORD_HASHING_ALGORITHM by 'repl';"
+            mysql-exec "$cluster" "primary" -e "create user if not exists 'gh-ost'@'%' identified with $MYSQL_PASSWORD_HASHING_ALGORITHM by 'gh-ost';"
+        fi
         mysql-exec "$cluster" "primary" -e "grant replication slave on *.* to 'repl'@'%'; flush privileges;"
-        mysql-exec "$cluster" "primary" -e "create user if not exists 'gh-ost'@'%' identified with $MYSQL_PASSWORD_HASHING_ALGORITHM by 'gh-ost';"
         mysql-exec "$cluster" "primary" -e "grant all on *.* to 'gh-ost'@'%';"
 
         primary_port=3307
@@ -85,7 +97,11 @@ setup() {
         fi
 
         sleep 1
-        if [[ $TEST_MYSQL_IMAGE =~ "mysql:8.4" ]]; then
+        if [[ $TEST_MYSQL_IMAGE =~ "mariadb" ]]; then
+            mysql-exec "$cluster" "replica" -e "reset master; set global gtid_slave_pos=''; set global gtid_domain_id=1;"
+            mysql-exec "$cluster" "replica" -e "change master to master_host='mysql-$cluster-primary', master_port=$primary_port, master_user='repl', master_password='repl', master_use_gtid=slave_pos;"
+            mysql-exec "$cluster" "replica" -e "start slave;"
+        elif [[ $TEST_MYSQL_IMAGE =~ "mysql:8.4" ]]; then
             mysql-exec "$cluster" "replica" -e "change replication source to source_host='mysql-$cluster-primary', source_port=$primary_port, source_user='repl', source_password='repl', source_auto_position=1, source_ssl=1;"
             mysql-exec "$cluster" "replica" -e "start replica;"
         else