Early instant DDL attempt

doc/command-line-flags.md

@@ -62,6 +62,8 @@ MySQL 8.0 supports "instant DDL" for some operations. If an alter statement can
 
 It is not reliable to parse the `ALTER` statement to determine if it is instant or not. This is because the table might be in an older row format, or have some other incompatibility that is difficult to identify.
 
+When `--attempt-instant-ddl` is enabled, `gh-ost` will attempt `ALGORITHM=INSTANT` **early**, right after connecting to the inspector and before creating ghost tables or starting binlog streaming. If instant DDL succeeds, the migration completes immediately without any of the normal setup overhead. This is especially beneficial for large tables where the ghost table creation and binlog streaming setup would otherwise add significant time.
+
 `--attempt-instant-ddl` is disabled by default, but the risks of enabling it are relatively minor: `gh-ost` may need to acquire a metadata lock at the start of the operation. This is not a problem for most scenarios, but it could be a problem for users that start the DDL during a period with long running transactions.
 
 `gh-ost` will automatically fallback to the normal DDL process if the attempt to use instant DDL is unsuccessful.

go/logic/applier.go

@@ -279,36 +279,6 @@ func (this *Applier) ValidateOrDropExistingTables() error {
 	return nil
 }
 
-// AttemptInstantDDL attempts to use instant DDL (from MySQL 8.0, and earlier in Aurora and some others).
-// If successful, the operation is only a meta-data change so a lot of time is saved!
-// The risk of attempting to instant DDL when not supported is that a metadata lock may be acquired.
-// This is minor, since gh-ost will eventually require a metadata lock anyway, but at the cut-over stage.
-// Instant operations include:
-// - Adding a column
-// - Dropping a column
-// - Dropping an index
-// - Extending a VARCHAR column
-// - Adding a virtual generated column
-// It is not reliable to parse the `alter` statement to determine if it is instant or not.
-// This is because the table might be in an older row format, or have some other incompatibility
-// that is difficult to identify.
-func (this *Applier) AttemptInstantDDL() error {
-	query := this.generateInstantDDLQuery()
-	this.migrationContext.Log.Infof("INSTANT DDL query is: %s", query)
-
-	// Reuse cut-over-lock-timeout from regular migration process to reduce risk
-	// in situations where there may be long-running transactions.
-	tableLockTimeoutSeconds := this.migrationContext.CutOverLockTimeoutSeconds * 2
-	this.migrationContext.Log.Infof("Setting LOCK timeout as %d seconds", tableLockTimeoutSeconds)
-	lockTimeoutQuery := fmt.Sprintf(`set /* gh-ost */ session lock_wait_timeout:=%d`, tableLockTimeoutSeconds)
-	if _, err := this.db.Exec(lockTimeoutQuery); err != nil {
-		return err
-	}
-	// We don't need a trx, because for instant DDL the SQL mode doesn't matter.
-	_, err := this.db.Exec(query)
-	return err
-}
-
 // CreateGhostTable creates the ghost table on the applier host
 func (this *Applier) CreateGhostTable() error {
 	query := fmt.Sprintf(`create /* gh-ost */ table %s.%s like %s.%s`,

go/logic/migrator.go

@@ -430,6 +430,23 @@ func (this *Migrator) Migrate() (err error) {
 	if err := this.checkAbort(); err != nil {
 		return err
 	}
+
+	// In MySQL 8.0 (and possibly earlier) some DDL statements can be applied instantly.
+	// Attempt this EARLY, before creating ghost tables or starting binlog streaming,
+	// to avoid unnecessary overhead for large tables when instant DDL is possible.
+	// Skip during resume (the DDL may have already been applied) and noop mode.
+	if this.migrationContext.AttemptInstantDDL && !this.migrationContext.Resume {
+		if this.migrationContext.Noop {
+			this.migrationContext.Log.Debugf("Noop operation; not really attempting instant DDL")
+		} else {
+			if err := this.attemptInstantDDLEarly(); err == nil {
+				return nil
+			} else {
+				this.migrationContext.Log.Infof("ALGORITHM=INSTANT not supported for this operation, proceeding with original algorithm")
+			}
+		}
+	}
+
 	// If we are resuming, we will initiateStreaming later when we know
 	// the binlog coordinates to resume streaming from.
 	// If not resuming, the streamer must be initiated before the applier,
@@ -451,27 +468,6 @@ func (this *Migrator) Migrate() (err error) {
 	if err := this.createFlagFiles(); err != nil {
 		return err
 	}
-	// In MySQL 8.0 (and possibly earlier) some DDL statements can be applied instantly.
-	// Attempt to do this if AttemptInstantDDL is set.
-	if this.migrationContext.AttemptInstantDDL {
-		if this.migrationContext.Noop {
-			this.migrationContext.Log.Debugf("Noop operation; not really attempting instant DDL")
-		} else {
-			this.migrationContext.Log.Infof("Attempting to execute alter with ALGORITHM=INSTANT")
-			if err := this.applier.AttemptInstantDDL(); err == nil {
-				if err := this.finalCleanup(); err != nil {
-					return nil
-				}
-				if err := this.hooksExecutor.onSuccess(); err != nil {
-					return err
-				}
-				this.migrationContext.Log.Infof("Success! table %s.%s migrated instantly", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName))
-				return nil
-			} else {
-				this.migrationContext.Log.Infof("ALGORITHM=INSTANT not supported for this operation, proceeding with original algorithm: %s", err)
-			}
-		}
-	}
 
 	initialLag, _ := this.inspector.getReplicationLag()
 	if !this.migrationContext.Resume {
@@ -1030,6 +1026,55 @@ func (this *Migrator) initiateServer() (err error) {
 	return nil
 }
 
+// attemptInstantDDLEarly attempts to execute the ALTER with ALGORITHM=INSTANT
+// before any ghost table or binlog streaming setup. This avoids the overhead of
+// creating ghost tables, changelog tables, and streaming binlog events for
+// operations that MySQL 8.0+ can execute as instant metadata-only changes.
+// If instant DDL succeeds, the migration is complete. If it fails, the caller
+// should proceed with the normal migration flow.
+func (this *Migrator) attemptInstantDDLEarly() error {
+	this.migrationContext.Log.Infof("Attempting to execute alter with ALGORITHM=INSTANT before full migration setup")
+
+	// Open a temporary connection to the master for the instant DDL attempt.
+	// This avoids initializing the full Applier (ghost table, changelog, etc.).
+	connConfig := this.migrationContext.ApplierConnectionConfig
+	uri := connConfig.GetDBUri(this.migrationContext.DatabaseName)
+	db, _, err := mysql.GetDB(this.migrationContext.Uuid, uri)
+	if err != nil {
+		this.migrationContext.Log.Infof("Could not open connection for instant DDL attempt: %s", err)
+		return err
+	}
+
+	tableLockTimeoutSeconds := this.migrationContext.CutOverLockTimeoutSeconds * 2
+	this.migrationContext.Log.Infof("Setting LOCK timeout as %d seconds for instant DDL attempt", tableLockTimeoutSeconds)
+	lockTimeoutQuery := fmt.Sprintf(`set /* gh-ost */ session lock_wait_timeout:=%d`, tableLockTimeoutSeconds)
+	if _, err := db.Exec(lockTimeoutQuery); err != nil {
+		this.migrationContext.Log.Infof("Could not set lock timeout for instant DDL: %s", err)
+		return err
+	}
+
+	query := fmt.Sprintf(`ALTER /* gh-ost */ TABLE %s.%s %s, ALGORITHM=INSTANT`,
+		sql.EscapeName(this.migrationContext.DatabaseName),
+		sql.EscapeName(this.migrationContext.OriginalTableName),
+		this.migrationContext.AlterStatementOptions,
+	)
+	this.migrationContext.Log.Infof("INSTANT DDL query: %s", query)
+
+	if _, err := db.Exec(query); err != nil {
+		this.migrationContext.Log.Infof("ALGORITHM=INSTANT is not supported for this operation, proceeding with regular algorithm: %s", err)
+		return err
+	}
+
+	if err := this.hooksExecutor.onSuccess(); err != nil {
+		return err
+	}
+	this.migrationContext.Log.Infof("Successfully executed instant DDL on %s.%s (no ghost table was needed)",
+		sql.EscapeName(this.migrationContext.DatabaseName),
+		sql.EscapeName(this.migrationContext.OriginalTableName),
+	)
+	return nil
+}
+
 // initiateInspector connects, validates and inspects the "inspector" server.
 // The "inspector" server is typically a replica; it is where we issue some
 // queries such as:

go/logic/migrator_test.go

@@ -386,6 +386,50 @@ func (suite *MigratorTestSuite) TestMigrateEmpty() {
 	suite.Require().Equal("_testing_del", tableName)
 }
 
+func (suite *MigratorTestSuite) TestMigrateInstantDDLEarly() {
+	ctx := context.Background()
+
+	_, err := suite.db.ExecContext(ctx, fmt.Sprintf("CREATE TABLE %s (id INT PRIMARY KEY, name VARCHAR(64))", getTestTableName()))
+	suite.Require().NoError(err)
+
+	connectionConfig, err := getTestConnectionConfig(ctx, suite.mysqlContainer)
+	suite.Require().NoError(err)
+
+	migrationContext := newTestMigrationContext()
+	migrationContext.ApplierConnectionConfig = connectionConfig
+	migrationContext.InspectorConnectionConfig = connectionConfig
+	migrationContext.SetConnectionConfig("innodb")
+	migrationContext.AttemptInstantDDL = true
+
+	// Adding a column is an instant DDL operation in MySQL 8.0+
+	migrationContext.AlterStatementOptions = "ADD COLUMN instant_col VARCHAR(255)"
+
+	migrator := NewMigrator(migrationContext, "0.0.0")
+
+	err = migrator.Migrate()
+	suite.Require().NoError(err)
+
+	// Verify the new column was added via instant DDL
+	var tableName, createTableSQL string
+	//nolint:execinquery
+	err = suite.db.QueryRow("SHOW CREATE TABLE "+getTestTableName()).Scan(&tableName, &createTableSQL)
+	suite.Require().NoError(err)
+
+	suite.Require().Contains(createTableSQL, "instant_col")
+
+	// Verify that NO ghost table was created (instant DDL should skip ghost table creation)
+	//nolint:execinquery
+	err = suite.db.QueryRow("SHOW TABLES IN test LIKE '_testing_gho'").Scan(&tableName)
+	suite.Require().Error(err, "ghost table should not exist after instant DDL")
+	suite.Require().Equal(gosql.ErrNoRows, err)
+
+	// Verify that NO changelog table was created
+	//nolint:execinquery
+	err = suite.db.QueryRow("SHOW TABLES IN test LIKE '_testing_ghc'").Scan(&tableName)
+	suite.Require().Error(err, "changelog table should not exist after instant DDL")
+	suite.Require().Equal(gosql.ErrNoRows, err)
+}
+
 func (suite *MigratorTestSuite) TestRetryBatchCopyWithHooks() {
 	ctx := context.Background()