MDEV-35732 Failed ALTER TABLE leaves cached table metadata corrupted

A failed ALTER TABLE ... RENAME INDEX ... ALGORITHM=INSTANT changed the
outcome of a subsequent, unrelated ALTER TABLE ... ADD FOREIGN KEY. A
failed statement must have no side effects, but here the second statement
wrongly succeeded where on a fresh table it correctly fails with
ER_DUP_KEYNAME.

Root cause: while rebuilding the key list, mysql_prepare_alter_table()
handled a RENAME INDEX request by clearing HA_GENERATED_KEY in place on
key_info->flags. key_info points into the (possibly cached) TABLE object
that is reused across statements. The ALGORITHM=INSTANT incompatibility is
only detected later, after mysql_prepare_alter_table() has returned, so the
statement fails with ER_ALTER_OPERATION_NOT_SUPPORTED with the cleared flag
never restored. The cached generated FK-support index (fk1) was thus left
permanently marked as user-defined.

That corrupted flag flips the de-duplication tie-break in the next ALTER:
adding FOREIGN KEY ind1 (b) creates a generated support index on column b
that prefix-matches fk1(b). Normally fk1 (generated) is dropped and the new
ind1 survives, colliding by name with the existing user index ind1(a) and
raising ER_DUP_KEYNAME. With fk1 no longer marked generated, the new ind1
is dropped instead, so no name collision is reached and the ADD FOREIGN KEY
silently succeeds.

Fix: do not mutate the cached key_info->flags. Track the "renamed => no
longer generated" decision in a per-key local variable (generated_key),
initialised from the flag, set to false on rename, and passed to the Key
constructor. This preserves the in-statement behaviour while leaving the
cached TABLE metadata untouched, so a failed ALTER has no lingering effect.

All new code of the whole pull request, including one or several files that
are either new files or modified ones, are contributed under the BSD-new
license. I am contributing on behalf of my employer Amazon Web Services, Inc.

Author: LukeYL026

mysql-test/main/alter_table.result

@@ -3225,5 +3225,28 @@ Tp4	CREATE TABLE `Tp4`-ok (
 drop table Another.Tp4;
 drop database Another;
 #
+# MDEV-35732: Failed ALTER TABLE causes inconsistency, changes behavior of the next statement
+#
+# It should be impossible for a failed rename index to affect
+# a subsequent duplicate name check for a foreign key.
+#
+create table t1 (f1 int, f2 int, key(f1), key(f2)) engine=innodb;
+create table t2 (pk int primary key, a int, b int, key ind1(a),
+foreign key fk1 (b) references t1 (f1)) engine=innodb;
+alter table t2 rename index fk1 to fk, algorithm=instant, order by a;
+ERROR 0A000: ALGORITHM=INSTANT is not supported for this operation. Try ALGORITHM=COPY
+alter table t2 add foreign key ind1 (b) references t1 (f2);
+ERROR 42000: Duplicate key name 'ind1'
+drop table t2, t1;
+#
+# It should be impossible to add a foreign key with a duplicate name.
+#
+create table t1 (f1 int, f2 int, key(f1), key(f2)) engine=innodb;
+create table t2 (pk int primary key, a int, b int, key ind1(a),
+foreign key fk1 (b) references t1 (f1)) engine=innodb;
+alter table t2 add foreign key ind1 (b) references t1 (f2);
+ERROR 42000: Duplicate key name 'ind1'
+drop table t2, t1;
+#
 # End of 11.4 tests
 #

mysql-test/main/alter_table.test

@@ -2520,6 +2520,36 @@ show create table Another.Tp4;
 drop table Another.Tp4;
 drop database Another;
 
+--echo #
+--echo # MDEV-35732: Failed ALTER TABLE causes inconsistency, changes behavior of the next statement
+--echo #
+--echo # It should be impossible for a failed rename index to affect
+--echo # a subsequent duplicate name check for a foreign key.
+--echo #
+create table t1 (f1 int, f2 int, key(f1), key(f2)) engine=innodb;
+create table t2 (pk int primary key, a int, b int, key ind1(a),
+                 foreign key fk1 (b) references t1 (f1)) engine=innodb;
+
+--error ER_ALTER_OPERATION_NOT_SUPPORTED
+alter table t2 rename index fk1 to fk, algorithm=instant, order by a;
+
+--error ER_DUP_KEYNAME
+alter table t2 add foreign key ind1 (b) references t1 (f2);
+
+drop table t2, t1;
+
+--echo #
+--echo # It should be impossible to add a foreign key with a duplicate name.
+--echo #
+create table t1 (f1 int, f2 int, key(f1), key(f2)) engine=innodb;
+create table t2 (pk int primary key, a int, b int, key ind1(a),
+                 foreign key fk1 (b) references t1 (f1)) engine=innodb;
+
+--error ER_DUP_KEYNAME
+alter table t2 add foreign key ind1 (b) references t1 (f2);
+
+drop table t2, t1;
+
 --echo #
 --echo # End of 11.4 tests
 --echo #

sql/sql_table.cc

@@ -9024,6 +9024,7 @@ mysql_prepare_alter_table(THD *thd, TABLE *table,
     bool long_hash_key= false;
     if (key_info->flags & HA_INVISIBLE_KEY)
       continue;
+    bool generated_key= key_info->flags & HA_GENERATED_KEY;
     const char *key_name= key_info->name.str;
     const bool primary_key= table->s->primary_key == i;
     const bool explicit_pk= primary_key &&
@@ -9112,7 +9113,7 @@ mysql_prepare_alter_table(THD *thd, TABLE *table,
           dropped by mysql_prepare_create_table() and this will confuse
           code in fill_alter_inplace_info().
         */
-        key_info->flags&= ~HA_GENERATED_KEY;
+        generated_key= false;
         break;
       }
     }
@@ -9312,7 +9313,7 @@ mysql_prepare_alter_table(THD *thd, TABLE *table,
       tmp_name.length= strlen(key_name);
       /* We dont need LONG_UNIQUE_HASH_FIELD flag because it will be autogenerated */
       key= new (thd->mem_root) Key(key_type, &tmp_name, &key_create_info,
-                   key_info->flags & HA_GENERATED_KEY,
+                   generated_key,
                    &key_parts, key_info->option_list, DDL_options());
       key->without_overlaps= key_info->without_overlaps;
       key->period= table->s->period.name;