Add support for index visibility for MySQL v8.0.0+

activerecord/CHANGELOG.md

@@ -1,3 +1,31 @@
+*   Support index visibility for MySQL v8.0.0+
+
+    MySQL 8.0.0 added option to allow indexes to be created or altered to be invisible. This allows the index
+    not to be used without the need to drop it. See https://dev.mysql.com/doc/refman/8.0/en/invisible-indexes.html for more details.
+
+    ActiveRecord now supports this option for MySQL 8.0.0+ for index creation and alteration where the new index option `visible: true/false` can be passed to column and index methods as below:
+
+    ```ruby
+        add_index :users, :email, visible: false
+        alter_index :users, :email, visible: true
+        add_column :users, :dob, :string, index: { visible: false }
+
+        change_table :users do |t|
+          t.index :name, visible: false
+          t.alter_index :dob, visible: true
+          t.column :username, :string, index: { visible: false }
+          t.references :account, index: { visible: false }
+        end
+
+        create_table :users do |t|
+          t.string :name, index: { visible: false }
+          t.string :email
+          t.index :email, visible: false
+        end
+    ```
+
+    *Merve Taner*
+
 *   `ActiveRecord::Coder::JSON` can be instantiated
 
     Options can now be passed to `ActiveRecord::Coder::JSON` when instantiating the coder. This allows:

activerecord/lib/active_record/connection_adapters/abstract/schema_statements.rb

@@ -912,6 +912,17 @@ def rename_column(table_name, column_name, new_column_name)
       # Concurrently adding an index is not supported in a transaction.
       #
       # For more information see the {"Transactional Migrations" section}[rdoc-ref:Migration].
+      #
+      # ====== Creating an invisible index
+      #
+      #   add_index(:developers, :name, visible: false)
+      #
+      # generates:
+      #
+      #   CREATE INDEX index_developers_on_name ON developers (name) INVISIBLE -- MySQL
+      #
+      # Note: only supported by MySQL version 8.0.0 and greater.
+      #
       def add_index(table_name, column_name, **options)
         create_index = build_create_index_definition(table_name, column_name, **options)
         execute schema_creation.accept(create_index)
@@ -1474,8 +1485,12 @@ def update_table_definition(table_name, base) # :nodoc:
         Table.new(table_name, base)
       end
 
+      def valid_index_options
+        [:unique, :length, :order, :opclass, :where, :type, :using, :comment, :algorithm, :include, :nulls_not_distinct]
+      end
+
       def add_index_options(table_name, column_name, name: nil, if_not_exists: false, internal: false, **options) # :nodoc:
-        options.assert_valid_keys(:unique, :length, :order, :opclass, :where, :type, :using, :comment, :algorithm, :include, :nulls_not_distinct)
+        options.assert_valid_keys(valid_index_options)
 
         column_names = index_column_names(column_name)
 
@@ -1484,7 +1499,7 @@ def add_index_options(table_name, column_name, name: nil, if_not_exists: false,
 
         validate_index_length!(table_name, index_name, internal)
 
-        index = IndexDefinition.new(
+        index = create_index_definition(
           table_name, index_name,
           options[:unique],
           column_names,
@@ -1539,6 +1554,13 @@ def change_column_comment(table_name, column_name, comment_or_changes)
         raise NotImplementedError, "#{self.class} does not support changing column comments"
       end
 
+      # Changes the visibility of an index and it is a reversible operation.
+      #
+      #   alter_index(:users, :email, visible: false)
+      def alter_index(table_name, index_name, visible: true)
+        raise NotImplementedError, "#{self.class} does not support altering index visibility"
+      end
+
       def create_schema_dumper(options) # :nodoc:
         SchemaDumper.create(self, options)
       end
@@ -1703,6 +1725,10 @@ def create_table_definition(name, **options)
           TableDefinition.new(self, name, **options)
         end
 
+        def create_index_definition(table_name, name, unique, columns, **options)
+          IndexDefinition.new(table_name, name, unique, columns, **options)
+        end
+
         def create_alter_table(name)
           AlterTable.new create_table_definition(name)
         end

activerecord/lib/active_record/connection_adapters/abstract_adapter.rb

@@ -569,6 +569,11 @@ def supports_nulls_not_distinct?
         false
       end
 
+      def supports_index_visibility?
+        false
+      end
+      alias_method :supports_alter_index?, :supports_index_visibility?
+
       def return_value_after_insert?(column) # :nodoc:
         column.auto_populated?
       end

activerecord/lib/active_record/connection_adapters/abstract_mysql_adapter.rb

@@ -178,6 +178,12 @@ def return_value_after_insert?(column) # :nodoc:
         supports_insert_returning? ? column.auto_populated? : column.auto_increment?
       end
 
+      # See https://dev.mysql.com/doc/refman/8.0/en/invisible-indexes.html for more details.
+      def supports_index_visibility?
+        !mariadb? && database_version >= "8.0.0"
+      end
+      alias_method :supports_alter_index?, :supports_index_visibility?
+
       def get_advisory_lock(lock_name, timeout = 0) # :nodoc:
         query_value("SELECT GET_LOCK(#{quote(lock_name.to_s)}, #{timeout})") == 1
       end
@@ -457,6 +463,20 @@ def build_create_index_definition(table_name, column_name, **options) # :nodoc:
         CreateIndexDefinition.new(index, algorithm)
       end
 
+      # Changes the visibility of an index.
+      #
+      #   alter_index(:users, :email, visible: false)
+      #
+      # Note: only supported by MySQL version 8.0.0 and greater.
+      def alter_index(table_name, index_name, visible:)
+        raise NotImplementedError unless supports_alter_index?
+
+        alter_index_query = <<~SQL
+          ALTER TABLE #{quote_table_name(table_name)} ALTER INDEX #{index_name} #{visible ? "VISIBLE" : "INVISIBLE"}
+        SQL
+        execute(alter_index_query)
+      end
+
       def add_sql_comment!(sql, comment) # :nodoc:
         sql << " COMMENT #{quote(comment)}" if comment.present?
         sql

activerecord/lib/active_record/connection_adapters/mysql/schema_creation.rb

@@ -49,6 +49,7 @@ def visit_IndexDefinition(o, create = false)
             sql << "USING #{o.using}" if o.using
             sql << "ON #{quote_table_name(o.table)}" if create
             sql << "(#{quoted_columns(o)})"
+            sql << "INVISIBLE" if o.invisible?
 
             add_sql_comment!(sql.join(" "), o.comment)
           end

activerecord/lib/active_record/connection_adapters/mysql/schema_definitions.rb

@@ -50,6 +50,32 @@ module ColumnMethods
         deprecate :unsigned_float, :unsigned_decimal, deprecator: ActiveRecord.deprecator
       end
 
+      # = Active Record MySQL Adapter \Index Definition
+      class IndexDefinition < ActiveRecord::ConnectionAdapters::IndexDefinition
+        attr_reader :visible
+
+        def initialize(*args, **kwargs)
+          visible = kwargs.delete(:visible)
+          super
+          @visible = visible.nil? ? true : visible
+        end
+
+        def visible=(value)
+          return if value.nil?
+
+          @visible = value
+        end
+
+        def defined_for?(columns = nil, name: nil, unique: nil, valid: nil, include: nil, nulls_not_distinct: nil, visible: nil, **options)
+          super(columns, name:, unique:, valid:, include:, nulls_not_distinct:, **options) &&
+            (visible.nil? || self.visible == visible)
+        end
+
+        def invisible?
+          !@visible
+        end
+      end
+
       # = Active Record MySQL Adapter \Table Definition
       class TableDefinition < ActiveRecord::ConnectionAdapters::TableDefinition
         include ColumnMethods
@@ -99,6 +125,17 @@ def integer_like_primary_key_type(type, options)
       # = Active Record MySQL Adapter \Table
       class Table < ActiveRecord::ConnectionAdapters::Table
         include ColumnMethods
+
+        # Changes the visibility of an index.
+        #
+        #   t.alter_index(:email, visible: false)
+        #
+        # Note: only supported by MySQL version 8.0.0 and greater.
+        #
+        # See {connection.alter_index}[rdoc-ref:SchemaStatements#alter_index]
+        def alter_index(index_name, visible:)
+          @base.alter_index(name, index_name, visible:)
+        end
       end
     end
   end

activerecord/lib/active_record/connection_adapters/mysql/schema_statements.rb

@@ -30,7 +30,8 @@ def indexes(table_name)
                 orders: {},
                 type: index_type,
                 using: index_using,
-                comment: row["Index_comment"].presence
+                comment: row["Index_comment"].presence,
+                visible: row["Visible"] == "YES",
               ]
             end
 
@@ -63,8 +64,7 @@ def indexes(table_name)
                 columns, order: orders, length: lengths
               ).values.join(", ")
             end
-
-            IndexDefinition.new(*index, **options)
+            MySQL::IndexDefinition.new(*index, **options)
           end
         rescue StatementInvalid => e
           if e.message.match?(/Table '.+' doesn't exist/)
@@ -74,6 +74,22 @@ def indexes(table_name)
           end
         end
 
+        def create_index_definition(table_name, name, unique, columns, **options)
+          MySQL::IndexDefinition.new(table_name, name, unique, columns, **options)
+        end
+
+        def add_index_options(table_name, column_name, name: nil, if_not_exists: false, internal: false, **options) # :nodoc:
+          index, algorithm, if_not_exists = super
+          index.visible = options[:visible]
+          [index, algorithm, if_not_exists]
+        end
+
+        def valid_index_options
+          index_options = super
+          index_options << :visible if supports_index_visibility?
+          index_options
+        end
+
         def remove_column(table_name, column_name, type = nil, **options)
           if foreign_key_exists?(table_name, column: column_name)
             remove_foreign_key(table_name, column: column_name)

activerecord/lib/active_record/migration/command_recorder.rb

@@ -44,6 +44,7 @@ class Migration
     # * rename_enum_value (must supply a +:from+ and +:to+ option)
     # * rename_index
     # * rename_table
+    # * alter_index
     class CommandRecorder
       ReversibleAndIrreversibleMethods = [
         :create_table, :create_join_table, :rename_table, :add_column, :remove_column,
@@ -58,7 +59,8 @@ class CommandRecorder
         :add_unique_constraint, :remove_unique_constraint,
         :create_enum, :drop_enum, :rename_enum, :add_enum_value, :rename_enum_value,
         :create_schema, :drop_schema,
-        :create_virtual_table, :drop_virtual_table
+        :create_virtual_table, :drop_virtual_table,
+        :alter_index
       ]
       include JoinTable
 
@@ -183,6 +185,11 @@ def invert_#{method}(args, &block)    # def invert_create_table(args, &block)
 
         include StraightReversions
 
+        def invert_alter_index(args)
+          table_name, index_name, options = args
+          [:alter_index, [table_name, index_name, visible: !options[:visible]]]
+        end
+
         def invert_transaction(args, &block)
           sub_recorder = CommandRecorder.new(delegate)
           sub_recorder.revert(&block)

activerecord/lib/active_record/schema_dumper.rb

@@ -277,6 +277,7 @@ def index_parts(index)
         index_parts << "nulls_not_distinct: #{index.nulls_not_distinct.inspect}" if index.nulls_not_distinct
         index_parts << "type: #{index.type.inspect}" if index.type
         index_parts << "comment: #{index.comment.inspect}" if index.comment
+        index_parts << "visible: #{index.visible.inspect}" if @connection.supports_index_visibility? && index.invisible?
         index_parts
       end
 

activerecord/test/cases/active_record_schema_test.rb

@@ -18,6 +18,7 @@ class ActiveRecordSchemaTest < ActiveRecord::TestCase
     @connection.drop_table :fruits rescue nil
     @connection.drop_table :has_timestamps rescue nil
     @connection.drop_table :multiple_indexes rescue nil
+    @connection.drop_table :invisible_index rescue nil
     @schema_migration.delete_all_versions
     ActiveRecord::Migration.verbose = @original_verbose
   end
@@ -120,6 +121,20 @@ def test_schema_load_with_multiple_indexes_for_column_of_different_names
     assert_equal ["multiple_indexes_foo_1", "multiple_indexes_foo_2"], indexes.collect(&:name).sort
   end
 
+  if ActiveRecord::Base.lease_connection.supports_index_visibility?
+    def test_schema_load_for_index_visibility
+      ActiveRecord::Schema.define do
+        create_table :invisible_index do |t|
+          t.string "foo"
+          t.index ["foo"], name: "invisible_foo_index", visible: false
+        end
+      end
+
+      indexes = @connection.indexes("invisible_index").find { |index| index.name == "invisible_foo_index" }
+      assert_predicate indexes, :invisible?
+    end
+  end
+
   if current_adapter?(:PostgreSQLAdapter)
     def test_timestamps_with_and_without_zones
       ActiveRecord::Schema.define do

activerecord/test/cases/migration/change_table_test.rb

@@ -214,6 +214,30 @@ def test_column_creates_column_with_index
         end
       end
 
+      if ActiveRecord::Base.lease_connection.supports_index_visibility?
+        def test_column_creates_column_with_invisible_index
+          with_change_table do |t|
+            expect :add_column, nil, [:delete_me, :bar, :integer]
+            expect :add_index, nil, [:delete_me, :bar], visible: false
+            t.column :bar, :integer, index: { visible: false }
+          end
+        end
+
+        def test_index_creates_invisible_index
+          with_change_table do |t|
+            expect :add_index, nil, [:delete_me, :bar], visible: false
+            t.index :bar, visible: false
+          end
+        end
+
+        def test_alter_index_changes_index_visibility
+          with_change_table do |t|
+            expect :alter_index, nil, [:delete_me, :bar], visible: false
+            t.alter_index :bar, visible: false
+          end
+        end
+      end
+
       def test_index_creates_index
         with_change_table do |t|
           expect :add_index, nil, [:delete_me, :bar]

activerecord/test/cases/migration/columns_test.rb

@@ -388,6 +388,19 @@ def test_column_with_index
         connection.drop_table(:my_table) rescue nil
       end
 
+      if ActiveRecord::Base.lease_connection.supports_index_visibility?
+        def test_column_with_invisible_index
+          connection.create_table "my_table", force: true do |t|
+            t.column "col_one", :bigint
+            t.column "col_two", :bigint, index: { visible: false }
+          end
+
+          assert connection.index_exists?("my_table", :col_two, visible: false)
+        ensure
+          connection.drop_table(:my_table) rescue nil
+        end
+      end
+
       def test_add_column_without_column_name
         e = assert_raise ArgumentError do
           connection.create_table "my_table", force: true do |t|

activerecord/test/cases/migration/command_recorder_test.rb

@@ -315,6 +315,23 @@ def test_invert_add_index_with_algorithm_option
         assert_equal [:remove_index, [:table, :one, algorithm: :concurrently], nil], remove
       end
 
+      if ActiveRecord::Base.lease_connection.supports_index_visibility?
+        def test_invert_add_index_with_invisible_option
+          remove = @recorder.inverse_of :add_index, [:table, :one, visible: false]
+          assert_equal [:remove_index, [:table, :one, visible: false], nil], remove
+        end
+
+        def test_invert_remove_index_with_invisible_option
+          add = @recorder.inverse_of :remove_index, [:table, :one, visible: false]
+          assert_equal [:add_index, [:table, :one, visible: false]], add
+        end
+
+        def test_invert_alter_index
+          alter = @recorder.inverse_of :alter_index, [:table, :invisible_index, visible: false]
+          assert_equal [:alter_index, [:table, :invisible_index, visible: true]], alter
+        end
+      end
+
       def test_invert_remove_index
         add = @recorder.inverse_of :remove_index, [:table, :one]
         assert_equal [:add_index, [:table, :one]], add