From aeb412e7b1c1f51f7a1c8a84ba10db36942fec26 Mon Sep 17 00:00:00 2001 From: Hartley McGuire Date: Tue, 21 Jul 2026 11:39:31 -0400 Subject: [PATCH 01/10] Use information_schema to query MySQL indexes On its own this change may appear worse than the previous solution since we have to manually construct the information schema query. However, using information schema enables a future refactor which will query the indexes for multiple tables at once (which SHOW KEYS FROM) cannot. This will be particularly useful for dumping the Schema Cache, which currently queries the indexes (among other things) for every table sequentially. One consequence of this change is that indexes will now be returned in alphabetical order instead of definition order for MySQL. This now aligns the MySQL behavior with PostgreSQL, where we are already sorting indexes by their name. --- activerecord/CHANGELOG.md | 7 ++++ .../mysql/schema_statements.rb | 35 +++++++++++++------ 2 files changed, 32 insertions(+), 10 deletions(-) diff --git a/activerecord/CHANGELOG.md b/activerecord/CHANGELOG.md index 47482d874e368..8331efab5061f 100644 --- a/activerecord/CHANGELOG.md +++ b/activerecord/CHANGELOG.md @@ -1,3 +1,10 @@ +* Use `information_schema` to query MySQL indexes. + + Results will now be returned in alphabetical order by index name, which + may result in a one-time reordering of indexes in `schema.rb`. + + *Hartley McGuire* + * Deprecate the `insert_returning` option in PostgreSQL database configurations, and the `PostgreSQLAdapter#use_insert_returning?` method. diff --git a/activerecord/lib/active_record/connection_adapters/mysql/schema_statements.rb b/activerecord/lib/active_record/connection_adapters/mysql/schema_statements.rb index 15d2006afb56e..e0bddbfd556e5 100644 --- a/activerecord/lib/active_record/connection_adapters/mysql/schema_statements.rb +++ b/activerecord/lib/active_record/connection_adapters/mysql/schema_statements.rb @@ -8,12 +8,33 @@ module SchemaStatements # :nodoc: def indexes(table_name) indexes = [] current_index = nil - query_all("SHOW KEYS FROM #{quote_table_name(table_name)}").each do |row| + + scope = quoted_scope(table_name) + + optional_columns = +"" + optional_columns << ", EXPRESSION AS 'Expression'" if supports_expression_index? + if supports_disabling_indexes? + optional_columns << (mariadb? ? ", IF(ignored = 'NO', 'YES', 'NO') AS 'enabled'" : ", is_visible AS 'enabled'") + end + + result = query_all(<<~SQL) + SELECT TABLE_NAME AS 'Table', INDEX_NAME AS 'Key_name', + NON_UNIQUE AS 'Non_unique', SEQ_IN_INDEX AS 'Seq_in_index', + COLUMN_NAME AS 'Column_name', COLLATION AS 'Collation', + SUB_PART AS 'Sub_part', LOWER(INDEX_TYPE) AS 'Index_type', + INDEX_COMMENT AS 'Index_comment'#{optional_columns} + FROM information_schema.statistics + WHERE table_schema = #{scope[:schema]} + AND table_name = #{scope[:name]} + AND index_name != 'PRIMARY' + ORDER BY index_name, seq_in_index + SQL + + result.each do |row| if current_index != row["Key_name"] - next if row["Key_name"] == "PRIMARY" # skip the primary key current_index = row["Key_name"] - mysql_index_type = row["Index_type"].downcase.to_sym + mysql_index_type = row["Index_type"].to_sym case mysql_index_type when :fulltext, :spatial index_type = mysql_index_type @@ -34,7 +55,7 @@ def indexes(table_name) ] if supports_disabling_indexes? - index[-1][:enabled] = mariadb? ? row["Ignored"] == "NO" : row["Visible"] == "YES" + index[-1][:enabled] = row["enabled"] == "YES" end indexes << index @@ -71,12 +92,6 @@ def indexes(table_name) end MySQL::IndexDefinition.new(*index, **options) end - rescue StatementInvalid => e - if e.message.match?(/Table '.+' doesn't exist/) - [] - else - raise - end end def create_index_definition(table_name, name, unique, columns, **options) From 74b0abf275388d20d36288c093272645a71a3780 Mon Sep 17 00:00:00 2001 From: Hartley McGuire Date: Tue, 21 Jul 2026 14:19:07 -0400 Subject: [PATCH 02/10] Batch index queries for MySQL schema cache dump Add internal indexes_for_tables interface which iterates through tables and calls indexes, but can be optimized by adapter subclasses to be batched. This new method is used in the Schema Cache dumping code to replace N queries to retrieve index metadata with a single query (when supported). Also implement this new method for MySQL. In my _very_ large application, it reduces schema dump time by ~25%. --- activerecord/CHANGELOG.md | 9 ++ .../abstract/schema_statements.rb | 6 ++ .../mysql/schema_statements.rb | 84 ++++++++++++------- .../connection_adapters/schema_cache.rb | 13 ++- .../abstract_mysql_adapter/schema_test.rb | 57 +++++++++++++ 5 files changed, 137 insertions(+), 32 deletions(-) diff --git a/activerecord/CHANGELOG.md b/activerecord/CHANGELOG.md index 8331efab5061f..caa25905fd522 100644 --- a/activerecord/CHANGELOG.md +++ b/activerecord/CHANGELOG.md @@ -1,3 +1,12 @@ +* Batch schema cache dump queries for MySQL and PostgreSQL. + + `SchemaCache#add_all` now loads primary keys, columns, and indexes for + every cached table in three batched queries instead of one query per table + per type, speeding up the schema cache dump by roughly 25% on databases + with many tables. + + *Hartley McGuire* + * Use `information_schema` to query MySQL indexes. Results will now be returned in alphabetical order by index name, which diff --git a/activerecord/lib/active_record/connection_adapters/abstract/schema_statements.rb b/activerecord/lib/active_record/connection_adapters/abstract/schema_statements.rb index e65989ecbff4d..7bc8648e51566 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract/schema_statements.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract/schema_statements.rb @@ -81,6 +81,12 @@ def indexes(table_name) raise NotImplementedError, "#indexes is not implemented" end + def indexes_for_tables(table_names) # :nodoc: + table_names.each_with_object({}) do |table_name, hash| + hash[table_name.to_s] = indexes(table_name) + end + end + # Checks to see if an index exists on a table for a given index definition. # # # Check an index exists diff --git a/activerecord/lib/active_record/connection_adapters/mysql/schema_statements.rb b/activerecord/lib/active_record/connection_adapters/mysql/schema_statements.rb index e0bddbfd556e5..de3e502bc6841 100644 --- a/activerecord/lib/active_record/connection_adapters/mysql/schema_statements.rb +++ b/activerecord/lib/active_record/connection_adapters/mysql/schema_statements.rb @@ -6,10 +6,11 @@ module MySQL module SchemaStatements # :nodoc: # Returns an array of indexes for the given table. def indexes(table_name) - indexes = [] - current_index = nil + indexes_for_tables([table_name]).each_value.first || [] + end - scope = quoted_scope(table_name) + def indexes_for_tables(table_names) # :nodoc: + return {} if table_names.empty? optional_columns = +"" optional_columns << ", EXPRESSION AS 'Expression'" if supports_expression_index? @@ -24,15 +25,21 @@ def indexes(table_name) SUB_PART AS 'Sub_part', LOWER(INDEX_TYPE) AS 'Index_type', INDEX_COMMENT AS 'Index_comment'#{optional_columns} FROM information_schema.statistics - WHERE table_schema = #{scope[:schema]} - AND table_name = #{scope[:name]} + WHERE (#{information_schema_scope_sql(table_names)}) AND index_name != 'PRIMARY' - ORDER BY index_name, seq_in_index + ORDER BY table_name, index_name, seq_in_index SQL + indexes_by_table = Hash.new { |h, k| h[k] = [] } + + current_index = nil + result.each do |row| - if current_index != row["Key_name"] - current_index = row["Key_name"] + table = row["Table"] + index_key = [table, row["Key_name"]] + + if current_index != index_key + current_index = index_key mysql_index_type = row["Index_type"].to_sym case mysql_index_type @@ -43,7 +50,7 @@ def indexes(table_name) end index = [ - row["Table"], + table, row["Key_name"], row["Non_unique"].to_i == 0, [], @@ -58,40 +65,46 @@ def indexes(table_name) index[-1][:enabled] = row["enabled"] == "YES" end - indexes << index + indexes_by_table[table] << index end + index = indexes_by_table[table].last + if expression = row["Expression"] expression = expression.gsub("\\'", "'") expression = +"(#{expression})" unless expression.start_with?("(") - indexes.last[-2] << expression - indexes.last[-1][:expressions] ||= {} - indexes.last[-1][:expressions][expression] = expression - indexes.last[-1][:orders][expression] = :desc if row["Collation"] == "D" + index[-2] << expression + index[-1][:expressions] ||= {} + index[-1][:expressions][expression] = expression + index[-1][:orders][expression] = :desc if row["Collation"] == "D" else - indexes.last[-2] << row["Column_name"] - indexes.last[-1][:lengths][row["Column_name"]] = row["Sub_part"].to_i if row["Sub_part"] - indexes.last[-1][:orders][row["Column_name"]] = :desc if row["Collation"] == "D" + index[-2] << row["Column_name"] + index[-1][:lengths][row["Column_name"]] = row["Sub_part"].to_i if row["Sub_part"] + index[-1][:orders][row["Column_name"]] = :desc if row["Collation"] == "D" end end - indexes.map do |index| - options = index.pop + indexes_by_table.transform_values! do |table_indexes| + table_indexes.map! do |index| + options = index.pop - if expressions = options.delete(:expressions) - orders = options.delete(:orders) - lengths = options.delete(:lengths) + if expressions = options.delete(:expressions) + orders = options.delete(:orders) + lengths = options.delete(:lengths) - columns = index[-1].to_h { |name| - [ name.to_sym, expressions[name] || +quote_column_name(name) ] - } + columns = index[-1].to_h { |name| + [ name.to_sym, expressions[name] || +quote_column_name(name) ] + } - index[-1] = add_options_for_index_columns( - columns, order: orders, length: lengths - ).values.join(", ") + index[-1] = add_options_for_index_columns( + columns, order: orders, length: lengths + ).values.join(", ") + end + MySQL::IndexDefinition.new(*index, **options) end - MySQL::IndexDefinition.new(*index, **options) end + + indexes_by_table end def create_index_definition(table_name, name, unique, columns, **options) @@ -298,6 +311,19 @@ def extract_schema_qualified_name(string) [schema, name] end + def information_schema_scope_sql(table_names) + by_schema = table_names.each_with_object(Hash.new { |h, k| h[k] = [] }) do |name, h| + schema, tbl = extract_schema_qualified_name(name) + h[schema] << tbl + end + + by_schema.map do |schema, names| + schema_sql = schema ? quote(schema) : "database()" + names_sql = names.map { |n| quote(n) }.join(", ") + "(table_schema = #{schema_sql} AND table_name IN (#{names_sql}))" + end.join(" OR ") + end + def type_with_size_to_sql(type, size) case size&.to_s when nil, "tiny", "medium", "long" diff --git a/activerecord/lib/active_record/connection_adapters/schema_cache.rb b/activerecord/lib/active_record/connection_adapters/schema_cache.rb index f1ee10f12bbd4..65a52ef566935 100644 --- a/activerecord/lib/active_record/connection_adapters/schema_cache.rb +++ b/activerecord/lib/active_record/connection_adapters/schema_cache.rb @@ -390,9 +390,16 @@ def clear_data_source_cache!(_connection, name) end def add_all(pool) # :nodoc: - pool.with_connection do - tables_to_cache(pool).each do |table| - add(pool, table) + pool.with_connection do |connection| + tables = tables_to_cache(pool) + tables.each { |table| @data_sources[deep_deduplicate(table)] = true } + indexes_by_table = connection.indexes_for_tables(tables) + tables.each do |table| + table = deep_deduplicate(table) + primary_keys(pool, table) + columns(pool, table) + columns_hash(pool, table) + @indexes[table] = deep_deduplicate(indexes_by_table[table.to_s]) end version(pool) diff --git a/activerecord/test/cases/adapters/abstract_mysql_adapter/schema_test.rb b/activerecord/test/cases/adapters/abstract_mysql_adapter/schema_test.rb index 4fd188f3ecc74..64c0f4178a7ca 100644 --- a/activerecord/test/cases/adapters/abstract_mysql_adapter/schema_test.rb +++ b/activerecord/test/cases/adapters/abstract_mysql_adapter/schema_test.rb @@ -92,6 +92,63 @@ def test_dump_indexes assert_equal :fulltext, index_c.type end + def test_indexes_for_multiple_tables + @connection.create_table(:idx_multi_a) { |t| t.string :email; t.string :name } + @connection.create_table(:idx_multi_b) { |t| t.string :email; t.string :other } + # "by_email" is idx_multi_a's alphabetically-last index and + # idx_multi_b's alphabetically-first, so the rows are adjacent across + # the two tables in the ORDER BY. A name-only index boundary would fail + # to start a new index for idx_multi_b's "by_email" and crash. + @connection.add_index :idx_multi_a, :name, name: "aaa_name" + @connection.add_index :idx_multi_a, :email, name: "by_email" + @connection.add_index :idx_multi_b, :email, name: "by_email" + @connection.add_index :idx_multi_b, :other, name: "zzz_other" + + # A single table name returns an Array of indexes (backward compatible). + single = @connection.indexes("idx_multi_a") + assert_kind_of Array, single + assert_equal %w[aaa_name by_email], single.map(&:name).sort + + # indexes_for_tables returns a Hash of table name => Array of indexes. + multi = @connection.indexes_for_tables(["idx_multi_a", "idx_multi_b"]) + assert_kind_of Hash, multi + assert_equal %w[idx_multi_a idx_multi_b], multi.keys.sort + assert multi.values.all?(Array) + assert_equal %w[aaa_name by_email], multi["idx_multi_a"].map(&:name).sort + assert_equal %w[by_email zzz_other], multi["idx_multi_b"].map(&:name).sort + + # A non-primary index name shared across two tables must not merge: + # each table's "by_email" keeps its own columns and table attribute. + a = multi["idx_multi_a"].find { |i| i.name == "by_email" } + b = multi["idx_multi_b"].find { |i| i.name == "by_email" } + assert_equal %w[email], a.columns + assert_equal %w[email], b.columns + assert_equal "idx_multi_a", a.table + assert_equal "idx_multi_b", b.table + ensure + @connection.drop_table :idx_multi_a, if_exists: true + @connection.drop_table :idx_multi_b, if_exists: true + end + + def test_indexes_for_multiple_tables_with_qualified_and_unqualified_names + @connection.create_table(:idx_mix_a) { |t| t.string :name } + @connection.create_table("#{@db_name}.idx_mix_b") { |t| t.string :other } + @connection.add_index :idx_mix_a, :name, name: "mix_a_name" + @connection.add_index "#{@db_name}.idx_mix_b", :other, name: "mix_b_other" + + # Mixing a schema-qualified name with an unqualified one yields a + # multi-clause (OR) scope. The trailing AND index_name != 'PRIMARY' + # must filter every clause; without parenthesizing the scope it binds + # only to the last clause and the primary key leaks in as a "PRIMARY" + # index for the other table. + multi = @connection.indexes_for_tables(["#{@db_name}.idx_mix_b", :idx_mix_a]) + assert_equal %w[mix_b_other], multi["idx_mix_b"].map(&:name).sort + assert_equal %w[mix_a_name], multi["idx_mix_a"].map(&:name).sort + ensure + @connection.drop_table :idx_mix_a, if_exists: true + @connection.drop_table "#{@db_name}.idx_mix_b", if_exists: true + end + unless mysql_enforcing_gtid_consistency? def test_drop_temporary_table @connection.transaction do From 5ff0f0f683fc3dc273313e3c7c273dc0897b7183 Mon Sep 17 00:00:00 2001 From: Hartley McGuire Date: Tue, 21 Jul 2026 15:59:58 -0400 Subject: [PATCH 03/10] Use information_schema to query MySQL columns MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit More verbose, but this will enable extending the API to support querying by multiple table_names at once. Since information_schema returns an empty set instead of raising for a missing table, `column_definitions` now raises `StatementInvalid` explicitly to preserve the old behavior. information_schema.COLUMNS.COLUMN_DEFAULT differs from the old `SHOW FULL FIELDS` Default in two ways, both handled here: - MariaDB reports a nullable no-default column's default as the string "NULL" rather than SQL NULL; normalize that to nil so the column is treated as having no default. This is gated on `mariadb?` — MySQL already returns SQL NULL for a no-default column and leaves a literal "NULL" string default unquoted, so normalizing unconditionally would wipe MySQL's literal "NULL" defaults. - MariaDB quotes string-literal defaults (e.g. `'abc'`) and escapes embedded single quotes — by doubling them in varchar columns (`''`) but with a backslash in text columns (`\'`). MySQL leaves string defaults unquoted, so the strip-and-unescape step is gated on `mariadb?`; it strips the wrapping quotes and undoes both escape forms so the default matches what `SHOW FULL FIELDS` reported. --- .../abstract_mysql_adapter.rb | 18 ++++++++++++++- .../mysql/schema_statements.rb | 22 +++++++++++++++---- activerecord/test/cases/defaults_test.rb | 22 +++++++++++++++++++ 3 files changed, 57 insertions(+), 5 deletions(-) diff --git a/activerecord/lib/active_record/connection_adapters/abstract_mysql_adapter.rb b/activerecord/lib/active_record/connection_adapters/abstract_mysql_adapter.rb index d1b24ba69a6b4..2bf041b8f73d5 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract_mysql_adapter.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract_mysql_adapter.rb @@ -1116,7 +1116,23 @@ def configure_connection end def column_definitions(table_name) # :nodoc: - fields = query_all("SHOW FULL FIELDS FROM #{quote_table_name(table_name)}") + scope = quoted_scope(table_name) + fields = query_all(<<~SQL) + SELECT COLUMN_NAME AS 'Field', + COLUMN_TYPE AS 'Type', + COLLATION_NAME AS 'Collation', + IS_NULLABLE AS 'Null', + COLUMN_KEY AS 'Key', + COLUMN_DEFAULT AS 'Default', + EXTRA AS 'Extra', + PRIVILEGES AS 'Privileges', + COLUMN_COMMENT AS 'Comment' + FROM information_schema.columns + WHERE table_schema = #{scope[:schema]} AND table_name = #{scope[:name]} + ORDER BY ORDINAL_POSITION + SQL + + raise ActiveRecord::StatementInvalid.new("Could not find table '#{table_name}'", connection_pool: @pool) if fields.empty? update_fields_for_mariadb(table_name, fields) if mariadb? diff --git a/activerecord/lib/active_record/connection_adapters/mysql/schema_statements.rb b/activerecord/lib/active_record/connection_adapters/mysql/schema_statements.rb index de3e502bc6841..04a31a260b445 100644 --- a/activerecord/lib/active_record/connection_adapters/mysql/schema_statements.rb +++ b/activerecord/lib/active_record/connection_adapters/mysql/schema_statements.rb @@ -225,7 +225,15 @@ def create_table_definition(name, **options) def new_column_from_field(table_name, field, _definitions) type_metadata = fetch_type_metadata(field["Type"], field["Extra"]) default, default_function = field["Default"], nil - + # MariaDB reports the COLUMN_DEFAULT of a nullable column with no + # default as the string "NULL" rather than SQL NULL; normalize it so + # the column is treated as having no default. MySQL already returns + # SQL NULL for a no-default column, and leaves a literal "NULL" + # string default unquoted, so this must be MariaDB-only — otherwise + # MySQL's literal "NULL" defaults would be wiped to nil. A literal + # "NULL" string default on MariaDB is quoted ('NULL') and is handled + # by the strip branch below. + default = nil if mariadb? && default == "NULL" if type_metadata.type == :datetime && /\ACURRENT_TIMESTAMP(?:\([0-6]?\))?\z/i.match?(default) default = "#{default} ON UPDATE #{default}" if /on update CURRENT_TIMESTAMP/i.match?(field["Extra"]) default, default_function = nil, default @@ -237,9 +245,15 @@ def new_column_from_field(table_name, field, _definitions) default = default.gsub("\\'", "'") default, default_function = nil, default end - elsif type_metadata.type == :text && default&.start_with?("'") - # strip and unescape quotes - default = default[1...-1].gsub("\\'", "'") + elsif mariadb? && default&.start_with?("'") + # MariaDB quotes string literal defaults in information_schema + # COLUMN_DEFAULT (e.g. `'abc'`, `'O''Connor'`) and escapes + # embedded single quotes — by doubling them in varchar defaults + # (`''`) but with a backslash in text defaults (`\'`). MySQL leaves + # string defaults unquoted, so this only runs on MariaDB. Strip the + # wrapping quotes and undo both escape forms so the default matches + # what SHOW FULL FIELDS reported. + default = default[1...-1].gsub("''", "'").gsub("\\'", "'") end MySQL::Column.new( diff --git a/activerecord/test/cases/defaults_test.rb b/activerecord/test/cases/defaults_test.rb index cc730c66c3daf..b16da086bc05c 100644 --- a/activerecord/test/cases/defaults_test.rb +++ b/activerecord/test/cases/defaults_test.rb @@ -182,6 +182,28 @@ class MysqlDefaultExpressionTest < ActiveRecord::TestCase end end + test "literal NULL string default is not mistaken for no default" do + connection = ActiveRecord::Base.lease_connection + connection.create_table :default_null_string, force: true do |t| + t.string :name, default: "NULL" + end + column = connection.columns(:default_null_string).find { |c| c.name == "name" } + assert_equal "NULL", column.default + ensure + connection&.drop_table :default_null_string, if_exists: true + end + + test "string default containing a single quote is unescaped correctly" do + connection = ActiveRecord::Base.lease_connection + connection.create_table :default_quote_string, force: true do |t| + t.string :name, default: "O'Connor" + end + column = connection.columns(:default_quote_string).find { |c| c.name == "name" } + assert_equal "O'Connor", column.default + ensure + connection&.drop_table :default_quote_string, if_exists: true + end + test "schema dump datetime includes default expression" do output = dump_table_schema("datetime_defaults") assert_match %r/t\.datetime\s+"modified_datetime",\s+precision: nil,\s+default: -> { "CURRENT_TIMESTAMP(?:\(\))?" }/i, output From f28ecba8e36b6bb90df45bc7644ea92250eca7ef Mon Sep 17 00:00:00 2001 From: Hartley McGuire Date: Thu, 23 Jul 2026 12:37:40 -0400 Subject: [PATCH 04/10] Avoid SHOW CREATE TABLE during MariaDB column introspection update_fields_for_mariadb now uses information_schema.COLUMNS.COLUMN_DEFAULT to distinguish function defaults from literal strings that look like function calls. MariaDB quotes string literals there (e.g. 'uuid()') but leaves function defaults unquoted (e.g. uuid()), so the disambiguation no longer requires parsing SHOW CREATE TABLE output. Because column_definitions already selects COLUMN_DEFAULT AS 'Default' from information_schema.columns, the field's Default value already carries the quoted/unquoted distinction - so no extra query is needed (unlike the SHOW FULL FIELDS based path), and update_fields_for_mariadb no longer needs the table name. --- .../abstract_mysql_adapter.rb | 30 +++++++++---------- activerecord/test/cases/migration_test.rb | 11 +++++++ 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/activerecord/lib/active_record/connection_adapters/abstract_mysql_adapter.rb b/activerecord/lib/active_record/connection_adapters/abstract_mysql_adapter.rb index 2bf041b8f73d5..d81c638260ff2 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract_mysql_adapter.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract_mysql_adapter.rb @@ -1134,30 +1134,30 @@ def column_definitions(table_name) # :nodoc: raise ActiveRecord::StatementInvalid.new("Could not find table '#{table_name}'", connection_pool: @pool) if fields.empty? - update_fields_for_mariadb(table_name, fields) if mariadb? + update_fields_for_mariadb(fields) if mariadb? fields end - def update_fields_for_mariadb(table_name, fields) + def update_fields_for_mariadb(fields) has_function_default_candidate = fields.any? do |field| default = field["Default"] default&.match?(/[a-zA-Z_]\w*\(/) && !/\ACURRENT_TIMESTAMP/i.match?(default) end + return unless has_function_default_candidate + + # MariaDB quotes string literal defaults in information_schema.COLUMNS.COLUMN_DEFAULT + # (e.g. `'uuid()'`) but leaves function/expression defaults unquoted (e.g. `uuid()`), + # unlike SHOW FULL FIELDS which reports both as `uuid()`. Use that to distinguish + # function defaults from literal strings that merely look like function calls, without + # parsing SHOW CREATE TABLE output. + fields.each do |field| + default = field["Default"] + next unless default&.match?(/[a-zA-Z_]\w*\(/) + next if /\ACURRENT_TIMESTAMP/i.match?(default) + next if default.start_with?("'") - if has_function_default_candidate - table_info = create_table_info(table_name) - fields.each do |field| - default = field["Default"] - next unless default&.match?(/[a-zA-Z_]\w*\(/) - next if /\ACURRENT_TIMESTAMP/i.match?(default) - - field_name = field["Field"] - match = table_info&.match(/`#{field_name}` .+ DEFAULT ('|\d+|[A-z]+)/) - if match && match[1].match?(/\A[A-z]/) - field["Extra"] = "DEFAULT_GENERATED" - end - end + field["Extra"] = "DEFAULT_GENERATED" end end diff --git a/activerecord/test/cases/migration_test.rb b/activerecord/test/cases/migration_test.rb index 1ea97f99b08df..be00635f908e9 100644 --- a/activerecord/test/cases/migration_test.rb +++ b/activerecord/test/cases/migration_test.rb @@ -1542,6 +1542,17 @@ def test_default_functions_on_columns end end + if current_adapter?(:Mysql2Adapter, :TrilogyAdapter) + def test_literal_default_that_looks_like_a_function_is_not_mistaken_for_one + with_bulk_change_table do |t| + t.string :name, default: "uuid()" + end + + assert_equal "uuid()", column(:name).default + assert_nil column(:name).default_function + end + end + if current_adapter?(:Mysql2Adapter, :TrilogyAdapter) def test_updating_auto_increment with_bulk_change_table do |t| From e760b95d9e19e255a9882cdab8bd78713ae9a757 Mon Sep 17 00:00:00 2001 From: Hartley McGuire Date: Tue, 21 Jul 2026 16:07:32 -0400 Subject: [PATCH 05/10] Add API to batch query columns for schema cache Introduce a `columns_for_tables` method on the abstract adapter whose default implementation just loops over each table and calls `columns`, leaving room for adapters to override it with a single batched query. `SchemaCache#add_all` now goes through it instead of calling `columns` once per table, mirroring the `indexes_for_tables` batch added previously. This sets up the next commit, which overrides `columns_for_tables` on the MySQL adapter to fetch every table's columns in one query. --- .../connection_adapters/abstract/schema_statements.rb | 6 ++++++ .../lib/active_record/connection_adapters/schema_cache.rb | 6 ++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/activerecord/lib/active_record/connection_adapters/abstract/schema_statements.rb b/activerecord/lib/active_record/connection_adapters/abstract/schema_statements.rb index 7bc8648e51566..cfc72519e702b 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract/schema_statements.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract/schema_statements.rb @@ -117,6 +117,12 @@ def columns(table_name) end end + def columns_for_tables(table_names) # :nodoc: + table_names.each_with_object({}) do |table_name, hash| + hash[table_name.to_s] = columns(table_name) + end + end + # Checks to see if a column exists in a given table. # # # Check a column exists diff --git a/activerecord/lib/active_record/connection_adapters/schema_cache.rb b/activerecord/lib/active_record/connection_adapters/schema_cache.rb index 65a52ef566935..1fac440e531e8 100644 --- a/activerecord/lib/active_record/connection_adapters/schema_cache.rb +++ b/activerecord/lib/active_record/connection_adapters/schema_cache.rb @@ -393,12 +393,14 @@ def add_all(pool) # :nodoc: pool.with_connection do |connection| tables = tables_to_cache(pool) tables.each { |table| @data_sources[deep_deduplicate(table)] = true } + columns_by_table = connection.columns_for_tables(tables) indexes_by_table = connection.indexes_for_tables(tables) tables.each do |table| table = deep_deduplicate(table) primary_keys(pool, table) - columns(pool, table) - columns_hash(pool, table) + table_columns = deep_deduplicate(columns_by_table[table.to_s]) + @columns[table] = table_columns + @columns_hash[table] = table_columns.index_by(&:name).freeze @indexes[table] = deep_deduplicate(indexes_by_table[table.to_s]) end From 3ccbee756323341c9b2e92e43c89795c44fb0200 Mon Sep 17 00:00:00 2001 From: Hartley McGuire Date: Tue, 21 Jul 2026 16:11:53 -0400 Subject: [PATCH 06/10] Batch column loading for MySQL schema cache dumps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Override `columns_for_tables` on the MySQL adapter to pull every requested table's columns from `information_schema.columns` in one go, ordering by (`table_name`, `ORDINAL_POSITION`) so the rows come back grouped by table. `column_definitions_for_tables` builds its result from the query rows (keyed by the database's own `TABLE_NAME`) rather than pre-seeding by the requested name, and `column_definitions` recovers its single table via `each_value.first`. This keeps single-table introspection correct under `lower_case_table_names = 1/2`, where the database's table name can differ in case from the requested one. `SchemaCache#add` keeps using the single-table `columns` getter rather than the batched `columns_for_tables`, because the batch keys results by the database's `TABLE_NAME` — so a miscased `add("MixedCase")` would be attributed to the database's lowercased name. `add_all` is the only batched caller, and it passes the real data-source names from `tables_to_cache`, which already match the database's `TABLE_NAME`. `SchemaCache#add_all` (used by `db:schema:cache:dump`) now issues one columns query instead of one per table. --- .../abstract_mysql_adapter.rb | 30 ++++++++---- .../mysql/schema_statements.rb | 9 ++++ .../case_insensitive_table_names_test.rb | 48 +++++++++++++++++++ .../abstract_mysql_adapter/schema_test.rb | 21 ++++++++ 4 files changed, 100 insertions(+), 8 deletions(-) create mode 100644 activerecord/test/cases/adapters/abstract_mysql_adapter/case_insensitive_table_names_test.rb diff --git a/activerecord/lib/active_record/connection_adapters/abstract_mysql_adapter.rb b/activerecord/lib/active_record/connection_adapters/abstract_mysql_adapter.rb index d81c638260ff2..88d9f91f7b552 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract_mysql_adapter.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract_mysql_adapter.rb @@ -1116,8 +1116,15 @@ def configure_connection end def column_definitions(table_name) # :nodoc: - scope = quoted_scope(table_name) - fields = query_all(<<~SQL) + fields = column_definitions_for_tables([table_name]).each_value.first + raise ActiveRecord::StatementInvalid.new("Could not find table '#{table_name}'", connection_pool: @pool) unless fields + fields + end + + def column_definitions_for_tables(table_names) # :nodoc: + return {} if table_names.empty? + + result = query_all(<<~SQL) SELECT COLUMN_NAME AS 'Field', COLUMN_TYPE AS 'Type', COLLATION_NAME AS 'Collation', @@ -1126,17 +1133,24 @@ def column_definitions(table_name) # :nodoc: COLUMN_DEFAULT AS 'Default', EXTRA AS 'Extra', PRIVILEGES AS 'Privileges', - COLUMN_COMMENT AS 'Comment' + COLUMN_COMMENT AS 'Comment', + TABLE_NAME AS 'table_name' FROM information_schema.columns - WHERE table_schema = #{scope[:schema]} AND table_name = #{scope[:name]} - ORDER BY ORDINAL_POSITION + WHERE #{information_schema_scope_sql(table_names)} + ORDER BY table_name, ORDINAL_POSITION SQL - raise ActiveRecord::StatementInvalid.new("Could not find table '#{table_name}'", connection_pool: @pool) if fields.empty? + fields_by_table = result.each_with_object(Hash.new { |h, k| h[k] = [] }) do |row, hash| + hash[row.delete("table_name")] << row + end - update_fields_for_mariadb(fields) if mariadb? + if mariadb? + fields_by_table.each_value do |fields| + update_fields_for_mariadb(fields) + end + end - fields + fields_by_table end def update_fields_for_mariadb(fields) diff --git a/activerecord/lib/active_record/connection_adapters/mysql/schema_statements.rb b/activerecord/lib/active_record/connection_adapters/mysql/schema_statements.rb index 04a31a260b445..6160de4c72923 100644 --- a/activerecord/lib/active_record/connection_adapters/mysql/schema_statements.rb +++ b/activerecord/lib/active_record/connection_adapters/mysql/schema_statements.rb @@ -107,6 +107,15 @@ def indexes_for_tables(table_names) # :nodoc: indexes_by_table end + def columns_for_tables(table_names) # :nodoc: + definitions_by_table = column_definitions_for_tables(table_names) + definitions_by_table.each_with_object({}) do |(table, definitions), hash| + hash[table] = definitions.map do |field| + new_column_from_field(table, field, definitions) + end + end + end + def create_index_definition(table_name, name, unique, columns, **options) MySQL::IndexDefinition.new(table_name, name, unique, columns, **options) end diff --git a/activerecord/test/cases/adapters/abstract_mysql_adapter/case_insensitive_table_names_test.rb b/activerecord/test/cases/adapters/abstract_mysql_adapter/case_insensitive_table_names_test.rb new file mode 100644 index 0000000000000..956c6f6b8d371 --- /dev/null +++ b/activerecord/test/cases/adapters/abstract_mysql_adapter/case_insensitive_table_names_test.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +require "cases/helper" + +module ActiveRecord + class CaseInsensitiveTableNamesTest < ActiveRecord::AbstractMysqlTestCase + def setup + @connection = ActiveRecord::Base.lease_connection + @connection.drop_table(:MixedCaseTbl, if_exists: true) + end + + def teardown + @connection.drop_table(:MixedCaseTbl, if_exists: true) + end + + # With lower_case_table_names = 1 or 2, table-name comparison is + # case-insensitive, so introspecting "mixedcasetbl" must find the table + # stored as "MixedCaseTbl". The single-table APIs and SchemaCache#add must + # attribute the result to the requested name without depending on the + # database's own (differently-cased) table name. + def test_introspection_with_wrong_case_table_name + skip "only relevant when table name comparison is case-insensitive" if lower_case_table_names.to_i == 0 + + @connection.create_table(:MixedCaseTbl, force: true) do |t| + t.string :name + t.integer :custom_id + end + @connection.add_index(:MixedCaseTbl, :name) + + assert_equal %w[id name custom_id], @connection.columns("mixedcasetbl").map(&:name) + assert_equal %w[id], @connection.primary_keys("mixedcasetbl") + assert_equal 1, @connection.indexes("mixedcasetbl").size + + cache = ActiveRecord::Base.connection_pool.schema_cache + cache.clear_data_source_cache!("mixedcasetbl") + cache.add("mixedcasetbl") + + assert_equal %w[id name custom_id], cache.columns("mixedcasetbl").map(&:name) + assert_equal "id", cache.primary_keys("mixedcasetbl") + assert_equal 1, cache.indexes("mixedcasetbl").size + end + + private + def lower_case_table_names + @connection.show_variable("lower_case_table_names") + end + end +end diff --git a/activerecord/test/cases/adapters/abstract_mysql_adapter/schema_test.rb b/activerecord/test/cases/adapters/abstract_mysql_adapter/schema_test.rb index 64c0f4178a7ca..76f5633e5a1bc 100644 --- a/activerecord/test/cases/adapters/abstract_mysql_adapter/schema_test.rb +++ b/activerecord/test/cases/adapters/abstract_mysql_adapter/schema_test.rb @@ -149,6 +149,27 @@ def test_indexes_for_multiple_tables_with_qualified_and_unqualified_names @connection.drop_table "#{@db_name}.idx_mix_b", if_exists: true end + def test_columns_for_multiple_tables + @connection.create_table(:cols_multi_a) { |t| t.string :title; t.integer :counter } + @connection.create_table(:cols_multi_b) { |t| t.string :name; t.text :body } + + # A single table name returns an Array of columns (backward compatible). + assert_kind_of Array, @connection.columns("cols_multi_a") + + # columns_for_tables returns a Hash of table name => Array of columns, + # matching #columns for each table (names and sql_types). + multi = @connection.columns_for_tables(["cols_multi_a", "cols_multi_b"]) + assert_kind_of Hash, multi + assert_equal %w[cols_multi_a cols_multi_b], multi.keys.sort + + profile = ->(cols) { cols.map { |c| [c.name, c.sql_type] } } + assert_equal profile.call(@connection.columns("cols_multi_a")), profile.call(multi["cols_multi_a"]) + assert_equal profile.call(@connection.columns("cols_multi_b")), profile.call(multi["cols_multi_b"]) + ensure + @connection.drop_table :cols_multi_a, if_exists: true + @connection.drop_table :cols_multi_b, if_exists: true + end + unless mysql_enforcing_gtid_consistency? def test_drop_temporary_table @connection.transaction do From 62ced825e952a1a555bf6aa6dd35c0ef148ef5d0 Mon Sep 17 00:00:00 2001 From: Hartley McGuire Date: Tue, 21 Jul 2026 16:11:53 -0400 Subject: [PATCH 07/10] Batch primary key queries for MySQL schema cache dump MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `SchemaCache#add_all` was still calling `primary_keys` once per table after the indexes and columns batches. Introduce a `primary_keys_for_tables` interface on the abstract adapter — defaulting to a per-table loop, like `indexes_for_tables` and `columns_for_tables` — and route `add_all` through it. On MySQL, override `primary_keys_for_tables` to read every requested table's primary key columns from `information_schema.statistics` in one query (the same table the indexes batch reads), ordered by `table_name, seq_in_index` so composite keys come back in order. The single-table `primary_keys` now delegates to it, so both paths share one query. `add_all` stores the key the way `primary_key` does — the column name for single-column keys, the array for composite ones — so `SchemaCache#primary_keys` keeps returning what callers expect. --- .../abstract/schema_statements.rb | 6 +++ .../abstract_mysql_adapter.rb | 18 ++++--- .../connection_adapters/schema_cache.rb | 6 ++- .../abstract_mysql_adapter/schema_test.rb | 52 +++++++++++++++++++ 4 files changed, 75 insertions(+), 7 deletions(-) diff --git a/activerecord/lib/active_record/connection_adapters/abstract/schema_statements.rb b/activerecord/lib/active_record/connection_adapters/abstract/schema_statements.rb index cfc72519e702b..e78b091630b42 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract/schema_statements.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract/schema_statements.rb @@ -159,6 +159,12 @@ def primary_key(table_name) pk end + def primary_keys_for_tables(table_names) # :nodoc: + table_names.each_with_object({}) do |table_name, hash| + hash[table_name.to_s] = primary_keys(table_name) + end + end + # Creates a new table with the name +table_name+. +table_name+ may either # be a String or a Symbol. # diff --git a/activerecord/lib/active_record/connection_adapters/abstract_mysql_adapter.rb b/activerecord/lib/active_record/connection_adapters/abstract_mysql_adapter.rb index 88d9f91f7b552..990e23806b3fb 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract_mysql_adapter.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract_mysql_adapter.rb @@ -690,17 +690,23 @@ def show_variable(name) def primary_keys(table_name) # :nodoc: raise ArgumentError unless table_name.present? + primary_keys_for_tables([table_name]).each_value.first || [] + end - scope = quoted_scope(table_name) + def primary_keys_for_tables(table_names) # :nodoc: + return {} if table_names.empty? - query_values(<<~SQL) - SELECT column_name + result = query_all(<<~SQL) + SELECT table_name AS 'table_name', column_name AS 'column_name' FROM information_schema.statistics WHERE index_name = 'PRIMARY' - AND table_schema = #{scope[:schema]} - AND table_name = #{scope[:name]} - ORDER BY seq_in_index + AND (#{information_schema_scope_sql(table_names)}) + ORDER BY table_name, seq_in_index SQL + + result.each_with_object(Hash.new { |h, k| h[k] = [] }) do |row, primary_keys_by_table| + primary_keys_by_table[row["table_name"]] << row["column_name"] + end end def case_sensitive_comparison(attribute, value) # :nodoc: diff --git a/activerecord/lib/active_record/connection_adapters/schema_cache.rb b/activerecord/lib/active_record/connection_adapters/schema_cache.rb index 1fac440e531e8..281bef356c0a0 100644 --- a/activerecord/lib/active_record/connection_adapters/schema_cache.rb +++ b/activerecord/lib/active_record/connection_adapters/schema_cache.rb @@ -393,11 +393,15 @@ def add_all(pool) # :nodoc: pool.with_connection do |connection| tables = tables_to_cache(pool) tables.each { |table| @data_sources[deep_deduplicate(table)] = true } + primary_keys_by_table = connection.primary_keys_for_tables(tables) columns_by_table = connection.columns_for_tables(tables) indexes_by_table = connection.indexes_for_tables(tables) tables.each do |table| table = deep_deduplicate(table) - primary_keys(pool, table) + table_primary_keys = primary_keys_by_table[table.to_s] + # Mirrors #primary_key, unwraps singular keys. + table_primary_keys = table_primary_keys.size > 1 ? table_primary_keys : table_primary_keys.first + @primary_keys[table] = deep_deduplicate(table_primary_keys) table_columns = deep_deduplicate(columns_by_table[table.to_s]) @columns[table] = table_columns @columns_hash[table] = table_columns.index_by(&:name).freeze diff --git a/activerecord/test/cases/adapters/abstract_mysql_adapter/schema_test.rb b/activerecord/test/cases/adapters/abstract_mysql_adapter/schema_test.rb index 76f5633e5a1bc..9006384280959 100644 --- a/activerecord/test/cases/adapters/abstract_mysql_adapter/schema_test.rb +++ b/activerecord/test/cases/adapters/abstract_mysql_adapter/schema_test.rb @@ -170,6 +170,58 @@ def test_columns_for_multiple_tables @connection.drop_table :cols_multi_b, if_exists: true end + def test_primary_keys_for_multiple_tables + @connection.create_table(:pk_multi_a, primary_key: :custom_id) { |t| t.string :name } + @connection.create_table(:pk_multi_b) { |t| t.string :name } + @connection.execute "CREATE TABLE pk_multi_c (a int NOT NULL, b int NOT NULL, c int NOT NULL, PRIMARY KEY (c, a, b))" + @connection.create_table(:pk_multi_none, id: false) { |t| t.string :name } + + # A single table name returns an Array of primary key columns (backward compatible). + assert_equal %w[custom_id], @connection.primary_keys("pk_multi_a") + assert_equal %w[id], @connection.primary_keys("pk_multi_b") + # Composite key columns arrive in PRIMARY KEY ordinal order (seq_in_index), + # not the table's column order. + assert_equal %w[c a b], @connection.primary_keys("pk_multi_c") + # A table without a primary key returns an empty Array, not a missing key. + assert_equal [], @connection.primary_keys("pk_multi_none") + + # primary_keys_for_tables returns a Hash of table name => Array of pk columns. + multi = @connection.primary_keys_for_tables(["pk_multi_a", "pk_multi_b", "pk_multi_c", "pk_multi_none"]) + assert_kind_of Hash, multi + assert_equal %w[pk_multi_a pk_multi_b pk_multi_c], multi.keys.sort + assert_equal %w[custom_id], multi["pk_multi_a"] + assert_equal %w[id], multi["pk_multi_b"] + assert_equal %w[c a b], multi["pk_multi_c"] + # No primary key => absent from the hash; the default-proc returns [] on + # access (so the cache reduces it to nil, as #primary_key does). + assert_not multi.key?("pk_multi_none") + assert_equal [], multi["pk_multi_none"] + ensure + @connection.drop_table :pk_multi_a, if_exists: true + @connection.drop_table :pk_multi_b, if_exists: true + @connection.drop_table :pk_multi_c, if_exists: true + @connection.drop_table :pk_multi_none, if_exists: true + end + + def test_primary_keys_for_multiple_tables_with_qualified_and_unqualified_names + @connection.create_table(:pk_mix_a) { |t| t.string :name } + @connection.create_table("#{@db_name}.pk_mix_b") { |t| t.string :other } + @connection.add_index :pk_mix_a, :name, name: "pk_mix_a_name" + @connection.add_index "#{@db_name}.pk_mix_b", :other, name: "pk_mix_b_other" + + # Mixing a schema-qualified name with an unqualified one yields a + # multi-clause (OR) scope. `index_name = 'PRIMARY' AND ` must + # filter every clause; without parenthesizing the scope, AND binds + # only to the first clause and the other table's non-primary indexes + # leak in as primary key columns. + multi = @connection.primary_keys_for_tables(["#{@db_name}.pk_mix_b", :pk_mix_a]) + assert_equal %w[id], multi["pk_mix_b"] + assert_equal %w[id], multi["pk_mix_a"] + ensure + @connection.drop_table :pk_mix_a, if_exists: true + @connection.drop_table "#{@db_name}.pk_mix_b", if_exists: true + end + unless mysql_enforcing_gtid_consistency? def test_drop_temporary_table @connection.transaction do From 6edc07c689f69ae22af5fc154c3cf7154690d948 Mon Sep 17 00:00:00 2001 From: Hartley McGuire Date: Tue, 21 Jul 2026 16:11:53 -0400 Subject: [PATCH 08/10] Batch primary key queries for PostgreSQL schema cache dump `SchemaCache#add_all` was still calling `primary_keys` once per table on PostgreSQL, after the indexes and columns batches. Override `primary_keys_for_tables` to fetch every requested table's primary key columns from `pg_index` / `pg_attribute` in one query, and route `add_all` through it (mirroring the MySQL batch). The query scopes `pg_index.indrelid` with `to_regclass` rather than casting the names to `regclass`: `to_regclass` resolves each name the same way (an unqualified name picks the first schema on the search path, a qualified name pins its schema) but returns NULL for a missing relation instead of raising, so one bad name can't fail the whole batch. Composite keys are ordered per table with `array_position(i.indkey, a.attnum)`. Rows are keyed by the requested table-name string (not the relname), so schema-qualified requests round-trip through `add_all`' `[table.to_s]`. `primary_keys` now delegates to `primary_keys_for_tables` for a single table; a missing relation returns `[]` instead of raising, which the schema cache already guards against with `data_source_exists?`. --- activerecord/CHANGELOG.md | 7 +++ .../postgresql/schema_statements.rb | 63 ++++++++++++++++--- .../cases/adapters/postgresql/schema_test.rb | 34 ++++++++++ 3 files changed, 97 insertions(+), 7 deletions(-) diff --git a/activerecord/CHANGELOG.md b/activerecord/CHANGELOG.md index caa25905fd522..a2a9c88521d37 100644 --- a/activerecord/CHANGELOG.md +++ b/activerecord/CHANGELOG.md @@ -1,3 +1,10 @@ +* Return `[]`/`nil` instead of raising when querying primary keys for a + missing PostgreSQL table. + + Aligns PostgreSQL with the MySQL adapter's existing behavior. + + *Hartley McGuire* + * Batch schema cache dump queries for MySQL and PostgreSQL. `SchemaCache#add_all` now loads primary keys, columns, and indexes for diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/schema_statements.rb b/activerecord/lib/active_record/connection_adapters/postgresql/schema_statements.rb index b5a8b114054af..ecb3d5e6d035c 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/schema_statements.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/schema_statements.rb @@ -585,16 +585,34 @@ def pk_and_sequence_for(table) # :nodoc: end def primary_keys(table_name) # :nodoc: - query_values(<<~SQL) - SELECT a.attname + primary_keys_for_tables([table_name])[table_name.to_s] + end + + def primary_keys_for_tables(table_names) # :nodoc: + return {} if table_names.empty? + + result_by_table = table_names.each_with_object({}) do |name, h| + h[name.to_s] = [] + end + key_for = table_key_lookup(table_names) + + result = query_all(<<~SQL) + SELECT n.nspname AS nspname, t.relname AS relname, a.attname AS column_name FROM pg_index i - JOIN pg_attribute a - ON a.attrelid = i.indrelid - AND a.attnum = ANY(i.indkey) - WHERE i.indrelid = #{quote(quote_table_name(table_name))}::regclass - AND i.indisprimary + INNER JOIN pg_class t ON t.oid = i.indrelid + INNER JOIN pg_namespace n ON n.oid = t.relnamespace + INNER JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = ANY(i.indkey) + WHERE i.indisprimary + AND i.indrelid = #{to_regclass_array_sql(table_names)} ORDER BY array_position(i.indkey, a.attnum) SQL + + result.each do |row| + key = key_for.call(row["nspname"], row["relname"]) + result_by_table[key] << row["column_name"] if key + end + + result_by_table end # Renames a table. @@ -1411,6 +1429,37 @@ def extract_schema_qualified_name(string) [name.schema, name.identifier] end + # Build a SQL expression scoping a relation-OID column (e.g. + # +i.indrelid+, +a.attrelid+) to the given table names via + # to_regclass. Each name resolves the way casting to regclass does + # (an unqualified name picks the first schema on the search path; a + # qualified name pins its schema), but to_regclass returns NULL for + # a missing relation instead of raising — so one bad name can't fail + # the whole batch. + def to_regclass_array_sql(table_names) + names_sql = table_names.map { |name| quote(quote_table_name(name)) }.join(", ") + "ANY(ARRAY(SELECT to_regclass(x)::oid FROM unnest(ARRAY[#{names_sql}]) AS x))" + end + + # Returns a proc mapping a row's (namespace, relname) back to the + # requested table-name string, so results are keyed the way + # SchemaCache#add_all looks them up (by +table.to_s+). Qualified + # requests match on (schema, relname); unqualified requests match on + # relname alone (resolved via the search path). + def table_key_lookup(table_names) + qualified = {} + unqualified = {} + table_names.each do |name| + schema, identifier = extract_schema_qualified_name(name) + if schema + qualified[[schema, identifier]] = name.to_s + else + unqualified[identifier] = name.to_s + end + end + ->(nsp, rel) { qualified[[nsp, rel]] || unqualified[rel] } + end + def decode_string_array(value) PG::TextDecoder::Array.new.decode(value) end diff --git a/activerecord/test/cases/adapters/postgresql/schema_test.rb b/activerecord/test/cases/adapters/postgresql/schema_test.rb index 844150ec451dc..52a6165349667 100644 --- a/activerecord/test/cases/adapters/postgresql/schema_test.rb +++ b/activerecord/test/cases/adapters/postgresql/schema_test.rb @@ -291,6 +291,40 @@ def test_data_source_exists_quoted_table end end + def test_primary_keys_for_multiple_tables + @connection.create_table(:pg_pk_multi_a, primary_key: :custom_id) { |t| t.string :name } + @connection.create_table(:pg_pk_multi_b) { |t| t.string :name } + @connection.execute "CREATE TABLE pg_pk_multi_c (a int NOT NULL, b int NOT NULL, c int NOT NULL, PRIMARY KEY (c, a, b))" + @connection.create_table(:pg_pk_multi_none, id: false) { |t| t.string :name } + + # A single table name returns an Array of primary key columns (backward compatible). + assert_equal %w[custom_id], @connection.primary_keys("pg_pk_multi_a") + assert_equal %w[id], @connection.primary_keys("pg_pk_multi_b") + # Composite key columns arrive in PRIMARY KEY ordinal order, not column order. + assert_equal %w[c a b], @connection.primary_keys("pg_pk_multi_c") + # A table without a primary key returns an empty Array, not a missing key. + assert_equal [], @connection.primary_keys("pg_pk_multi_none") + + # primary_keys_for_tables returns a Hash keyed by the requested table name. + multi = @connection.primary_keys_for_tables([ + "pg_pk_multi_a", "pg_pk_multi_b", "pg_pk_multi_c", "pg_pk_multi_none", + "#{SCHEMA_NAME}.#{PK_TABLE_NAME}", + ]) + assert_kind_of Hash, multi + assert_equal %w[pg_pk_multi_a pg_pk_multi_b pg_pk_multi_c pg_pk_multi_none test_schema.table_with_pk], multi.keys.sort + assert_equal %w[custom_id], multi["pg_pk_multi_a"] + assert_equal %w[id], multi["pg_pk_multi_b"] + assert_equal %w[c a b], multi["pg_pk_multi_c"] + assert_equal [], multi["pg_pk_multi_none"] + # A schema-qualified request is keyed by the requested string, not the relname. + assert_equal %w[id], multi["#{SCHEMA_NAME}.#{PK_TABLE_NAME}"] + ensure + @connection.drop_table :pg_pk_multi_a, if_exists: true + @connection.drop_table :pg_pk_multi_b, if_exists: true + @connection.drop_table :pg_pk_multi_c, if_exists: true + @connection.drop_table :pg_pk_multi_none, if_exists: true + end + def test_with_schema_prefixed_table_name assert_nothing_raised do assert_equal COLUMNS, columns("#{SCHEMA_NAME}.#{TABLE_NAME}") From c4fbe339b87bdb7dc16868f91924b1e8386b5252 Mon Sep 17 00:00:00 2001 From: Hartley McGuire Date: Tue, 21 Jul 2026 16:11:53 -0400 Subject: [PATCH 09/10] Batch column queries for PostgreSQL schema cache dump `SchemaCache#add_all` was still calling `columns` once per table on PostgreSQL. Override `columns_for_tables` to fetch every requested table's columns in one query, and route `add_all` through it (mirroring the MySQL batch). The batch lives in `column_definitions_for_tables`, which reads `pg_attribute` for all the tables at once, scoping `a.attrelid` with `to_regclass` (as `primary_keys_for_tables` does) and ordering by `attrelid, attnum` so each table's columns come back in order. Rows are keyed by the requested table-name string, so schema-qualified requests round-trip through `add_all`'s `.fetch(table.to_s)`. `columns_for_tables` maps each table's fields to columns with the existing `new_column_from_field`. `column_definitions` now delegates to `column_definitions_for_tables` for a single table. Because `to_regclass` returns NULL for a missing relation instead of raising, and the batch pre-seeds an empty array for every requested name, an empty result no longer distinguishes a missing table from a zero-column one (PostgreSQL permits `CREATE TABLE t ()`). `column_definitions` therefore checks `to_regclass` resolution and raises `StatementInvalid` only when the relation is actually missing, preserving the previous behavior for both missing and zero-column tables. --- .../postgresql/schema_statements.rb | 9 +++++ .../connection_adapters/postgresql_adapter.rb | 32 ++++++++++++++-- .../cases/adapters/postgresql/schema_test.rb | 37 +++++++++++++++++++ 3 files changed, 74 insertions(+), 4 deletions(-) diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/schema_statements.rb b/activerecord/lib/active_record/connection_adapters/postgresql/schema_statements.rb index ecb3d5e6d035c..c941f174a99ab 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/schema_statements.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/schema_statements.rb @@ -615,6 +615,15 @@ def primary_keys_for_tables(table_names) # :nodoc: result_by_table end + def columns_for_tables(table_names) # :nodoc: + definitions_by_table = column_definitions_for_tables(table_names) + definitions_by_table.each_with_object({}) do |(table, definitions), hash| + hash[table] = definitions.map do |field| + new_column_from_field(table, field, definitions) + end + end + end + # Renames a table. # Also renames a table's primary key sequence if the sequence name exists and # matches the Active Record default. diff --git a/activerecord/lib/active_record/connection_adapters/postgresql_adapter.rb b/activerecord/lib/active_record/connection_adapters/postgresql_adapter.rb index 2299a6204a238..c4e454f911f38 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql_adapter.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql_adapter.rb @@ -1259,20 +1259,44 @@ def internal_set_config(setting, value) # - format_type includes the column size constraint, e.g. varchar(50) # - ::regclass is a function that gives the id for a table name def column_definitions(table_name) - query_rows(<<~SQL) - SELECT a.attname, format_type(a.atttypid, a.atttypmod), + fields = column_definitions_for_tables([table_name])[table_name.to_s] + if fields.empty? && query_value("SELECT to_regclass(#{quote(quote_table_name(table_name))})").nil? + raise ActiveRecord::StatementInvalid.new("Could not find table '#{table_name}'", connection_pool: @pool) + end + fields + end + + def column_definitions_for_tables(table_names) # :nodoc: + return {} if table_names.empty? + + result_by_table = table_names.each_with_object({}) do |name, h| + h[name.to_s] = [] + end + key_for = table_key_lookup(table_names) + + result = query_rows(<<~SQL) + SELECT n.nspname, k.relname, a.attname, format_type(a.atttypid, a.atttypmod), pg_get_expr(d.adbin, d.adrelid), a.attnotnull, a.atttypid, a.atttypmod, c.collname, col_description(a.attrelid, a.attnum) AS comment, #{supports_identity_columns? ? 'attidentity' : quote('')} AS identity, #{supports_virtual_columns? ? 'attgenerated' : quote('')} as attgenerated FROM pg_attribute a + INNER JOIN pg_class k ON k.oid = a.attrelid + INNER JOIN pg_namespace n ON n.oid = k.relnamespace LEFT JOIN pg_attrdef d ON a.attrelid = d.adrelid AND a.attnum = d.adnum LEFT JOIN pg_type t ON a.atttypid = t.oid LEFT JOIN pg_collation c ON a.attcollation = c.oid AND a.attcollation <> t.typcollation - WHERE a.attrelid = #{quote(quote_table_name(table_name))}::regclass - AND a.attnum > 0 AND NOT a.attisdropped + WHERE a.attnum > 0 AND NOT a.attisdropped + AND a.attrelid = #{to_regclass_array_sql(table_names)} ORDER BY a.attnum SQL + + result.each do |row| + key = key_for.call(row[0], row[1]) + result_by_table[key] << row[2..] if key + end + + result_by_table end def arel_visitor diff --git a/activerecord/test/cases/adapters/postgresql/schema_test.rb b/activerecord/test/cases/adapters/postgresql/schema_test.rb index 52a6165349667..f694bc3500eb5 100644 --- a/activerecord/test/cases/adapters/postgresql/schema_test.rb +++ b/activerecord/test/cases/adapters/postgresql/schema_test.rb @@ -325,6 +325,43 @@ def test_primary_keys_for_multiple_tables @connection.drop_table :pg_pk_multi_none, if_exists: true end + def test_columns_for_multiple_tables + @connection.create_table(:pg_cols_multi_a) { |t| t.string :title; t.integer :counter } + @connection.create_table(:pg_cols_multi_b) { |t| t.string :name; t.text :body } + + # A single table name returns an Array of columns (backward compatible). + assert_kind_of Array, @connection.columns("pg_cols_multi_a") + + # columns_for_tables returns a Hash of table name => Array of columns, + # matching #columns for each table (names and sql_types). + multi = @connection.columns_for_tables(["pg_cols_multi_a", "pg_cols_multi_b", "#{SCHEMA_NAME}.#{TABLE_NAME}"]) + assert_kind_of Hash, multi + assert_equal %w[pg_cols_multi_a pg_cols_multi_b test_schema.things], multi.keys.sort + assert_equal 6, multi["#{SCHEMA_NAME}.#{TABLE_NAME}"].size + + profile = ->(cols) { cols.map { |c| [c.name, c.sql_type] } } + assert_equal profile.call(@connection.columns("pg_cols_multi_a")), profile.call(multi["pg_cols_multi_a"]) + assert_equal profile.call(@connection.columns("pg_cols_multi_b")), profile.call(multi["pg_cols_multi_b"]) + # A schema-qualified request is keyed by the requested string and matches #columns. + assert_equal profile.call(@connection.columns("#{SCHEMA_NAME}.#{TABLE_NAME}")), profile.call(multi["#{SCHEMA_NAME}.#{TABLE_NAME}"]) + ensure + @connection.drop_table :pg_cols_multi_a, if_exists: true + @connection.drop_table :pg_cols_multi_b, if_exists: true + end + + def test_columns_for_zero_column_table + @connection.execute 'CREATE TABLE "pg_zero_col" ()' + assert_equal [], @connection.columns("pg_zero_col") + ensure + @connection.drop_table :pg_zero_col, if_exists: true + end + + def test_columns_raises_for_missing_table + assert_raises(ActiveRecord::StatementInvalid) do + @connection.columns("pg_does_not_exist") + end + end + def test_with_schema_prefixed_table_name assert_nothing_raised do assert_equal COLUMNS, columns("#{SCHEMA_NAME}.#{TABLE_NAME}") From 6f84b265a2915502f8df1fbc35c4aedeee27b428 Mon Sep 17 00:00:00 2001 From: Hartley McGuire Date: Wed, 22 Jul 2026 14:07:06 -0400 Subject: [PATCH 10/10] Batch index queries for PostgreSQL schema cache dump Building a schema cache dump issues one query per table for indexes, columns, and primary keys. The columns and primary keys queries were already batched for PostgreSQL; indexes were not, so dumping a cache for a database with many tables paid a round-trip per table just to read indexes. Add indexes_for_tables, which fetches indexes for several tables in one query by scoping pg_class/pg_namespace to the requested names. It reuses the same WHERE shape as indexes (relname + search-path namespace for unqualified names, relname + pinned namespace for qualified names), so the existing multi-schema search-path behavior is preserved exactly. The single-table indexes method now delegates to indexes_for_tables, and the row-to-IndexDefinition logic is extracted into a private helper so both paths share it. Indexes are keyed by the requested table-name string, matching how SchemaCache#add_all looks them up, so schema-qualified names round-trip correctly. --- .../postgresql/schema_statements.rb | 141 +++++++++++------- .../cases/adapters/postgresql/schema_test.rb | 34 +++++ 2 files changed, 123 insertions(+), 52 deletions(-) diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/schema_statements.rb b/activerecord/lib/active_record/connection_adapters/postgresql/schema_statements.rb index c941f174a99ab..d880f9c374f58 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/schema_statements.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/schema_statements.rb @@ -96,10 +96,19 @@ def index_name_exists?(table_name, index_name) # Returns an array of indexes for the given table. def indexes(table_name) # :nodoc: - scope = quoted_scope(table_name) + indexes_for_tables([table_name])[table_name.to_s] + end + + def indexes_for_tables(table_names) # :nodoc: + return {} if table_names.empty? + + result_by_table = table_names.each_with_object({}) do |name, h| + h[name.to_s] = [] + end + key_for = table_key_lookup(table_names) result = query_rows(<<~SQL) - SELECT distinct i.relname, d.indisunique, d.indkey, pg_get_indexdef(d.indexrelid), + SELECT distinct n.nspname, t.relname, i.relname, d.indisunique, d.indkey, pg_get_indexdef(d.indexrelid), pg_catalog.obj_description(i.oid, 'pg_class') AS comment, d.indisvalid, ARRAY( SELECT pg_get_indexdef(d.indexrelid, k + 1, true) @@ -112,60 +121,16 @@ def indexes(table_name) # :nodoc: LEFT JOIN pg_namespace n ON n.oid = t.relnamespace WHERE i.relkind IN ('i', 'I') AND d.indisprimary = 'f' - AND t.relname = #{scope[:name]} - AND n.nspname = #{scope[:schema]} + AND (#{table_scope_sql(table_names)}) ORDER BY i.relname SQL - result.map do |row| - index_name = row[0] - unique = row[1] - indkey = row[2].split(" ").map(&:to_i) - inddef = row[3] - comment = row[4] - valid = row[5] - columns = decode_string_array(row[6]).map { |c| Utils.unquote_identifier(c.strip.gsub('""', '"')) } - - using, expressions, include, nulls_not_distinct, where = inddef.scan(/ USING (\w+?) \((.+?)\)(?: INCLUDE \((.+?)\))?( NULLS NOT DISTINCT)?(?: WHERE (.+))?\z/m).flatten - - orders = {} - opclasses = {} - include_columns = include ? include.split(",").map { |c| Utils.unquote_identifier(c.strip.gsub('""', '"')) } : [] - - if indkey.include?(0) - columns = expressions - else - # prevent INCLUDE columns from being matched - columns.reject! { |c| include_columns.include?(c) } - - # add info on sort order (only desc order is explicitly specified, asc is the default) - # and non-default opclasses - expressions.scan(/(?\w+)"?\s?(?(?:\w+\.)?\w+_ops(_\w+)?)?\s?(?DESC)?\s?(?NULLS (?:FIRST|LAST))?/).each do |column, opclass, desc, nulls| - opclasses[column] = opclass.split(".").last.to_sym if opclass - - if nulls - orders[column] = [desc, nulls].compact.join(" ") - else - orders[column] = :desc if desc - end - end - end - - IndexDefinition.new( - table_name, - index_name, - unique, - columns, - orders: orders, - opclasses: opclasses, - where: where, - using: using.to_sym, - include: include_columns.presence, - nulls_not_distinct: nulls_not_distinct.present?, - comment: comment.presence, - valid: valid - ) + result.each do |row| + key = key_for.call(row[0], row[1]) + result_by_table[key] << index_from_row(row[2..], key) if key end + + result_by_table end def table_options(table_name) # :nodoc: @@ -1469,6 +1434,78 @@ def table_key_lookup(table_names) ->(nsp, rel) { qualified[[nsp, rel]] || unqualified[rel] } end + # Build a WHERE clause scoping pg_class (t) / pg_namespace (n) to the + # given table names, grouped by schema: unqualified names resolve + # against the search path (matching every schema on it that has the + # relation, as +#indexes+ does), qualified names pin their schema. + # Used for relation-name-based queries where matching all search-path + # schemas is the existing behavior. + def table_scope_sql(table_names) + by_schema = table_names.each_with_object(Hash.new { |h, k| h[k] = [] }) do |name, h| + schema, tbl = extract_schema_qualified_name(name) + h[schema] << tbl + end + + by_schema.map do |schema, names| + schema_sql = schema ? quote(schema) : "ANY (current_schemas(false))" + names_sql = names.map { |n| quote(n) }.join(", ") + "(n.nspname = #{schema_sql} AND t.relname IN (#{names_sql}))" + end.join(" OR ") + end + + # Turns one +#indexes_for_tables+ row — the seven columns after the + # leading namespace/relname key columns — into an IndexDefinition, + # attributed to +table_name+ (the requested table-name string). + def index_from_row(row, table_name) + index_name = row[0] + unique = row[1] + indkey = row[2].split(" ").map(&:to_i) + inddef = row[3] + comment = row[4] + valid = row[5] + columns = decode_string_array(row[6]).map { |c| Utils.unquote_identifier(c.strip.gsub('""', '"')) } + + using, expressions, include, nulls_not_distinct, where = inddef.scan(/ USING (\w+?) \((.+?)\)(?: INCLUDE \((.+?)\))?( NULLS NOT DISTINCT)?(?: WHERE (.+))?\z/m).flatten + + orders = {} + opclasses = {} + include_columns = include ? include.split(",").map { |c| Utils.unquote_identifier(c.strip.gsub('""', '"')) } : [] + + if indkey.include?(0) + columns = expressions + else + # prevent INCLUDE columns from being matched + columns.reject! { |c| include_columns.include?(c) } + + # add info on sort order (only desc order is explicitly specified, asc is the default) + # and non-default opclasses + expressions.scan(/(?\w+)"?\s?(?(?:\w+\.)?\w+_ops(_\w+)?)?\s?(?DESC)?\s?(?NULLS (?:FIRST|LAST))?/).each do |column, opclass, desc, nulls| + opclasses[column] = opclass.split(".").last.to_sym if opclass + + if nulls + orders[column] = [desc, nulls].compact.join(" ") + else + orders[column] = :desc if desc + end + end + end + + IndexDefinition.new( + table_name, + index_name, + unique, + columns, + orders: orders, + opclasses: opclasses, + where: where, + using: using.to_sym, + include: include_columns.presence, + nulls_not_distinct: nulls_not_distinct.present?, + comment: comment.presence, + valid: valid + ) + end + def decode_string_array(value) PG::TextDecoder::Array.new.decode(value) end diff --git a/activerecord/test/cases/adapters/postgresql/schema_test.rb b/activerecord/test/cases/adapters/postgresql/schema_test.rb index f694bc3500eb5..f3298a7e54073 100644 --- a/activerecord/test/cases/adapters/postgresql/schema_test.rb +++ b/activerecord/test/cases/adapters/postgresql/schema_test.rb @@ -362,6 +362,40 @@ def test_columns_raises_for_missing_table end end + def test_indexes_for_multiple_tables + @connection.create_table(:pg_idx_multi_a) { |t| t.string :email; t.string :name } + @connection.create_table(:pg_idx_multi_b) { |t| t.string :email; t.string :other } + @connection.add_index :pg_idx_multi_a, :name, name: "idx_a_name" + @connection.add_index :pg_idx_multi_a, :email, name: "idx_a_email" + @connection.add_index :pg_idx_multi_b, :email, name: "idx_b_email" + @connection.add_index :pg_idx_multi_b, :other, name: "idx_b_other" + + # A single table name returns an Array of indexes (backward compatible). + single = @connection.indexes("pg_idx_multi_a") + assert_kind_of Array, single + assert_equal %w[idx_a_email idx_a_name], single.map(&:name).sort + + # indexes_for_tables returns a Hash of table name => Array of indexes. + multi = @connection.indexes_for_tables(["pg_idx_multi_a", "pg_idx_multi_b", "#{SCHEMA_NAME}.#{TABLE_NAME}"]) + assert_kind_of Hash, multi + assert_equal %w[pg_idx_multi_a pg_idx_multi_b test_schema.things], multi.keys.sort + assert multi.values.all?(Array) + assert_equal %w[idx_a_email idx_a_name], multi["pg_idx_multi_a"].map(&:name).sort + assert_equal %w[idx_b_email idx_b_other], multi["pg_idx_multi_b"].map(&:name).sort + + # Each index is attributed to its requested table name. + a = multi["pg_idx_multi_a"].find { |i| i.name == "idx_a_email" } + assert_equal %w[email], a.columns + assert_equal "pg_idx_multi_a", a.table + + # A schema-qualified request is keyed by the requested string and matches #indexes. + assert_equal @connection.indexes("#{SCHEMA_NAME}.#{TABLE_NAME}").map(&:name).sort, + multi["#{SCHEMA_NAME}.#{TABLE_NAME}"].map(&:name).sort + ensure + @connection.drop_table :pg_idx_multi_a, if_exists: true + @connection.drop_table :pg_idx_multi_b, if_exists: true + end + def test_with_schema_prefixed_table_name assert_nothing_raised do assert_equal COLUMNS, columns("#{SCHEMA_NAME}.#{TABLE_NAME}")