Fix null values being included in some indexes (#17711)

* Fix null values being included in some indexes

* Update lib/mastodon/migration_helpers.rb

Co-authored-by: Claire <claire.github-309c@sitedethib.com>

* Add documentation link to corruption error message

Co-authored-by: Claire <claire.github-309c@sitedethib.com>
This commit is contained in:
Eugen Rochko 2022-03-12 08:12:57 +01:00 committed by GitHub
parent bc320d6cec
commit 75e33fd08f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
23 changed files with 354 additions and 37 deletions

View file

@ -16,7 +16,7 @@ class AddFixedLowercaseIndexToAccounts < ActiveRecord::Migration[5.2]
add_index :accounts, "lower (username), COALESCE(lower(domain), '')", name: 'index_accounts_on_username_and_domain_lower', unique: true, algorithm: :concurrently
rescue ActiveRecord::RecordNotUnique
remove_index :accounts, name: 'index_accounts_on_username_and_domain_lower'
raise CorruptionError
raise CorruptionError.new('index_accounts_on_username_and_domain_lower')
end
remove_index :accounts, name: 'old_index_accounts_on_username_and_domain_lower' if index_name_exists?(:accounts, 'old_index_accounts_on_username_and_domain_lower')

View file

@ -10,7 +10,7 @@ class AddCaseInsensitiveBtreeIndexToTags < ActiveRecord::Migration[5.2]
safety_assured { execute 'CREATE UNIQUE INDEX CONCURRENTLY index_tags_on_name_lower_btree ON tags (lower(name) text_pattern_ops)' }
rescue ActiveRecord::StatementInvalid => e
remove_index :tags, name: 'index_tags_on_name_lower_btree'
raise CorruptionError if e.is_a?(ActiveRecord::RecordNotUnique)
raise CorruptionError.new('index_tags_on_name_lower_btree') if e.is_a?(ActiveRecord::RecordNotUnique)
raise e
end

View file

@ -0,0 +1,17 @@
# frozen_string_literal: true
require Rails.root.join('lib', 'mastodon', 'migration_helpers')
class OptimizeNullIndexConversationsUri < ActiveRecord::Migration[5.2]
include Mastodon::MigrationHelpers
disable_ddl_transaction!
def up
update_index :conversations, 'index_conversations_on_uri', :uri, unique: true, where: 'uri IS NOT NULL', opclass: :text_pattern_ops
end
def down
update_index :conversations, 'index_conversations_on_uri', :uri, unique: true
end
end

View file

@ -0,0 +1,17 @@
# frozen_string_literal: true
require Rails.root.join('lib', 'mastodon', 'migration_helpers')
class OptimizeNullIndexStatusesInReplyToAccountId < ActiveRecord::Migration[5.2]
include Mastodon::MigrationHelpers
disable_ddl_transaction!
def up
update_index :statuses, 'index_statuses_on_in_reply_to_account_id', :in_reply_to_account_id, where: 'in_reply_to_account_id IS NOT NULL'
end
def down
update_index :statuses, 'index_statuses_on_in_reply_to_account_id', :in_reply_to_account_id
end
end

View file

@ -0,0 +1,17 @@
# frozen_string_literal: true
require Rails.root.join('lib', 'mastodon', 'migration_helpers')
class OptimizeNullIndexStatusesInReplyToId < ActiveRecord::Migration[5.2]
include Mastodon::MigrationHelpers
disable_ddl_transaction!
def up
update_index :statuses, 'index_statuses_on_in_reply_to_id', :in_reply_to_id, where: 'in_reply_to_id IS NOT NULL'
end
def down
update_index :statuses, 'index_statuses_on_in_reply_to_id', :in_reply_to_id
end
end

View file

@ -0,0 +1,17 @@
# frozen_string_literal: true
require Rails.root.join('lib', 'mastodon', 'migration_helpers')
class OptimizeNullIndexMediaAttachmentsScheduledStatusId < ActiveRecord::Migration[5.2]
include Mastodon::MigrationHelpers
disable_ddl_transaction!
def up
update_index :media_attachments, 'index_media_attachments_on_scheduled_status_id', :scheduled_status_id, where: 'scheduled_status_id IS NOT NULL'
end
def down
update_index :media_attachments, 'index_media_attachments_on_scheduled_status_id', :scheduled_status_id
end
end

View file

@ -0,0 +1,17 @@
# frozen_string_literal: true
require Rails.root.join('lib', 'mastodon', 'migration_helpers')
class OptimizeNullIndexMediaAttachmentsShortcode < ActiveRecord::Migration[5.2]
include Mastodon::MigrationHelpers
disable_ddl_transaction!
def up
update_index :media_attachments, 'index_media_attachments_on_shortcode', :shortcode, unique: true, where: 'shortcode IS NOT NULL', opclass: :text_pattern_ops
end
def down
update_index :media_attachments, 'index_media_attachments_on_shortcode', :shortcode, unique: true
end
end

View file

@ -0,0 +1,17 @@
# frozen_string_literal: true
require Rails.root.join('lib', 'mastodon', 'migration_helpers')
class OptimizeNullIndexUsersResetPasswordToken < ActiveRecord::Migration[5.2]
include Mastodon::MigrationHelpers
disable_ddl_transaction!
def up
update_index :users, 'index_users_on_reset_password_token', :reset_password_token, unique: true, where: 'reset_password_token IS NOT NULL', opclass: :text_pattern_ops
end
def down
update_index :users, 'index_users_on_reset_password_token', :reset_password_token, unique: true
end
end

View file

@ -0,0 +1,17 @@
# frozen_string_literal: true
require Rails.root.join('lib', 'mastodon', 'migration_helpers')
class OptimizeNullIndexUsersCreatedByApplicationId < ActiveRecord::Migration[5.2]
include Mastodon::MigrationHelpers
disable_ddl_transaction!
def up
update_index :users, 'index_users_on_created_by_application_id', :created_by_application_id, where: 'created_by_application_id IS NOT NULL'
end
def down
update_index :users, 'index_users_on_created_by_application_id', :created_by_application_id
end
end

View file

@ -0,0 +1,17 @@
# frozen_string_literal: true
require Rails.root.join('lib', 'mastodon', 'migration_helpers')
class OptimizeNullIndexStatusesUri < ActiveRecord::Migration[5.2]
include Mastodon::MigrationHelpers
disable_ddl_transaction!
def up
update_index :statuses, 'index_statuses_on_uri', :uri, unique: true, where: 'uri IS NOT NULL', opclass: :text_pattern_ops
end
def down
update_index :statuses, 'index_statuses_on_uri', :uri, unique: true
end
end

View file

@ -0,0 +1,17 @@
# frozen_string_literal: true
require Rails.root.join('lib', 'mastodon', 'migration_helpers')
class OptimizeNullIndexAccountsMovedToAccountId < ActiveRecord::Migration[5.2]
include Mastodon::MigrationHelpers
disable_ddl_transaction!
def up
update_index :accounts, 'index_accounts_on_moved_to_account_id', :moved_to_account_id, where: 'moved_to_account_id IS NOT NULL'
end
def down
update_index :accounts, 'index_accounts_on_moved_to_account_id', :moved_to_account_id
end
end

View file

@ -0,0 +1,17 @@
# frozen_string_literal: true
require Rails.root.join('lib', 'mastodon', 'migration_helpers')
class OptimizeNullIndexOauthAccessTokensRefreshToken < ActiveRecord::Migration[5.2]
include Mastodon::MigrationHelpers
disable_ddl_transaction!
def up
update_index :oauth_access_tokens, 'index_oauth_access_tokens_on_refresh_token', :refresh_token, unique: true, where: 'refresh_token IS NOT NULL', opclass: :text_pattern_ops
end
def down
update_index :oauth_access_tokens, 'index_oauth_access_tokens_on_refresh_token', :refresh_token, unique: true
end
end

View file

@ -0,0 +1,17 @@
# frozen_string_literal: true
require Rails.root.join('lib', 'mastodon', 'migration_helpers')
class OptimizeNullIndexAccountsURL < ActiveRecord::Migration[5.2]
include Mastodon::MigrationHelpers
disable_ddl_transaction!
def up
update_index :accounts, 'index_accounts_on_url', :url, where: 'url IS NOT NULL', opclass: :text_pattern_ops
end
def down
update_index :accounts, 'index_accounts_on_url', :url
end
end

View file

@ -0,0 +1,17 @@
# frozen_string_literal: true
require Rails.root.join('lib', 'mastodon', 'migration_helpers')
class OptimizeNullIndexOauthAccessTokensResourceOwnerId < ActiveRecord::Migration[5.2]
include Mastodon::MigrationHelpers
disable_ddl_transaction!
def up
update_index :oauth_access_tokens, 'index_oauth_access_tokens_on_resource_owner_id', :resource_owner_id, where: 'resource_owner_id IS NOT NULL'
end
def down
update_index :oauth_access_tokens, 'index_oauth_access_tokens_on_resource_owner_id', :resource_owner_id
end
end

View file

@ -0,0 +1,17 @@
# frozen_string_literal: true
require Rails.root.join('lib', 'mastodon', 'migration_helpers')
class OptimizeNullIndexAnnouncementReactionsCustomEmojiId < ActiveRecord::Migration[5.2]
include Mastodon::MigrationHelpers
disable_ddl_transaction!
def up
update_index :announcement_reactions, 'index_announcement_reactions_on_custom_emoji_id', :custom_emoji_id, where: 'custom_emoji_id IS NOT NULL'
end
def down
update_index :announcement_reactions, 'index_announcement_reactions_on_custom_emoji_id', :custom_emoji_id
end
end

View file

@ -0,0 +1,17 @@
# frozen_string_literal: true
require Rails.root.join('lib', 'mastodon', 'migration_helpers')
class OptimizeNullIndexAppealsApprovedByAccountId < ActiveRecord::Migration[5.2]
include Mastodon::MigrationHelpers
disable_ddl_transaction!
def up
update_index :appeals, 'index_appeals_on_approved_by_account_id', :approved_by_account_id, where: 'approved_by_account_id IS NOT NULL'
end
def down
update_index :appeals, 'index_appeals_on_approved_by_account_id', :approved_by_account_id
end
end

View file

@ -0,0 +1,17 @@
# frozen_string_literal: true
require Rails.root.join('lib', 'mastodon', 'migration_helpers')
class OptimizeNullIndexAccountMigrationsTargetAccountId < ActiveRecord::Migration[5.2]
include Mastodon::MigrationHelpers
disable_ddl_transaction!
def up
update_index :account_migrations, 'index_account_migrations_on_target_account_id', :target_account_id, where: 'target_account_id IS NOT NULL'
end
def down
update_index :account_migrations, 'index_account_migrations_on_target_account_id', :target_account_id
end
end

View file

@ -0,0 +1,17 @@
# frozen_string_literal: true
require Rails.root.join('lib', 'mastodon', 'migration_helpers')
class OptimizeNullIndexAppealsRejectedByAccountId < ActiveRecord::Migration[5.2]
include Mastodon::MigrationHelpers
disable_ddl_transaction!
def up
update_index :appeals, 'index_appeals_on_rejected_by_account_id', :rejected_by_account_id, where: 'rejected_by_account_id IS NOT NULL'
end
def down
update_index :appeals, 'index_appeals_on_rejected_by_account_id', :rejected_by_account_id
end
end

View file

@ -0,0 +1,17 @@
# frozen_string_literal: true
require Rails.root.join('lib', 'mastodon', 'migration_helpers')
class OptimizeNullIndexListAccountsFollowId < ActiveRecord::Migration[5.2]
include Mastodon::MigrationHelpers
disable_ddl_transaction!
def up
update_index :list_accounts, 'index_list_accounts_on_follow_id', :follow_id, where: 'follow_id IS NOT NULL'
end
def down
update_index :list_accounts, 'index_list_accounts_on_follow_id', :follow_id
end
end

View file

@ -0,0 +1,17 @@
# frozen_string_literal: true
require Rails.root.join('lib', 'mastodon', 'migration_helpers')
class OptimizeNullIndexWebPushSubscriptionsAccessTokenId < ActiveRecord::Migration[5.2]
include Mastodon::MigrationHelpers
disable_ddl_transaction!
def up
update_index :web_push_subscriptions, 'index_web_push_subscriptions_on_access_token_id', :access_token_id, where: 'access_token_id IS NOT NULL'
end
def down
update_index :web_push_subscriptions, 'index_web_push_subscriptions_on_access_token_id', :access_token_id
end
end

View file

@ -10,7 +10,7 @@
#
# It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema.define(version: 2022_03_09_213005) do
ActiveRecord::Schema.define(version: 2022_03_10_060959) do
# These are extensions that must be enabled in order to support this database
enable_extension "plpgsql"
@ -59,7 +59,7 @@ ActiveRecord::Schema.define(version: 2022_03_09_213005) do
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
t.index ["account_id"], name: "index_account_migrations_on_account_id"
t.index ["target_account_id"], name: "index_account_migrations_on_target_account_id"
t.index ["target_account_id"], name: "index_account_migrations_on_target_account_id", where: "(target_account_id IS NOT NULL)"
end
create_table "account_moderation_notes", force: :cascade do |t|
@ -188,9 +188,9 @@ ActiveRecord::Schema.define(version: 2022_03_09_213005) do
t.datetime "requested_review_at"
t.index "(((setweight(to_tsvector('simple'::regconfig, (display_name)::text), 'A'::\"char\") || setweight(to_tsvector('simple'::regconfig, (username)::text), 'B'::\"char\")) || setweight(to_tsvector('simple'::regconfig, (COALESCE(domain, ''::character varying))::text), 'C'::\"char\")))", name: "search_index", using: :gin
t.index "lower((username)::text), COALESCE(lower((domain)::text), ''::text)", name: "index_accounts_on_username_and_domain_lower", unique: true
t.index ["moved_to_account_id"], name: "index_accounts_on_moved_to_account_id"
t.index ["moved_to_account_id"], name: "index_accounts_on_moved_to_account_id", where: "(moved_to_account_id IS NOT NULL)"
t.index ["uri"], name: "index_accounts_on_uri"
t.index ["url"], name: "index_accounts_on_url"
t.index ["url"], name: "index_accounts_on_url", opclass: :text_pattern_ops, where: "(url IS NOT NULL)"
end
create_table "accounts_tags", id: false, force: :cascade do |t|
@ -230,7 +230,7 @@ ActiveRecord::Schema.define(version: 2022_03_09_213005) do
t.datetime "updated_at", null: false
t.index ["account_id", "announcement_id", "name"], name: "index_announcement_reactions_on_account_id_and_announcement_id", unique: true
t.index ["announcement_id"], name: "index_announcement_reactions_on_announcement_id"
t.index ["custom_emoji_id"], name: "index_announcement_reactions_on_custom_emoji_id"
t.index ["custom_emoji_id"], name: "index_announcement_reactions_on_custom_emoji_id", where: "(custom_emoji_id IS NOT NULL)"
end
create_table "announcements", force: :cascade do |t|
@ -258,8 +258,8 @@ ActiveRecord::Schema.define(version: 2022_03_09_213005) do
t.datetime "updated_at", precision: 6, null: false
t.index ["account_id"], name: "index_appeals_on_account_id"
t.index ["account_warning_id"], name: "index_appeals_on_account_warning_id", unique: true
t.index ["approved_by_account_id"], name: "index_appeals_on_approved_by_account_id"
t.index ["rejected_by_account_id"], name: "index_appeals_on_rejected_by_account_id"
t.index ["approved_by_account_id"], name: "index_appeals_on_approved_by_account_id", where: "(approved_by_account_id IS NOT NULL)"
t.index ["rejected_by_account_id"], name: "index_appeals_on_rejected_by_account_id", where: "(rejected_by_account_id IS NOT NULL)"
end
create_table "backups", force: :cascade do |t|
@ -311,7 +311,7 @@ ActiveRecord::Schema.define(version: 2022_03_09_213005) do
t.string "uri"
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
t.index ["uri"], name: "index_conversations_on_uri", unique: true
t.index ["uri"], name: "index_conversations_on_uri", unique: true, opclass: :text_pattern_ops, where: "(uri IS NOT NULL)"
end
create_table "custom_emoji_categories", force: :cascade do |t|
@ -509,7 +509,7 @@ ActiveRecord::Schema.define(version: 2022_03_09_213005) do
t.bigint "account_id", null: false
t.bigint "follow_id"
t.index ["account_id", "list_id"], name: "index_list_accounts_on_account_id_and_list_id", unique: true
t.index ["follow_id"], name: "index_list_accounts_on_follow_id"
t.index ["follow_id"], name: "index_list_accounts_on_follow_id", where: "(follow_id IS NOT NULL)"
t.index ["list_id", "account_id"], name: "index_list_accounts_on_list_id_and_account_id"
end
@ -568,8 +568,8 @@ ActiveRecord::Schema.define(version: 2022_03_09_213005) do
t.datetime "thumbnail_updated_at"
t.string "thumbnail_remote_url"
t.index ["account_id", "status_id"], name: "index_media_attachments_on_account_id_and_status_id", order: { status_id: :desc }
t.index ["scheduled_status_id"], name: "index_media_attachments_on_scheduled_status_id"
t.index ["shortcode"], name: "index_media_attachments_on_shortcode", unique: true
t.index ["scheduled_status_id"], name: "index_media_attachments_on_scheduled_status_id", where: "(scheduled_status_id IS NOT NULL)"
t.index ["shortcode"], name: "index_media_attachments_on_shortcode", unique: true, opclass: :text_pattern_ops, where: "(shortcode IS NOT NULL)"
t.index ["status_id"], name: "index_media_attachments_on_status_id"
end
@ -631,8 +631,8 @@ ActiveRecord::Schema.define(version: 2022_03_09_213005) do
t.bigint "resource_owner_id"
t.datetime "last_used_at"
t.inet "last_used_ip"
t.index ["refresh_token"], name: "index_oauth_access_tokens_on_refresh_token", unique: true
t.index ["resource_owner_id"], name: "index_oauth_access_tokens_on_resource_owner_id"
t.index ["refresh_token"], name: "index_oauth_access_tokens_on_refresh_token", unique: true, opclass: :text_pattern_ops, where: "(refresh_token IS NOT NULL)"
t.index ["resource_owner_id"], name: "index_oauth_access_tokens_on_resource_owner_id", where: "(resource_owner_id IS NOT NULL)"
t.index ["token"], name: "index_oauth_access_tokens_on_token", unique: true
end
@ -899,10 +899,10 @@ ActiveRecord::Schema.define(version: 2022_03_09_213005) do
t.index ["deleted_at"], name: "index_statuses_on_deleted_at", where: "(deleted_at IS NOT NULL)"
t.index ["id", "account_id"], name: "index_statuses_local_20190824", order: { id: :desc }, where: "((local OR (uri IS NULL)) AND (deleted_at IS NULL) AND (visibility = 0) AND (reblog_of_id IS NULL) AND ((NOT reply) OR (in_reply_to_account_id = account_id)))"
t.index ["id", "account_id"], name: "index_statuses_public_20200119", order: { id: :desc }, where: "((deleted_at IS NULL) AND (visibility = 0) AND (reblog_of_id IS NULL) AND ((NOT reply) OR (in_reply_to_account_id = account_id)))"
t.index ["in_reply_to_account_id"], name: "index_statuses_on_in_reply_to_account_id"
t.index ["in_reply_to_id"], name: "index_statuses_on_in_reply_to_id"
t.index ["in_reply_to_account_id"], name: "index_statuses_on_in_reply_to_account_id", where: "(in_reply_to_account_id IS NOT NULL)"
t.index ["in_reply_to_id"], name: "index_statuses_on_in_reply_to_id", where: "(in_reply_to_id IS NOT NULL)"
t.index ["reblog_of_id", "account_id"], name: "index_statuses_on_reblog_of_id_and_account_id"
t.index ["uri"], name: "index_statuses_on_uri", unique: true
t.index ["uri"], name: "index_statuses_on_uri", unique: true, opclass: :text_pattern_ops, where: "(uri IS NOT NULL)"
end
create_table "statuses_tags", id: false, force: :cascade do |t|
@ -996,9 +996,9 @@ ActiveRecord::Schema.define(version: 2022_03_09_213005) do
t.boolean "skip_sign_in_token"
t.index ["account_id"], name: "index_users_on_account_id"
t.index ["confirmation_token"], name: "index_users_on_confirmation_token", unique: true
t.index ["created_by_application_id"], name: "index_users_on_created_by_application_id"
t.index ["created_by_application_id"], name: "index_users_on_created_by_application_id", where: "(created_by_application_id IS NOT NULL)"
t.index ["email"], name: "index_users_on_email", unique: true
t.index ["reset_password_token"], name: "index_users_on_reset_password_token", unique: true
t.index ["reset_password_token"], name: "index_users_on_reset_password_token", unique: true, opclass: :text_pattern_ops, where: "(reset_password_token IS NOT NULL)"
end
create_table "web_push_subscriptions", force: :cascade do |t|
@ -1010,7 +1010,7 @@ ActiveRecord::Schema.define(version: 2022_03_09_213005) do
t.datetime "updated_at", null: false
t.bigint "access_token_id"
t.bigint "user_id"
t.index ["access_token_id"], name: "index_web_push_subscriptions_on_access_token_id"
t.index ["access_token_id"], name: "index_web_push_subscriptions_on_access_token_id", where: "(access_token_id IS NOT NULL)"
t.index ["user_id"], name: "index_web_push_subscriptions_on_user_id"
end

View file

@ -42,8 +42,14 @@
module Mastodon
module MigrationHelpers
class CorruptionError < StandardError
def initialize(message = nil)
super(message.presence || 'Migration failed because of index corruption, see https://docs.joinmastodon.org/admin/troubleshooting/index-corruption/#fixing')
attr_reader :index_name
def initialize(index_name)
@index_name = index_name
super "The index `#{index_name}` seems to be corrupted, it contains duplicate rows. " \
'For information on how to fix this, see our documentation: ' \
'https://docs.joinmastodon.org/admin/troubleshooting/index-corruption/'
end
def cause
@ -802,6 +808,24 @@ module Mastodon
columns(table).find { |column| column.name == name }
end
# Update the configuration of an index by creating a new one and then
# removing the old one
def update_index(table_name, index_name, columns, **index_options)
if index_name_exists?(table_name, "#{index_name}_new") && index_name_exists?(table_name, index_name)
remove_index table_name, "#{index_name}_new"
end
begin
add_index table_name, columns, **index_options.merge(name: "#{index_name}_new", algorithm: :concurrently)
rescue ActiveRecord::RecordNotUnique
remove_index table_name, name: "#{index_name}_new"
raise CorruptionError.new(index_name)
end
remove_index table_name, name: index_name if index_name_exists?(table_name, index_name)
rename_index table_name, "#{index_name}_new", index_name
end
# This will replace the first occurrence of a string in a column with
# the replacement
# On postgresql we can use `regexp_replace` for that.

View file

@ -17,23 +17,10 @@ namespace :db do
end
end
task :post_migration_hook do
at_exit do
unless %w(C POSIX).include?(ActiveRecord::Base.connection.select_one('SELECT datcollate FROM pg_database WHERE datname = current_database();')['datcollate'])
warn <<~WARNING
Your database collation may be susceptible to index corruption.
(This warning does not indicate that index corruption has occurred, and it can be ignored if you've previously checked for index corruption)
(To learn more, visit: https://docs.joinmastodon.org/admin/troubleshooting/index-corruption/)
WARNING
end
end
end
task :pre_migration_check do
version = ActiveRecord::Base.connection.select_one("SELECT current_setting('server_version_num') AS v")['v'].to_i
abort 'ERROR: This version of Mastodon requires PostgreSQL 9.5 or newer. Please update PostgreSQL before updating Mastodon.' if version < 90_500
abort 'This version of Mastodon requires PostgreSQL 9.5 or newer. Please update PostgreSQL before updating Mastodon' if version < 90_500
end
Rake::Task['db:migrate'].enhance(['db:pre_migration_check'])
Rake::Task['db:migrate'].enhance(['db:post_migration_hook'])
end