Support for import/export of instance-level domain blocks/allows for 4.x w/ additional fixes (#20597)

* Allow import/export of instance-level domain blocks/allows (#1754)

* Allow import/export of instance-level domain blocks/allows.
Fixes #15095

* Pacify circleci

* Address simple code review feedback

* Add headers to exported CSV

* Extract common import/export functionality to
AdminExportControllerConcern

* Add additional fields to instance-blocked domain export

* Address review feedback

* Split instance domain block/allow import/export into separate pages/controllers

* Address code review feedback

* Pacify DeepSource

* Work around Paperclip::HasAttachmentFile for Rails 6

* Fix deprecated API warning in export tests

* Remove after_commit workaround

(cherry picked from commit 94e98864e39c010635e839fea984f2b4893bef1a)

* Add confirmation page when importing blocked domains (#1773)

* Move glitch-soc-specific strings to glitch-soc-specific locale files

* Add confirmation page when importing blocked domains

(cherry picked from commit b91196f4b73fff91997b8077619ae25b6d04a59e)

* Fix authorization check in domain blocks controller

(cherry picked from commit 75279377583c6e2aa04cc8d7380c593979630b38)

* Fix error strings for domain blocks and email-domain blocks

Corrected issue with non-error message used for Mastodon:NotPermittedError in Domain Blocks
Corrected issue Domain Blocks using the Email Domain Blocks message on ActionContoller::ParameterMissing
Corrected issue with Email Domain Blocks using the not_permitted string from "custom emojii's"

* Ran i18n-tasks normalize to address test failure

* Removed unused admin.export_domain_blocks.not_permitted string

Removing unused string as indicated by Check i18n

* Fix tests

(cherry picked from commit 9094c2f52c24e1c00b594e7c11cd00e4a07eb431)

* Fix domain block export not exporting blocks with only media rejection

(cherry picked from commit 26ff48ee48a5c03a2a4b0bd03fd322529e6bd960)

* Fix various issues with domain block import

- stop using Paperclip for processing domain allow/block imports
- stop leaving temporary files
- better error handling
- assume CSV files are UTF-8-encoded

(cherry picked from commit cad824d8f501b95377e4f0a957e5a00d517a1902)

Co-authored-by: Levi Bard <taktaktaktaktaktaktaktaktaktak@gmail.com>
Co-authored-by: Claire <claire.github-309c@sitedethib.com>
This commit is contained in:
lenore gilbert 2022-11-17 03:05:09 -07:00 committed by GitHub
parent e7deea62d1
commit c373148b3d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
23 changed files with 537 additions and 2 deletions

View file

@ -4,6 +4,18 @@ module Admin
class DomainBlocksController < BaseController
before_action :set_domain_block, only: [:show, :destroy, :edit, :update]
def batch
authorize :domain_block, :create?
@form = Form::DomainBlockBatch.new(form_domain_block_batch_params.merge(current_account: current_account, action: action_from_button))
@form.save
rescue ActionController::ParameterMissing
flash[:alert] = I18n.t('admin.domain_blocks.no_domain_block_selected')
rescue Mastodon::NotPermittedError
flash[:alert] = I18n.t('admin.domain_blocks.not_permitted')
else
redirect_to admin_instances_path(limited: '1'), notice: I18n.t('admin.domain_blocks.created_msg')
end
def new
authorize :domain_block, :create?
@domain_block = DomainBlock.new(domain: params[:_domain])
@ -76,5 +88,15 @@ module Admin
def resource_params
params.require(:domain_block).permit(:domain, :severity, :reject_media, :reject_reports, :private_comment, :public_comment, :obfuscate)
end
def form_domain_block_batch_params
params.require(:form_domain_block_batch).permit(domain_blocks_attributes: [:enabled, :domain, :severity, :reject_media, :reject_reports, :private_comment, :public_comment, :obfuscate])
end
def action_from_button
if params[:save]
'save'
end
end
end
end

View file

@ -19,7 +19,7 @@ module Admin
rescue ActionController::ParameterMissing
flash[:alert] = I18n.t('admin.email_domain_blocks.no_email_domain_block_selected')
rescue Mastodon::NotPermittedError
flash[:alert] = I18n.t('admin.custom_emojis.not_permitted')
flash[:alert] = I18n.t('admin.email_domain_blocks.not_permitted')
ensure
redirect_to admin_email_domain_blocks_path
end

View file

@ -0,0 +1,60 @@
# frozen_string_literal: true
require 'csv'
module Admin
class ExportDomainAllowsController < BaseController
include AdminExportControllerConcern
before_action :set_dummy_import!, only: [:new]
def new
authorize :domain_allow, :create?
end
def export
authorize :instance, :index?
send_export_file
end
def import
authorize :domain_allow, :create?
begin
@import = Admin::Import.new(import_params)
return render :new unless @import.validate
parse_import_data!(export_headers)
@data.take(Admin::Import::ROWS_PROCESSING_LIMIT).each do |row|
domain = row['#domain'].strip
next if DomainAllow.allowed?(domain)
domain_allow = DomainAllow.new(domain: domain)
log_action :create, domain_allow if domain_allow.save
end
flash[:notice] = I18n.t('admin.domain_allows.created_msg')
rescue ActionController::ParameterMissing
flash[:error] = I18n.t('admin.export_domain_allows.no_file')
end
redirect_to admin_instances_path
end
private
def export_filename
'domain_allows.csv'
end
def export_headers
%w(#domain)
end
def export_data
CSV.generate(headers: export_headers, write_headers: true) do |content|
DomainAllow.allowed_domains.each do |instance|
content << [instance.domain]
end
end
end
end
end

View file

@ -0,0 +1,71 @@
# frozen_string_literal: true
require 'csv'
module Admin
class ExportDomainBlocksController < BaseController
include AdminExportControllerConcern
before_action :set_dummy_import!, only: [:new]
def new
authorize :domain_block, :create?
end
def export
authorize :instance, :index?
send_export_file
end
def import
authorize :domain_block, :create?
@import = Admin::Import.new(import_params)
return render :new unless @import.validate
parse_import_data!(export_headers)
@global_private_comment = I18n.t('admin.export_domain_blocks.import.private_comment_template', source: @import.data_file_name, date: I18n.l(Time.now.utc))
@form = Form::DomainBlockBatch.new
@domain_blocks = @data.take(Admin::Import::ROWS_PROCESSING_LIMIT).filter_map do |row|
domain = row['#domain'].strip
next if DomainBlock.rule_for(domain).present?
domain_block = DomainBlock.new(domain: domain,
severity: row['#severity'].strip,
reject_media: row['#reject_media'].strip,
reject_reports: row['#reject_reports'].strip,
private_comment: @global_private_comment,
public_comment: row['#public_comment']&.strip,
obfuscate: row['#obfuscate'].strip)
domain_block if domain_block.valid?
end
@warning_domains = Instance.where(domain: @domain_blocks.map(&:domain)).where('EXISTS (SELECT 1 FROM follows JOIN accounts ON follows.account_id = accounts.id OR follows.target_account_id = accounts.id WHERE accounts.domain = instances.domain)').pluck(:domain)
rescue ActionController::ParameterMissing
flash.now[:alert] = I18n.t('admin.export_domain_blocks.no_file')
set_dummy_import!
render :new
end
private
def export_filename
'domain_blocks.csv'
end
def export_headers
%w(#domain #severity #reject_media #reject_reports #public_comment #obfuscate)
end
def export_data
CSV.generate(headers: export_headers, write_headers: true) do |content|
DomainBlock.with_limitations.each do |instance|
content << [instance.domain, instance.severity, instance.reject_media, instance.reject_reports, instance.public_comment, instance.obfuscate]
end
end
end
end
end

View file

@ -0,0 +1,39 @@
# frozen_string_literal: true
module AdminExportControllerConcern
extend ActiveSupport::Concern
private
def send_export_file
respond_to do |format|
format.csv { send_data export_data, filename: export_filename }
end
end
def export_data
raise 'Override in controller'
end
def export_filename
raise 'Override in controller'
end
def set_dummy_import!
@import = Admin::Import.new
end
def import_params
params.require(:admin_import).permit(:data)
end
def import_data_path
params[:admin_import][:data].path
end
def parse_import_data!(default_headers)
data = CSV.read(import_data_path, headers: true, encoding: 'UTF-8')
data = CSV.read(import_data_path, headers: default_headers, encoding: 'UTF-8') unless data.headers&.first&.strip&.include?(default_headers[0])
@data = data.reject(&:blank?)
end
end

View file

@ -185,6 +185,12 @@ ready(() => {
const registrationMode = document.getElementById('form_admin_settings_registrations_mode');
if (registrationMode) onChangeRegistrationMode(registrationMode);
const checkAllElement = document.querySelector('#batch_checkbox_all');
if (checkAllElement) {
checkAllElement.checked = [].every.call(document.querySelectorAll(batchCheckboxClassName), (content) => content.checked);
checkAllElement.indeterminate = !checkAllElement.checked && [].some.call(document.querySelectorAll(batchCheckboxClassName), (content) => content.checked);
}
document.querySelector('a#add-instance-button')?.addEventListener('click', (e) => {
const domain = document.getElementById('by_domain')?.value;

View file

@ -0,0 +1,32 @@
# frozen_string_literal: true
# A non-activerecord helper class for csv upload
class Admin::Import
include ActiveModel::Model
ROWS_PROCESSING_LIMIT = 20_000
attr_accessor :data
validates :data, presence: true
validate :validate_data
def data_file_name
data.original_filename
end
private
def validate_data
return if data.blank?
csv_data = CSV.read(data.path, encoding: 'UTF-8')
row_count = csv_data.size
row_count -= 1 if csv_data.first&.first == '#domain'
errors.add(:data, I18n.t('imports.errors.over_rows_processing_limit', count: ROWS_PROCESSING_LIMIT)) if row_count > ROWS_PROCESSING_LIMIT
rescue CSV::MalformedCSVError => e
errors.add(:data, I18n.t('imports.errors.invalid_csv_file', error: e.message))
end
end

View file

@ -28,6 +28,10 @@ class DomainAllow < ApplicationRecord
!rule_for(domain).nil?
end
def allowed_domains
select(:domain)
end
def rule_for(domain)
return if domain.blank?

View file

@ -29,6 +29,7 @@ class DomainBlock < ApplicationRecord
scope :matches_domain, ->(value) { where(arel_table[:domain].matches("%#{value}%")) }
scope :with_user_facing_limitations, -> { where(severity: [:silence, :suspend]) }
scope :with_limitations, -> { where(severity: [:silence, :suspend]).or(where(reject_media: true)) }
scope :by_severity, -> { order(Arel.sql('(CASE severity WHEN 0 THEN 1 WHEN 1 THEN 2 WHEN 2 THEN 0 END), domain')) }
def to_log_human_identifier

View file

@ -0,0 +1,35 @@
# frozen_string_literal: true
class Form::DomainBlockBatch
include ActiveModel::Model
include Authorization
include AccountableConcern
attr_accessor :domain_blocks_attributes, :action, :current_account
def save
case action
when 'save'
save!
end
end
private
def domain_blocks
@domain_blocks ||= domain_blocks_attributes.values.filter_map do |attributes|
DomainBlock.new(attributes.without('enabled')) if ActiveModel::Type::Boolean.new.cast(attributes['enabled'])
end
end
def save!
domain_blocks.each do |domain_block|
authorize(domain_block, :create?)
next if DomainBlock.rule_for(domain_block.domain).present?
domain_block.save!
DomainBlockWorker.perform_async(domain_block.id)
log_action :create, domain_block
end
end
end

View file

@ -0,0 +1,10 @@
- content_for :page_title do
= t('.title')
= simple_form_for @import, url: import_admin_export_domain_allows_path, html: { multipart: true } do |f|
.fields-row
.fields-group.fields-row__column.fields-row__column-6
= f.input :data, wrapper: :with_block_label, hint: t('simple_form.hints.imports.data'), as: :file
.actions
= f.button :button, t('imports.upload'), type: :submit

View file

@ -0,0 +1,27 @@
- existing_relationships ||= false
.batch-table__row{ class: [existing_relationships && 'batch-table__row--attention'] }
%label.batch-table__row__select.batch-table__row__select--aligned.batch-checkbox
= f.check_box :enabled, checked: !existing_relationships
.batch-table__row__content.pending-account
.pending-account__header
%strong
= f.object.domain
= f.hidden_field :domain
= f.hidden_field :severity
= f.hidden_field :reject_media
= f.hidden_field :reject_reports
= f.hidden_field :obfuscate
= f.hidden_field :private_comment
= f.hidden_field :public_comment
%br/
= f.object.policies.map { |policy| t(policy, scope: 'admin.instances.content_policies.policies') }.join(' • ')
- if f.object.public_comment.present?
= f.object.public_comment
- if existing_relationships
= fa_icon 'warning fw'
= t('admin.export_domain_blocks.import.existing_relationships_warning')

View file

@ -0,0 +1,21 @@
- content_for :page_title do
= t('admin.export_domain_blocks.import.title')
%p= t('admin.export_domain_blocks.import.description_html')
- if defined?(@global_private_comment) && @global_private_comment.present?
%p= t('admin.export_domain_blocks.import.private_comment_description_html', comment: @global_private_comment)
= form_for(@form, url: batch_admin_domain_blocks_path) do |f|
.batch-table
.batch-table__toolbar
%label.batch-table__toolbar__select.batch-checkbox-all
= check_box_tag :batch_checkbox_all, nil, false
.batch-table__toolbar__actions
= f.button safe_join([fa_icon('copy'), t('admin.domain_blocks.import')]), name: :save, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
.batch-table__body
- if @domain_blocks.empty?
= nothing_here 'nothing-here--under-tabs'
- else
= f.simple_fields_for :domain_blocks, @domain_blocks do |ff|
= render 'domain_block', f: ff, existing_relationships: @warning_domains.include?(ff.object.domain)

View file

@ -0,0 +1,10 @@
- content_for :page_title do
= t('.title')
= simple_form_for @import, url: import_admin_export_domain_blocks_path, html: { multipart: true } do |f|
.fields-row
.fields-group.fields-row__column.fields-row__column-6
= f.input :data, wrapper: :with_block_label, hint: t('simple_form.hints.imports.data'), as: :file
.actions
= f.button :button, t('imports.upload'), type: :submit

View file

@ -7,8 +7,12 @@
- content_for :heading_actions do
- if whitelist_mode?
= link_to t('admin.domain_allows.add_new'), new_admin_domain_allow_path, class: 'button', id: 'add-instance-button'
= link_to t('admin.domain_allows.export'), export_admin_export_domain_allows_path(format: :csv), class: 'button'
= link_to t('admin.domain_allows.import'), new_admin_export_domain_allow_path, class: 'button'
- else
= link_to t('admin.domain_blocks.add_new'), new_admin_domain_block_path, class: 'button', id: 'add-instance-button'
= link_to t('admin.domain_blocks.export'), export_admin_export_domain_blocks_path(format: :csv), class: 'button'
= link_to t('admin.domain_blocks.import'), new_admin_export_domain_block_path, class: 'button'
.filters
.filter-subset

View file

@ -373,6 +373,8 @@ en:
add_new: Allow federation with domain
created_msg: Domain has been successfully allowed for federation
destroyed_msg: Domain has been disallowed from federation
export: Export
import: Import
undo: Disallow federation with domain
domain_blocks:
add_new: Add new domain block
@ -382,6 +384,8 @@ en:
edit: Edit domain block
existing_domain_block: You have already imposed stricter limits on %{name}.
existing_domain_block_html: You have already imposed stricter limits on %{name}, you need to <a href="%{unblock_url}">unblock it</a> first.
export: Export
import: Import
new:
create: Create block
hint: The domain block will not prevent creation of account entries in the database, but will retroactively and automatically apply specific moderation methods on those accounts.
@ -391,6 +395,8 @@ en:
silence: Limit
suspend: Suspend
title: New domain block
no_domain_block_selected: No domain blocks were changed as none were selected
not_permitted: You are not permitted to perform this action
obfuscate: Obfuscate domain name
obfuscate_hint: Partially obfuscate the domain name in the list if advertising the list of domain limitations is enabled
private_comment: Private comment
@ -422,6 +428,20 @@ en:
resolved_dns_records_hint_html: The domain name resolves to the following MX domains, which are ultimately responsible for accepting e-mail. Blocking an MX domain will block sign-ups from any e-mail address which uses the same MX domain, even if the visible domain name is different. <strong>Be careful not to block major e-mail providers.</strong>
resolved_through_html: Resolved through %{domain}
title: Blocked e-mail domains
export_domain_allows:
new:
title: Import domain allows
no_file: No file selected
export_domain_blocks:
import:
description_html: You are about to import a list of domain blocks. Please review this list very carefully, especially if you have not authored this list yourself.
existing_relationships_warning: Existing follow relationships
private_comment_description_html: 'To help you track where imported blocks come from, imported blocks will be created with the following private comment: <q>%{comment}</q>'
private_comment_template: Imported from %{source} on %{date}
title: Import domain blocks
new:
title: Import domain blocks
no_file: No file selected
follow_recommendations:
description_html: "<strong>Follow recommendations help new users quickly find interesting content</strong>. When a user has not interacted with others enough to form personalized follow recommendations, these accounts are recommended instead. They are re-calculated on a daily basis from a mix of accounts with the highest recent engagements and highest local follower counts for a given language."
language: For language
@ -1159,6 +1179,7 @@ en:
invalid_markup: 'contains invalid HTML markup: %{error}'
imports:
errors:
invalid_csv_file: 'Invalid CSV file. Error: %{error}'
over_rows_processing_limit: contains more than %{count} rows
modes:
merge: Merge

View file

@ -225,7 +225,25 @@ Rails.application.routes.draw do
get '/dashboard', to: 'dashboard#index'
resources :domain_allows, only: [:new, :create, :show, :destroy]
resources :domain_blocks, only: [:new, :create, :destroy, :update, :edit]
resources :domain_blocks, only: [:new, :create, :show, :destroy, :update, :edit] do
collection do
post :batch
end
end
resources :export_domain_allows, only: [:new] do
collection do
get :export, constraints: { format: :csv }
post :import
end
end
resources :export_domain_blocks, only: [:new] do
collection do
get :export, constraints: { format: :csv }
post :import
end
end
resources :email_domain_blocks, only: [:index, :new, :create] do
collection do
@ -523,6 +541,7 @@ Rails.application.routes.draw do
end
resource :domain_blocks, only: [:show, :create, :destroy]
resource :directory, only: [:show]
resources :follow_requests, only: [:index] do

View file

@ -0,0 +1,48 @@
require 'rails_helper'
RSpec.describe Admin::DomainAllowsController, type: :controller do
render_views
before do
sign_in Fabricate(:user, role: UserRole.find_by(name: 'Admin')), scope: :user
end
describe 'GET #new' do
it 'assigns a new domain allow' do
get :new
expect(assigns(:domain_allow)).to be_instance_of(DomainAllow)
expect(response).to have_http_status(200)
end
end
describe 'POST #create' do
it 'blocks the domain when succeeded to save' do
post :create, params: { domain_allow: { domain: 'example.com' } }
expect(flash[:notice]).to eq I18n.t('admin.domain_allows.created_msg')
expect(response).to redirect_to(admin_instances_path)
end
it 'renders new when failed to save' do
Fabricate(:domain_allow, domain: 'example.com')
post :create, params: { domain_allow: { domain: 'example.com' } }
expect(response).to render_template :new
end
end
describe 'DELETE #destroy' do
it 'disallows the domain' do
service = double(call: true)
allow(UnallowDomainService).to receive(:new).and_return(service)
domain_allow = Fabricate(:domain_allow)
delete :destroy, params: { id: domain_allow.id }
expect(service).to have_received(:call).with(domain_allow)
expect(flash[:notice]).to eq I18n.t('admin.domain_allows.destroyed_msg')
expect(response).to redirect_to(admin_instances_path)
end
end
end

View file

@ -16,6 +16,27 @@ RSpec.describe Admin::DomainBlocksController, type: :controller do
end
end
describe 'POST #batch' do
it 'blocks the domains when succeeded to save' do
allow(DomainBlockWorker).to receive(:perform_async).and_return(true)
post :batch, params: {
save: '',
form_domain_block_batch: {
domain_blocks_attributes: {
'0' => { enabled: '1', domain: 'example.com', severity: 'silence' },
'1' => { enabled: '0', domain: 'mastodon.social', severity: 'suspend' },
'2' => { enabled: '1', domain: 'mastodon.online', severity: 'suspend' }
}
}
}
expect(DomainBlockWorker).to have_received(:perform_async).exactly(2).times
expect(flash[:notice]).to eq I18n.t('admin.domain_blocks.created_msg')
expect(response).to redirect_to(admin_instances_path(limited: '1'))
end
end
describe 'POST #create' do
it 'blocks the domain when succeeded to save' do
allow(DomainBlockWorker).to receive(:perform_async).and_return(true)

View file

@ -0,0 +1,42 @@
require 'rails_helper'
RSpec.describe Admin::ExportDomainAllowsController, type: :controller do
render_views
before do
sign_in Fabricate(:user, role: UserRole.find_by(name: 'Admin')), scope: :user
end
describe 'GET #export' do
it 'renders instances' do
Fabricate(:domain_allow, domain: 'good.domain')
Fabricate(:domain_allow, domain: 'better.domain')
get :export, params: { format: :csv }
expect(response).to have_http_status(200)
expect(response.body).to eq(IO.read(File.join(file_fixture_path, 'domain_allows.csv')))
end
end
describe 'POST #import' do
it 'allows imported domains' do
post :import, params: { admin_import: { data: fixture_file_upload('domain_allows.csv') } }
expect(response).to redirect_to(admin_instances_path)
# Header should not be imported
expect(DomainAllow.where(domain: '#domain').present?).to eq(false)
# Domains should now be added
get :export, params: { format: :csv }
expect(response).to have_http_status(200)
expect(response.body).to eq(IO.read(File.join(file_fixture_path, 'domain_allows.csv')))
end
it 'displays error on no file selected' do
post :import, params: { admin_import: {} }
expect(response).to redirect_to(admin_instances_path)
expect(flash[:error]).to eq(I18n.t('admin.export_domain_allows.no_file'))
end
end
end

View file

@ -0,0 +1,35 @@
require 'rails_helper'
RSpec.describe Admin::ExportDomainBlocksController, type: :controller do
render_views
before do
sign_in Fabricate(:user, role: UserRole.find_by(name: 'Admin')), scope: :user
end
describe 'GET #export' do
it 'renders instances' do
Fabricate(:domain_block, domain: 'bad.domain', severity: 'silence', public_comment: 'bad')
Fabricate(:domain_block, domain: 'worse.domain', severity: 'suspend', reject_media: true, reject_reports: true, public_comment: 'worse', obfuscate: true)
Fabricate(:domain_block, domain: 'reject.media', severity: 'noop', reject_media: true, public_comment: 'reject media')
Fabricate(:domain_block, domain: 'no.op', severity: 'noop', public_comment: 'noop')
get :export, params: { format: :csv }
expect(response).to have_http_status(200)
expect(response.body).to eq(IO.read(File.join(file_fixture_path, 'domain_blocks.csv')))
end
end
describe 'POST #import' do
it 'blocks imported domains' do
post :import, params: { admin_import: { data: fixture_file_upload('domain_blocks.csv') } }
expect(assigns(:domain_blocks).map(&:domain)).to match_array ['bad.domain', 'worse.domain', 'reject.media']
end
end
it 'displays error on no file selected' do
post :import, params: { admin_import: {} }
expect(flash[:alert]).to eq(I18n.t('admin.export_domain_blocks.no_file'))
end
end

3
spec/fixtures/files/domain_allows.csv vendored Normal file
View file

@ -0,0 +1,3 @@
#domain
good.domain
better.domain
1 #domain
2 good.domain
3 better.domain

4
spec/fixtures/files/domain_blocks.csv vendored Normal file
View file

@ -0,0 +1,4 @@
#domain,#severity,#reject_media,#reject_reports,#public_comment,#obfuscate
bad.domain,silence,false,false,bad,false
worse.domain,suspend,true,true,worse,true
reject.media,noop,true,false,reject media,false
1 #domain #severity #reject_media #reject_reports #public_comment #obfuscate
2 bad.domain silence false false bad false
3 worse.domain suspend true true worse true
4 reject.media noop true false reject media false