From 1ad64b5557f8980b8ec54ac09cd79ac51223a2ea Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 31 Aug 2023 19:54:10 +0200 Subject: [PATCH 1/6] Backport container build changes to the stable-4.1 branch (#26738) Co-authored-by: Renaud Chaput --- .github/workflows/build-container-image.yml | 89 +++++++++++++++++++++ .github/workflows/build-image.yml | 70 ---------------- .github/workflows/build-releases.yml | 27 +++++++ 3 files changed, 116 insertions(+), 70 deletions(-) create mode 100644 .github/workflows/build-container-image.yml delete mode 100644 .github/workflows/build-image.yml create mode 100644 .github/workflows/build-releases.yml diff --git a/.github/workflows/build-container-image.yml b/.github/workflows/build-container-image.yml new file mode 100644 index 000000000..599879460 --- /dev/null +++ b/.github/workflows/build-container-image.yml @@ -0,0 +1,89 @@ +on: + workflow_call: + inputs: + platforms: + required: true + type: string + use_native_arm64_builder: + type: boolean + push_to_images: + type: string + flavor: + type: string + tags: + type: string + labels: + type: string + +jobs: + build-image: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + + - uses: docker/setup-qemu-action@v2 + if: contains(inputs.platforms, 'linux/arm64') && !inputs.use_native_arm64_builder + + - uses: docker/setup-buildx-action@v2 + id: buildx + if: ${{ !(inputs.use_native_arm64_builder && contains(inputs.platforms, 'linux/arm64')) }} + + - name: Start a local Docker Builder + if: inputs.use_native_arm64_builder && contains(inputs.platforms, 'linux/arm64') + run: | + docker run --rm -d --name buildkitd -p 1234:1234 --privileged moby/buildkit:latest --addr tcp://0.0.0.0:1234 + + - uses: docker/setup-buildx-action@v2 + id: buildx-native + if: inputs.use_native_arm64_builder && contains(inputs.platforms, 'linux/arm64') + with: + driver: remote + endpoint: tcp://localhost:1234 + platforms: linux/amd64 + append: | + - endpoint: tcp://${{ vars.DOCKER_BUILDER_HETZNER_ARM64_01_HOST }}:13865 + platforms: linux/arm64 + name: mastodon-docker-builder-arm64-01 + driver-opts: + - servername=mastodon-docker-builder-arm64-01 + env: + BUILDER_NODE_1_AUTH_TLS_CACERT: ${{ secrets.DOCKER_BUILDER_HETZNER_ARM64_01_CACERT }} + BUILDER_NODE_1_AUTH_TLS_CERT: ${{ secrets.DOCKER_BUILDER_HETZNER_ARM64_01_CERT }} + BUILDER_NODE_1_AUTH_TLS_KEY: ${{ secrets.DOCKER_BUILDER_HETZNER_ARM64_01_KEY }} + + - name: Log in to Docker Hub + if: contains(inputs.push_to_images, 'tootsuite') + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Log in to the Github Container registry + if: contains(inputs.push_to_images, 'ghcr.io') + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - uses: docker/metadata-action@v4 + id: meta + if: ${{ inputs.push_to_images != '' }} + with: + images: ${{ inputs.push_to_images }} + flavor: ${{ inputs.flavor }} + tags: ${{ inputs.tags }} + labels: ${{ inputs.labels }} + + - uses: docker/build-push-action@v4 + with: + context: . + platforms: ${{ inputs.platforms }} + provenance: false + builder: ${{ steps.buildx.outputs.name || steps.buildx-native.outputs.name }} + push: ${{ inputs.push_to_images != '' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml deleted file mode 100644 index b3aa9f45c..000000000 --- a/.github/workflows/build-image.yml +++ /dev/null @@ -1,70 +0,0 @@ -name: Build container image -on: - workflow_dispatch: - push: - branches: - - 'main' - tags: - - '*' - pull_request: - paths: - - .github/workflows/build-image.yml - - Dockerfile -permissions: - contents: read - packages: write - -jobs: - build-image: - runs-on: ubuntu-latest - - concurrency: - group: ${{ github.ref }} - cancel-in-progress: true - - steps: - - uses: actions/checkout@v3 - - uses: hadolint/hadolint-action@v3.1.0 - - uses: docker/setup-qemu-action@v2 - - uses: docker/setup-buildx-action@v2 - - - name: Log in to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - if: github.repository == 'mastodon/mastodon' && github.event_name != 'pull_request' - - - name: Log in to the Github Container registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - if: github.repository == 'mastodon/mastodon' && github.event_name != 'pull_request' - - - uses: docker/metadata-action@v4 - id: meta - with: - images: | - tootsuite/mastodon - ghcr.io/mastodon/mastodon - flavor: | - latest=auto - tags: | - type=edge,branch=main - type=pep440,pattern={{raw}} - type=pep440,pattern=v{{major}}.{{minor}} - type=ref,event=pr - - - uses: docker/build-push-action@v4 - with: - context: . - platforms: linux/amd64,linux/arm64 - provenance: false - builder: ${{ steps.buildx.outputs.name }} - push: ${{ github.repository == 'mastodon/mastodon' && github.event_name != 'pull_request' }} - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - cache-from: type=gha - cache-to: type=gha,mode=max diff --git a/.github/workflows/build-releases.yml b/.github/workflows/build-releases.yml new file mode 100644 index 000000000..f739a69d9 --- /dev/null +++ b/.github/workflows/build-releases.yml @@ -0,0 +1,27 @@ +name: Build container release images +on: + push: + tags: + - '*' + +permissions: + contents: read + packages: write + +jobs: + build-image: + uses: ./.github/workflows/build-container-image.yml + with: + platforms: linux/amd64,linux/arm64 + use_native_arm64_builder: true + push_to_images: | + tootsuite/mastodon + ghcr.io/mastodon/mastodon + # Only tag with latest when ran against the latest stable branch + # This needs to be updated after each minor version release + flavor: | + latest=${{ startsWith(github.ref, 'refs/tags/v4.1.') }} + tags: | + type=pep440,pattern={{raw}} + type=pep440,pattern=v{{major}}.{{minor}} + secrets: inherit From 86afbf25d01349bd2fe0ee98f1d60890ef71b7b9 Mon Sep 17 00:00:00 2001 From: Claire Date: Wed, 30 Aug 2023 17:36:16 +0200 Subject: [PATCH 2/6] Change text extraction in `PlainTextFormatter` to be faster (#26727) --- app/lib/plain_text_formatter.rb | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/app/lib/plain_text_formatter.rb b/app/lib/plain_text_formatter.rb index 6fa2bc5d2..d1ff6808b 100644 --- a/app/lib/plain_text_formatter.rb +++ b/app/lib/plain_text_formatter.rb @@ -1,9 +1,7 @@ # frozen_string_literal: true class PlainTextFormatter - include ActionView::Helpers::TextHelper - - NEWLINE_TAGS_RE = /(
|
|<\/p>)+/.freeze + NEWLINE_TAGS_RE = %r{(
|
|

)+} attr_reader :text, :local @@ -18,7 +16,10 @@ class PlainTextFormatter if local? text else - html_entities.decode(strip_tags(insert_newlines)).chomp + node = Nokogiri::HTML.fragment(insert_newlines) + # Elements that are entirely removed with our Sanitize config + node.xpath('.//iframe|.//math|.//noembed|.//noframes|.//noscript|.//plaintext|.//script|.//style|.//svg|.//xmp').remove + node.text.chomp end end @@ -27,8 +28,4 @@ class PlainTextFormatter def insert_newlines text.gsub(NEWLINE_TAGS_RE) { |match| "#{match}\n" } end - - def html_entities - HTMLEntities.new - end end From 6339806f05cbbe630639df69421394119433896a Mon Sep 17 00:00:00 2001 From: Claire Date: Wed, 9 Aug 2023 09:39:36 +0200 Subject: [PATCH 3/6] Fix blocking subdomains of an already-blocked domain (#26392) --- app/controllers/admin/domain_blocks_controller.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/admin/domain_blocks_controller.rb b/app/controllers/admin/domain_blocks_controller.rb index 74764640b..746623a06 100644 --- a/app/controllers/admin/domain_blocks_controller.rb +++ b/app/controllers/admin/domain_blocks_controller.rb @@ -37,7 +37,7 @@ module Admin @domain_block.errors.delete(:domain) render :new else - if existing_domain_block.present? + if existing_domain_block.present? && existing_domain_block.domain == TagManager.instance.normalize_domain(@domain_block.domain.strip) @domain_block = existing_domain_block @domain_block.update(resource_params) end From ea7fa048f374b31956211b87e265800e83574476 Mon Sep 17 00:00:00 2001 From: Daniel M Brasil Date: Thu, 31 Aug 2023 08:53:24 -0300 Subject: [PATCH 4/6] Fix `/api/v1/timelines/tag/:hashtag` allowing for unauthenticated access when public preview is disabled (#26237) --- .../api/v1/timelines/tag_controller.rb | 5 ++ .../api/v1/timelines/tag_controller_spec.rb | 66 ++++++++++++++----- 2 files changed, 53 insertions(+), 18 deletions(-) diff --git a/app/controllers/api/v1/timelines/tag_controller.rb b/app/controllers/api/v1/timelines/tag_controller.rb index 64a1db58d..3f41eb688 100644 --- a/app/controllers/api/v1/timelines/tag_controller.rb +++ b/app/controllers/api/v1/timelines/tag_controller.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true class Api::V1::Timelines::TagController < Api::BaseController + before_action -> { doorkeeper_authorize! :read, :'read:statuses' }, only: :show, if: :require_auth? before_action :load_tag after_action :insert_pagination_headers, unless: -> { @statuses.empty? } @@ -11,6 +12,10 @@ class Api::V1::Timelines::TagController < Api::BaseController private + def require_auth? + !Setting.timeline_preview + end + def load_tag @tag = Tag.find_normalized(params[:id]) end diff --git a/spec/controllers/api/v1/timelines/tag_controller_spec.rb b/spec/controllers/api/v1/timelines/tag_controller_spec.rb index 718911083..1c60798fc 100644 --- a/spec/controllers/api/v1/timelines/tag_controller_spec.rb +++ b/spec/controllers/api/v1/timelines/tag_controller_spec.rb @@ -5,36 +5,66 @@ require 'rails_helper' describe Api::V1::Timelines::TagController do render_views - let(:user) { Fabricate(:user) } + let(:user) { Fabricate(:user) } + let(:token) { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: 'read:statuses') } before do allow(controller).to receive(:doorkeeper_token) { token } end - context 'with a user context' do - let(:token) { Fabricate(:accessible_access_token, resource_owner_id: user.id) } + describe 'GET #show' do + subject do + get :show, params: { id: 'test' } + end - describe 'GET #show' do - before do - PostStatusService.new.call(user.account, text: 'It is a #test') + before do + PostStatusService.new.call(user.account, text: 'It is a #test') + end + + context 'when the instance allows public preview' do + context 'when the user is not authenticated' do + let(:token) { nil } + + it 'returns http success', :aggregate_failures do + subject + + expect(response).to have_http_status(200) + expect(response.headers['Link'].links.size).to eq(2) + end end - it 'returns http success' do - get :show, params: { id: 'test' } - expect(response).to have_http_status(200) - expect(response.headers['Link'].links.size).to eq(2) + context 'when the user is authenticated' do + it 'returns http success', :aggregate_failures do + subject + + expect(response).to have_http_status(200) + expect(response.headers['Link'].links.size).to eq(2) + end end end - end - context 'without a user context' do - let(:token) { Fabricate(:accessible_access_token, resource_owner_id: nil) } + context 'when the instance does not allow public preview' do + before do + Form::AdminSettings.new(timeline_preview: false).save + end - describe 'GET #show' do - it 'returns http success' do - get :show, params: { id: 'test' } - expect(response).to have_http_status(200) - expect(response.headers['Link']).to be_nil + context 'when the user is not authenticated' do + let(:token) { nil } + + it 'returns http unauthorized' do + subject + + expect(response).to have_http_status(401) + end + end + + context 'when the user is authenticated' do + it 'returns http success', :aggregate_failures do + subject + + expect(response).to have_http_status(200) + expect(response.headers['Link'].links.size).to eq(2) + end end end end From cf80d54cbae952705af250a9764c6e25e77cc3c7 Mon Sep 17 00:00:00 2001 From: Emelia Smith Date: Mon, 22 May 2023 13:15:21 +0200 Subject: [PATCH 5/6] Allow reports with long comments from remote instances, but truncate (#25028) --- app/lib/activitypub/activity/flag.rb | 6 ++++- app/lib/activitypub/tag_manager.rb | 4 +++ app/models/report.rb | 9 +++---- spec/lib/activitypub/activity/flag_spec.rb | 31 ++++++++++++++++++++++ spec/models/report_spec.rb | 11 ++++++-- spec/services/report_service_spec.rb | 8 ++++++ 6 files changed, 61 insertions(+), 8 deletions(-) diff --git a/app/lib/activitypub/activity/flag.rb b/app/lib/activitypub/activity/flag.rb index b0443849a..7539bda42 100644 --- a/app/lib/activitypub/activity/flag.rb +++ b/app/lib/activitypub/activity/flag.rb @@ -16,7 +16,7 @@ class ActivityPub::Activity::Flag < ActivityPub::Activity @account, target_account, status_ids: target_statuses.nil? ? [] : target_statuses.map(&:id), - comment: @json['content'] || '', + comment: report_comment, uri: report_uri ) end @@ -35,4 +35,8 @@ class ActivityPub::Activity::Flag < ActivityPub::Activity def report_uri @json['id'] unless @json['id'].nil? || invalid_origin?(@json['id']) end + + def report_comment + (@json['content'] || '')[0...5000] + end end diff --git a/app/lib/activitypub/tag_manager.rb b/app/lib/activitypub/tag_manager.rb index 3d6b28ef5..e05c06522 100644 --- a/app/lib/activitypub/tag_manager.rb +++ b/app/lib/activitypub/tag_manager.rb @@ -27,6 +27,8 @@ class ActivityPub::TagManager when :note, :comment, :activity return activity_account_status_url(target.account, target) if target.reblog? short_account_status_url(target.account, target) + when :flag + target.uri end end @@ -41,6 +43,8 @@ class ActivityPub::TagManager account_status_url(target.account, target) when :emoji emoji_url(target) + when :flag + target.uri end end diff --git a/app/models/report.rb b/app/models/report.rb index 525d22ad5..3ae5c10dd 100644 --- a/app/models/report.rb +++ b/app/models/report.rb @@ -39,7 +39,10 @@ class Report < ApplicationRecord scope :resolved, -> { where.not(action_taken_at: nil) } scope :with_accounts, -> { includes([:account, :target_account, :action_taken_by_account, :assigned_account].index_with({ user: [:invite_request, :invite] })) } - validates :comment, length: { maximum: 1_000 } + # A report is considered local if the reporter is local + delegate :local?, to: :account + + validates :comment, length: { maximum: 1_000 }, if: :local? validates :rule_ids, absence: true, unless: :violation? validate :validate_rule_ids @@ -50,10 +53,6 @@ class Report < ApplicationRecord violation: 2_000, } - def local? - false # Force uri_for to use uri attribute - end - before_validation :set_uri, only: :create after_create_commit :trigger_webhooks diff --git a/spec/lib/activitypub/activity/flag_spec.rb b/spec/lib/activitypub/activity/flag_spec.rb index 2f2d13876..6d7a8a7ec 100644 --- a/spec/lib/activitypub/activity/flag_spec.rb +++ b/spec/lib/activitypub/activity/flag_spec.rb @@ -37,6 +37,37 @@ RSpec.describe ActivityPub::Activity::Flag do end end + context 'when the report comment is excessively long' do + subject do + described_class.new({ + '@context': 'https://www.w3.org/ns/activitystreams', + id: flag_id, + type: 'Flag', + content: long_comment, + actor: ActivityPub::TagManager.instance.uri_for(sender), + object: [ + ActivityPub::TagManager.instance.uri_for(flagged), + ActivityPub::TagManager.instance.uri_for(status), + ], + }.with_indifferent_access, sender) + end + + let(:long_comment) { Faker::Lorem.characters(number: 6000) } + + before do + subject.perform + end + + it 'creates a report but with a truncated comment' do + report = Report.find_by(account: sender, target_account: flagged) + + expect(report).to_not be_nil + expect(report.comment.length).to eq 5000 + expect(report.comment).to eq long_comment[0...5000] + expect(report.status_ids).to eq [status.id] + end + end + context 'when the reported status is private and should not be visible to the remote server' do let(:status) { Fabricate(:status, account: flagged, uri: 'foobar', visibility: :private) } diff --git a/spec/models/report_spec.rb b/spec/models/report_spec.rb index 874be4132..c485a4a3c 100644 --- a/spec/models/report_spec.rb +++ b/spec/models/report_spec.rb @@ -125,10 +125,17 @@ describe Report do expect(report).to be_valid end - it 'is invalid if comment is longer than 1000 characters' do + let(:remote_account) { Fabricate(:account, domain: 'example.com', protocol: :activitypub, inbox_url: 'http://example.com/inbox') } + + it 'is invalid if comment is longer than 1000 characters only if reporter is local' do report = Fabricate.build(:report, comment: Faker::Lorem.characters(number: 1001)) - report.valid? + expect(report.valid?).to be false expect(report).to model_have_error_on_field(:comment) end + + it 'is valid if comment is longer than 1000 characters and reporter is not local' do + report = Fabricate.build(:report, account: remote_account, comment: Faker::Lorem.characters(number: 1001)) + expect(report.valid?).to be true + end end end diff --git a/spec/services/report_service_spec.rb b/spec/services/report_service_spec.rb index 02bc42ac1..1737a05ae 100644 --- a/spec/services/report_service_spec.rb +++ b/spec/services/report_service_spec.rb @@ -4,6 +4,14 @@ RSpec.describe ReportService, type: :service do subject { described_class.new } let(:source_account) { Fabricate(:account) } + let(:target_account) { Fabricate(:account) } + + context 'with a local account' do + it 'has a uri' do + report = subject.call(source_account, target_account) + expect(report.uri).to_not be_nil + end + end context 'for a remote account' do let(:remote_account) { Fabricate(:account, domain: 'example.com', protocol: :activitypub, inbox_url: 'http://example.com/inbox') } From 5f88a2d70bee2a48b43bb34f0fde780ae9749162 Mon Sep 17 00:00:00 2001 From: Claire Date: Tue, 5 Sep 2023 14:50:09 +0200 Subject: [PATCH 6/6] Bump version to v4.1.7 --- CHANGELOG.md | 12 ++++++++++++ docker-compose.yml | 6 +++--- lib/mastodon/version.rb | 2 +- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aeb89ef5c..3c9b8517e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,18 @@ Changelog All notable changes to this project will be documented in this file. +## [4.1.7] - 2023-09-05 + +### Changed + +- Change remote report processing to accept reports with long comments, but truncate them ([ThisIsMissEm](https://github.com/mastodon/mastodon/pull/25028)) + +### Fixed + +- **Fix blocking subdomains of an already-blocked domain** ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26392)) +- Fix `/api/v1/timelines/tag/:hashtag` allowing for unauthenticated access when public preview is disabled ([danielmbrasil](https://github.com/mastodon/mastodon/pull/26237)) +- Fix inefficiencies in `PlainTextFormatter` ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26727)) + ## [4.1.6] - 2023-07-31 ### Fixed diff --git a/docker-compose.yml b/docker-compose.yml index ae6aefb37..d739156e2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -56,7 +56,7 @@ services: web: build: . - image: ghcr.io/mastodon/mastodon:v4.1.6 + image: ghcr.io/mastodon/mastodon:v4.1.7 restart: always env_file: .env.production command: bash -c "rm -f /mastodon/tmp/pids/server.pid; bundle exec rails s -p 3000" @@ -77,7 +77,7 @@ services: streaming: build: . - image: ghcr.io/mastodon/mastodon:v4.1.6 + image: ghcr.io/mastodon/mastodon:v4.1.7 restart: always env_file: .env.production command: node ./streaming @@ -95,7 +95,7 @@ services: sidekiq: build: . - image: ghcr.io/mastodon/mastodon:v4.1.6 + image: ghcr.io/mastodon/mastodon:v4.1.7 restart: always env_file: .env.production command: bundle exec sidekiq diff --git a/lib/mastodon/version.rb b/lib/mastodon/version.rb index 1277084f0..2e46cdf00 100644 --- a/lib/mastodon/version.rb +++ b/lib/mastodon/version.rb @@ -13,7 +13,7 @@ module Mastodon end def patch - 6 + 7 end def flags