Merge pull request #2556 from ClearlyClaire/glitch-soc/merge-upstream

Merge upstream changes up to f92d8c654d

.rubocop.yml

@@ -74,14 +74,12 @@ Metrics/ModuleLength:
 Metrics/AbcSize:
   Exclude:
     - 'lib/mastodon/cli/*.rb'
-    - db/*migrate/**/*
 
 # Reason: Currently disabled in .rubocop_todo.yml
 # https://docs.rubocop.org/rubocop/cops_metrics.html#metricscyclomaticcomplexity
 Metrics/CyclomaticComplexity:
   Exclude:
     - lib/mastodon/cli/*.rb
-    - db/*migrate/**/*
 
 # Reason:
 # https://docs.rubocop.org/rubocop/cops_metrics.html#metricsparameterlists

SECURITY.md

@@ -14,9 +14,7 @@ A "vulnerability in Mastodon" is a vulnerability in the code distributed through
 ## Supported Versions
 
 | Version | Supported |
-| ------- | ---------------- |
+| ------- | --------- |
 | 4.2.x   | Yes       |
 | 4.1.x   | Yes       |
-| 4.0.x   | No               |
+| < 4.1   | No        |
-| 3.5.x   | Until 2023-12-31 |
-| < 3.5   | No               |

app/controllers/admin/export_domain_blocks_controller.rb

@@ -68,7 +68,7 @@ module Admin
 
     def export_data
       CSV.generate(headers: export_headers, write_headers: true) do |content|
-        DomainBlock.with_limitations.each do |instance|
+        DomainBlock.with_limitations.order(id: :asc).each do |instance|
           content << [instance.domain, instance.severity, instance.reject_media, instance.reject_reports, instance.public_comment, instance.obfuscate]
         end
       end

db/post_migrate/20221101190723_backfill_admin_action_logs.rb

@@ -77,82 +77,130 @@ class BackfillAdminActionLogs < ActiveRecord::Migration[6.1]
 
   def up
     safety_assured do
+      process_logs_for_account
+      process_logs_for_user
+      process_logs_for_report
+      process_logs_for_domain_block
+      process_logs_for_domain_allow
+      process_logs_for_email_domain_block
+      process_logs_for_unavailable_domain
+      process_logs_for_status
+      process_logs_for_account_warning
+      process_logs_for_announcement
+      process_logs_for_ip_block
+      process_logs_for_custom_emoji
+      process_logs_for_canonical_email_block
+      process_logs_for_appeal
+    end
+  end
+
+  def down; end
+
+  private
+
+  def process_logs_for_account
     AdminActionLog.includes(:account).where(target_type: 'Account', human_identifier: nil).find_each do |log|
       next if log.account.nil?
 
       log.update_attribute('human_identifier', log.account.acct)
     end
+  end
 
+  def process_logs_for_user
     AdminActionLog.includes(user: :account).where(target_type: 'User', human_identifier: nil).find_each do |log|
       next if log.user.nil?
 
       log.update_attribute('human_identifier', log.user.account.acct)
       log.update_attribute('route_param', log.user.account_id)
     end
+  end
 
+  def process_logs_for_report
     AdminActionLog.where(target_type: 'Report', human_identifier: nil).in_batches.update_all('human_identifier = target_id::text')
+  end
 
+  def process_logs_for_domain_block
     AdminActionLog.includes(:domain_block).where(target_type: 'DomainBlock').find_each do |log|
       next if log.domain_block.nil?
 
       log.update_attribute('human_identifier', log.domain_block.domain)
     end
+  end
 
+  def process_logs_for_domain_allow
     AdminActionLog.includes(:domain_allow).where(target_type: 'DomainAllow').find_each do |log|
       next if log.domain_allow.nil?
 
       log.update_attribute('human_identifier', log.domain_allow.domain)
     end
+  end
 
+  def process_logs_for_email_domain_block
     AdminActionLog.includes(:email_domain_block).where(target_type: 'EmailDomainBlock').find_each do |log|
       next if log.email_domain_block.nil?
 
       log.update_attribute('human_identifier', log.email_domain_block.domain)
     end
+  end
 
+  def process_logs_for_unavailable_domain
     AdminActionLog.includes(:unavailable_domain).where(target_type: 'UnavailableDomain').find_each do |log|
       next if log.unavailable_domain.nil?
 
       log.update_attribute('human_identifier', log.unavailable_domain.domain)
     end
+  end
 
+  def process_logs_for_status
     AdminActionLog.includes(status: :account).where(target_type: 'Status', human_identifier: nil).find_each do |log|
       next if log.status.nil?
 
       log.update_attribute('human_identifier', log.status.account.acct)
       log.update_attribute('permalink', log.status.uri)
     end
+  end
 
+  def process_logs_for_account_warning
     AdminActionLog.includes(account_warning: :account).where(target_type: 'AccountWarning', human_identifier: nil).find_each do |log|
       next if log.account_warning.nil?
 
       log.update_attribute('human_identifier', log.account_warning.account.acct)
     end
+  end
 
+  def process_logs_for_announcement
     AdminActionLog.includes(:announcement).where(target_type: 'Announcement', human_identifier: nil).find_each do |log|
       next if log.announcement.nil?
 
       log.update_attribute('human_identifier', log.announcement.text)
     end
+  end
 
+  def process_logs_for_ip_block
     AdminActionLog.includes(:ip_block).where(target_type: 'IpBlock', human_identifier: nil).find_each do |log|
       next if log.ip_block.nil?
 
       log.update_attribute('human_identifier', "#{log.ip_block.ip}/#{log.ip_block.ip.prefix}")
     end
+  end
 
+  def process_logs_for_custom_emoji
     AdminActionLog.includes(:custom_emoji).where(target_type: 'CustomEmoji', human_identifier: nil).find_each do |log|
       next if log.custom_emoji.nil?
 
       log.update_attribute('human_identifier', log.custom_emoji.shortcode)
     end
+  end
 
+  def process_logs_for_canonical_email_block
     AdminActionLog.includes(:canonical_email_block).where(target_type: 'CanonicalEmailBlock', human_identifier: nil).find_each do |log|
       next if log.canonical_email_block.nil?
 
       log.update_attribute('human_identifier', log.canonical_email_block.canonical_email_hash)
     end
+  end
 
+  def process_logs_for_appeal
     AdminActionLog.includes(appeal: :account).where(target_type: 'Appeal', human_identifier: nil).find_each do |log|
       next if log.appeal.nil?
 
@@ -160,7 +208,4 @@ class BackfillAdminActionLogs < ActiveRecord::Migration[6.1]
       log.update_attribute('route_param', log.appeal.account_warning_id)
     end
   end
-  end
-
-  def down; end
 end

db/post_migrate/20221206114142_backfill_admin_action_logs_again.rb

@@ -77,82 +77,130 @@ class BackfillAdminActionLogsAgain < ActiveRecord::Migration[6.1]
 
   def up
     safety_assured do
+      process_logs_for_account
+      process_logs_for_user
+      process_logs_for_report
+      process_logs_for_domain_block
+      process_logs_for_domain_allow
+      process_logs_for_email_domain_block
+      process_logs_for_unavailable_domain
+      process_logs_for_status
+      process_logs_for_account_warning
+      process_logs_for_announcement
+      process_logs_for_ip_block
+      process_logs_for_custom_emoji
+      process_logs_for_canonical_email_block
+      process_logs_for_appeal
+    end
+  end
+
+  def down; end
+
+  private
+
+  def process_logs_for_account
     AdminActionLog.includes(:account).where(target_type: 'Account', human_identifier: nil).find_each do |log|
       next if log.account.nil?
 
       log.update_attribute('human_identifier', log.account.acct)
     end
+  end
 
+  def process_logs_for_user
     AdminActionLog.includes(user: :account).where(target_type: 'User', human_identifier: nil).find_each do |log|
       next if log.user.nil?
 
       log.update_attribute('human_identifier', log.user.account.acct)
       log.update_attribute('route_param', log.user.account_id)
     end
+  end
 
+  def process_logs_for_report
     AdminActionLog.where(target_type: 'Report', human_identifier: nil).in_batches.update_all('human_identifier = target_id::text')
+  end
 
+  def process_logs_for_domain_block
     AdminActionLog.includes(:domain_block).where(target_type: 'DomainBlock').find_each do |log|
       next if log.domain_block.nil?
 
       log.update_attribute('human_identifier', log.domain_block.domain)
     end
+  end
 
+  def process_logs_for_domain_allow
     AdminActionLog.includes(:domain_allow).where(target_type: 'DomainAllow').find_each do |log|
       next if log.domain_allow.nil?
 
       log.update_attribute('human_identifier', log.domain_allow.domain)
     end
+  end
 
+  def process_logs_for_email_domain_block
     AdminActionLog.includes(:email_domain_block).where(target_type: 'EmailDomainBlock').find_each do |log|
       next if log.email_domain_block.nil?
 
       log.update_attribute('human_identifier', log.email_domain_block.domain)
     end
+  end
 
+  def process_logs_for_unavailable_domain
     AdminActionLog.includes(:unavailable_domain).where(target_type: 'UnavailableDomain').find_each do |log|
       next if log.unavailable_domain.nil?
 
       log.update_attribute('human_identifier', log.unavailable_domain.domain)
     end
+  end
 
+  def process_logs_for_status
     AdminActionLog.includes(status: :account).where(target_type: 'Status', human_identifier: nil).find_each do |log|
       next if log.status.nil?
 
       log.update_attribute('human_identifier', log.status.account.acct)
       log.update_attribute('permalink', log.status.uri)
     end
+  end
 
+  def process_logs_for_account_warning
     AdminActionLog.includes(account_warning: :account).where(target_type: 'AccountWarning', human_identifier: nil).find_each do |log|
       next if log.account_warning.nil?
 
       log.update_attribute('human_identifier', log.account_warning.account.acct)
     end
+  end
 
+  def process_logs_for_announcement
     AdminActionLog.includes(:announcement).where(target_type: 'Announcement', human_identifier: nil).find_each do |log|
       next if log.announcement.nil?
 
       log.update_attribute('human_identifier', log.announcement.text)
     end
+  end
 
+  def process_logs_for_ip_block
     AdminActionLog.includes(:ip_block).where(target_type: 'IpBlock', human_identifier: nil).find_each do |log|
       next if log.ip_block.nil?
 
       log.update_attribute('human_identifier', "#{log.ip_block.ip}/#{log.ip_block.ip.prefix}")
     end
+  end
 
+  def process_logs_for_custom_emoji
     AdminActionLog.includes(:custom_emoji).where(target_type: 'CustomEmoji', human_identifier: nil).find_each do |log|
       next if log.custom_emoji.nil?
 
       log.update_attribute('human_identifier', log.custom_emoji.shortcode)
     end
+  end
 
+  def process_logs_for_canonical_email_block
     AdminActionLog.includes(:canonical_email_block).where(target_type: 'CanonicalEmailBlock', human_identifier: nil).find_each do |log|
       next if log.canonical_email_block.nil?
 
       log.update_attribute('human_identifier', log.canonical_email_block.canonical_email_hash)
     end
+  end
 
+  def process_logs_for_appeal
     AdminActionLog.includes(appeal: :account).where(target_type: 'Appeal', human_identifier: nil).find_each do |log|
       next if log.appeal.nil?
 
@@ -160,7 +208,4 @@ class BackfillAdminActionLogsAgain < ActiveRecord::Migration[6.1]
       log.update_attribute('route_param', log.appeal.account_warning_id)
     end
   end
-  end
-
-  def down; end
 end

lib/mastodon/cli/base.rb

@@ -4,6 +4,7 @@ require_relative '../../../config/boot'
 require_relative '../../../config/environment'
 
 require 'thor'
+require 'pastel'
 require_relative 'progress_helper'
 
 module Mastodon

lib/mastodon/cli/federation.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require 'tty-prompt'
-
 module Mastodon::CLI
   module Federation
     extend ActiveSupport::Concern
@@ -30,45 +28,39 @@ module Mastodon::CLI
       LONG_DESC
       def self_destruct
         if SelfDestructHelper.self_destruct?
-          prompt.ok('Self-destruct mode is already enabled for this Mastodon server')
+          say('Self-destruct mode is already enabled for this Mastodon server', :green)
 
           pending_accounts = Account.local.without_suspended.count + Account.local.suspended.joins(:deletion_request).count
           sidekiq_stats = Sidekiq::Stats.new
 
           if pending_accounts.positive?
-            prompt.warn("#{pending_accounts} accounts are still pending deletion.")
+            say("#{pending_accounts} accounts are still pending deletion.", :yellow)
           elsif sidekiq_stats.enqueued.positive?
-            prompt.warn('Deletion notices are still being processed')
+            say('Deletion notices are still being processed', :yellow)
           elsif sidekiq_stats.retry_size.positive?
-            prompt.warn('At least one delivery attempt for each deletion notice has been made, but some have failed and are scheduled for retry')
+            say('At least one delivery attempt for each deletion notice has been made, but some have failed and are scheduled for retry', :yellow)
           else
-            prompt.ok('Every deletion notice has been sent! You can safely delete all data and decomission your servers!')
+            say('Every deletion notice has been sent! You can safely delete all data and decomission your servers!', :green)
           end
 
           exit(0)
         end
 
-        exit(1) unless prompt.ask('Type in the domain of the server to confirm:', required: true) == Rails.configuration.x.local_domain
+        exit(1) unless ask('Type in the domain of the server to confirm:') == Rails.configuration.x.local_domain
 
-        prompt.warn('This operation WILL NOT be reversible.')
+        say('This operation WILL NOT be reversible.', :yellow)
-        prompt.warn('While the data won\'t be erased locally, the server will be in a BROKEN STATE afterwards.')
+        say('While the data won\'t be erased locally, the server will be in a BROKEN STATE afterwards.', :yellow)
-        prompt.warn('The deletion process itself may take a long time, and will be handled by Sidekiq, so do not shut it down until it has finished (you will be able to re-run this command to see the state of the self-destruct process).')
+        say('The deletion process itself may take a long time, and will be handled by Sidekiq, so do not shut it down until it has finished (you will be able to re-run this command to see the state of the self-destruct process).', :yellow)
 
-        exit(1) if prompt.no?('Are you sure you want to proceed?')
+        exit(1) if no?('Are you sure you want to proceed?')
 
         self_destruct_value = Rails.application.message_verifier('self-destruct').generate(Rails.configuration.x.local_domain)
-        prompt.ok('To switch Mastodon to self-destruct mode, add the following variable to your evironment (e.g. by adding a line to your `.env.production`) and restart all Mastodon processes:')
+        say('To switch Mastodon to self-destruct mode, add the following variable to your evironment (e.g. by adding a line to your `.env.production`) and restart all Mastodon processes:', :green)
-        prompt.ok("  SELF_DESTRUCT=#{self_destruct_value}")
+        say("  SELF_DESTRUCT=#{self_destruct_value}", :green)
-        prompt.ok("\nYou can re-run this command to see the state of the self-destruct process.")
+        say("\nYou can re-run this command to see the state of the self-destruct process.", :green)
-      rescue TTY::Reader::InputInterrupt
+      rescue Interrupt
         exit(1)
       end
-
-      private
-
-      def prompt
-        @prompt ||= TTY::Prompt.new
-      end
     end
   end
 end

lib/mastodon/cli/preview_cards.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require 'tty-prompt'
 require_relative 'base'
 
 module Mastodon::CLI

spec/lib/mastodon/cli/main_spec.rb

@@ -20,4 +20,157 @@ describe Mastodon::CLI::Main do
         .to output_results(Mastodon::Version.to_s)
     end
   end
+
+  describe '#self_destruct' do
+    let(:action) { :self_destruct }
+
+    context 'with self destruct mode enabled' do
+      before do
+        allow(SelfDestructHelper).to receive(:self_destruct?).and_return(true)
+      end
+
+      context 'with pending accounts' do
+        before { Fabricate(:account) }
+
+        it 'reports about pending accounts' do
+          expect { subject }
+            .to output_results(
+              'already enabled',
+              'still pending deletion'
+            )
+            .and raise_error(SystemExit)
+        end
+      end
+
+      context 'with sidekiq notices being processed' do
+        before do
+          Account.delete_all
+          stats_double = instance_double(Sidekiq::Stats, enqueued: 5)
+          allow(Sidekiq::Stats).to receive(:new).and_return(stats_double)
+        end
+
+        it 'reports about notices' do
+          expect { subject }
+            .to output_results(
+              'already enabled',
+              'notices are still being'
+            )
+            .and raise_error(SystemExit)
+        end
+      end
+
+      context 'with sidekiq failed deliveries' do
+        before do
+          Account.delete_all
+          stats_double = instance_double(Sidekiq::Stats, enqueued: 0, retry_size: 10)
+          allow(Sidekiq::Stats).to receive(:new).and_return(stats_double)
+        end
+
+        it 'reports about notices' do
+          expect { subject }
+            .to output_results(
+              'already enabled',
+              'some have failed and are scheduled'
+            )
+            .and raise_error(SystemExit)
+        end
+      end
+
+      context 'with self descruct mode ready' do
+        before do
+          Account.delete_all
+          stats_double = instance_double(Sidekiq::Stats, enqueued: 0, retry_size: 0)
+          allow(Sidekiq::Stats).to receive(:new).and_return(stats_double)
+        end
+
+        it 'reports about notices' do
+          expect { subject }
+            .to output_results(
+              'already enabled',
+              'can safely delete all data'
+            )
+            .and raise_error(SystemExit)
+        end
+      end
+    end
+
+    context 'with self destruct mode disabled' do
+      before do
+        allow(SelfDestructHelper).to receive(:self_destruct?).and_return(false)
+      end
+
+      context 'with an incorrect response to hostname' do
+        before do
+          answer_hostname_incorrectly
+        end
+
+        it 'exits silently' do
+          expect { subject }
+            .to raise_error(SystemExit)
+        end
+      end
+
+      context 'with a correct response to hostname but no to proceed' do
+        before do
+          answer_hostname_correctly
+          decline_proceed
+        end
+
+        it 'passes first step but stops before instructions' do
+          expect { subject }
+            .to output_results('operation WILL NOT')
+            .and raise_error(SystemExit)
+        end
+      end
+
+      context 'with a correct response to hostname and yes to proceed' do
+        before do
+          answer_hostname_correctly
+          accept_proceed
+        end
+
+        it 'instructs to set the appropriate environment variable' do
+          expect { subject }
+            .to output_results(
+              'operation WILL NOT',
+              'the following variable'
+            )
+        end
+      end
+
+      private
+
+      def answer_hostname_incorrectly
+        allow(cli.shell)
+          .to receive(:ask)
+          .with('Type in the domain of the server to confirm:')
+          .and_return('wrong.host')
+          .once
+      end
+
+      def answer_hostname_correctly
+        allow(cli.shell)
+          .to receive(:ask)
+          .with('Type in the domain of the server to confirm:')
+          .and_return(Rails.configuration.x.local_domain)
+          .once
+      end
+
+      def decline_proceed
+        allow(cli.shell)
+          .to receive(:no?)
+          .with('Are you sure you want to proceed?')
+          .and_return(true)
+          .once
+      end
+
+      def accept_proceed
+        allow(cli.shell)
+          .to receive(:no?)
+          .with('Are you sure you want to proceed?')
+          .and_return(false)
+          .once
+      end
+    end
+  end
 end

spec/lib/mastodon/cli/media_spec.rb

@@ -184,4 +184,58 @@ describe Mastodon::CLI::Media do
       end
     end
   end
+
+  describe '#remove_orphans' do
+    let(:action) { :remove_orphans }
+
+    before do
+      FileUtils.mkdir_p Rails.public_path.join('system')
+    end
+
+    context 'without any options' do
+      it 'runs without error' do
+        expect { subject }
+          .to output_results('Removed', 'orphans (approx')
+      end
+    end
+
+    context 'when in azure mode' do
+      before do
+        allow(Paperclip::Attachment).to receive(:default_options).and_return(storage: :azure)
+      end
+
+      it 'warns about usage and exits' do
+        expect { subject }
+          .to output_results('azure storage driver is not supported')
+          .and raise_error(SystemExit)
+      end
+    end
+
+    context 'when in fog mode' do
+      before do
+        allow(Paperclip::Attachment).to receive(:default_options).and_return(storage: :fog)
+      end
+
+      it 'warns about usage and exits' do
+        expect { subject }
+          .to output_results('fog storage driver is not supported')
+          .and raise_error(SystemExit)
+      end
+    end
+
+    context 'when in filesystem mode' do
+      before do
+        allow(File).to receive(:delete).and_return(true)
+        media_attachment.delete
+      end
+
+      let(:media_attachment) { Fabricate(:media_attachment) }
+
+      it 'removes the unlinked files' do
+        expect { subject }
+          .to output_results('Removed', 'orphans (approx')
+        expect(File).to have_received(:delete).with(media_attachment.file.path)
+      end
+    end
+  end
 end