Merge branch 'upstream' into main

.bundler-audit.yml

@@ -1,3 +1,6 @@
 ---
 ignore:
-  - CVE-2015-9284 # Mitigation following https://github.com/omniauth/omniauth/wiki/Resolving-CVE-2015-9284#mitigating-in-rails-applications
+  # devise-two-factor advisory about brute-forcing TOTP
+  # We have rate-limits on authentication endpoints in place (including second
+  # factor verification) since Mastodon v3.2.0
+  - CVE-2024-0227

.devcontainer/Dockerfile

@@ -4,11 +4,7 @@ FROM mcr.microsoft.com/devcontainers/ruby:1-3.2-bullseye
 # Install Rails
 # RUN gem install rails webdrivers
 
-# Default value to allow debug server to serve content over GitHub Codespace's port forwarding service
+ARG NODE_VERSION="20"
-# The value is a comma-separated list of allowed domains
-ENV RAILS_DEVELOPMENT_HOSTS=".githubpreview.dev,.preview.app.github.dev,.app.github.dev"
-
-ARG NODE_VERSION="16"
 RUN su vscode -c "source /usr/local/share/nvm/nvm.sh && nvm install ${NODE_VERSION} 2>&1"
 
 # [Optional] Uncomment this section to install additional OS packages.
@@ -19,6 +15,6 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
 RUN gem install foreman
 
 # [Optional] Uncomment this line to install global node packages.
-RUN su vscode -c "source /usr/local/share/nvm/nvm.sh && npm install -g yarn" 2>&1
+RUN su vscode -c "source /usr/local/share/nvm/nvm.sh && corepack enable" 2>&1
 
 COPY welcome-message.txt /usr/local/etc/vscode-dev-containers/first-run-notice.txt

.devcontainer/codespaces/devcontainer.json

@@ -0,0 +1,49 @@
+{
+  "name": "Mastodon on GitHub Codespaces",
+  "dockerComposeFile": "../docker-compose.yml",
+  "service": "app",
+  "workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}",
+
+  "features": {
+    "ghcr.io/devcontainers/features/sshd:1": {},
+  },
+
+  "runServices": ["app", "db", "redis"],
+
+  "forwardPorts": [3000, 4000],
+
+  "portsAttributes": {
+    "3000": {
+      "label": "web",
+      "onAutoForward": "notify",
+    },
+    "4000": {
+      "label": "stream",
+      "onAutoForward": "silent",
+    },
+  },
+
+  "otherPortsAttributes": {
+    "onAutoForward": "silent",
+  },
+
+  "remoteEnv": {
+    "LOCAL_DOMAIN": "${localEnv:CODESPACE_NAME}-3000.app.github.dev",
+    "LOCAL_HTTPS": "true",
+    "STREAMING_API_BASE_URL": "https://${localEnv:CODESPACE_NAME}-4000.app.github.dev",
+    "DISABLE_FORGERY_REQUEST_PROTECTION": "true",
+    "ES_ENABLED": "",
+    "LIBRE_TRANSLATE_ENDPOINT": "",
+  },
+
+  "onCreateCommand": "git config --global --add safe.directory ${containerWorkspaceFolder}",
+  "postCreateCommand": ".devcontainer/post-create.sh",
+  "waitFor": "postCreateCommand",
+
+  "customizations": {
+    "vscode": {
+      "settings": {},
+      "extensions": ["EditorConfig.EditorConfig", "webben.browserslist"],
+    },
+  },
+}

.devcontainer/devcontainer.json

@@ -1,32 +1,40 @@
-// For more details, see https://aka.ms/devcontainer.json.
 {
-  "name": "Mastodon",
+  "name": "Mastodon on local machine",
   "dockerComposeFile": "docker-compose.yml",
   "service": "app",
   "workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}",
 
-  // Features to add to the dev container. More info: https://containers.dev/features.
   "features": {
-    "ghcr.io/devcontainers/features/sshd:1": {}
+    "ghcr.io/devcontainers/features/sshd:1": {},
   },
 
-  // Use 'forwardPorts' to make a list of ports inside the container available locally.
-  // This can be used to network with other containers or the host.
   "forwardPorts": [3000, 4000],
 
-  // Use 'postCreateCommand' to run commands after the container is created.
+  "portsAttributes": {
+    "3000": {
+      "label": "web",
+      "onAutoForward": "notify",
+      "requireLocalPort": true,
+    },
+    "4000": {
+      "label": "stream",
+      "onAutoForward": "silent",
+      "requireLocalPort": true,
+    },
+  },
+
+  "otherPortsAttributes": {
+    "onAutoForward": "silent",
+  },
+
   "onCreateCommand": "git config --global --add safe.directory ${containerWorkspaceFolder}",
   "postCreateCommand": ".devcontainer/post-create.sh",
   "waitFor": "postCreateCommand",
 
-  // Configure tool-specific properties.
   "customizations": {
-    // Configure properties specific to VS Code.
     "vscode": {
-      // Set *default* container specific settings.json values on container create.
       "settings": {},
-      // Add the IDs of extensions you want installed when the container is created.
+      "extensions": ["EditorConfig.EditorConfig", "webben.browserslist"],
-      "extensions": ["EditorConfig.EditorConfig", "webben.browserslist"]
+    },
-    }
+  },
-  }
 }

.devcontainer/docker-compose.yml

@@ -25,6 +25,7 @@ services:
     command: sleep infinity
     ports:
       - '127.0.0.1:3000:3000'
+      - '127.0.0.1:3035:3035'
       - '127.0.0.1:4000:4000'
     networks:
       - external_network
@@ -69,7 +70,7 @@ services:
         hard: -1
 
   libretranslate:
-    image: libretranslate/libretranslate:v1.3.11
+    image: libretranslate/libretranslate:v1.5.4
     restart: unless-stopped
     volumes:
       - lt-data:/home/libretranslate/.local

.devcontainer/post-create.sh

@@ -11,7 +11,8 @@ bundle install
 git checkout -- Gemfile.lock
 
 # Fetch Javascript dependencies
-yarn --frozen-lockfile
+corepack prepare
+yarn install --immutable
 
 # [re]create, migrate, and seed the test database
 RAILS_ENV=test ./bin/rails db:setup
@@ -23,4 +24,4 @@ RAILS_ENV=development ./bin/rails db:setup
 RAILS_ENV=development ./bin/rails assets:precompile
 
 # Precompile assets for test
-RAILS_ENV=test NODE_ENV=tests ./bin/rails assets:precompile
+RAILS_ENV=test ./bin/rails assets:precompile

.dockerignore

@@ -8,6 +8,7 @@
 public/system
 public/assets
 public/packs
+public/packs-test
 node_modules
 neo4j
 vendor/bundle

.env.test

@@ -1,5 +1,5 @@
-# Node.js
+# In test, compile the NodeJS code as if we are in production
-NODE_ENV=tests
+NODE_ENV=production
 # Federation
 LOCAL_DOMAIN=cb6e6126.ngrok.io
 LOCAL_HTTPS=true

.env.vagrant

@@ -2,3 +2,7 @@ VAGRANT=true
 LOCAL_DOMAIN=mastodon.local
 BIND=0.0.0.0
 DB_HOST=/var/run/postgresql/
+
+ES_ENABLED=true
+ES_HOST=localhost
+ES_PORT=9200

.eslintrc.js

@@ -1,4 +1,7 @@
-module.exports = {
+// @ts-check
+const { defineConfig } = require('eslint-define-config');
+
+module.exports = defineConfig({
   root: true,
 
   extends: [
@@ -9,7 +12,6 @@ module.exports = {
     'plugin:import/recommended',
     'plugin:promise/recommended',
     'plugin:jsdoc/recommended',
-    'plugin:prettier/recommended',
   ],
 
   env: {
@@ -63,7 +65,9 @@ module.exports = {
     'consistent-return': 'error',
     'dot-notation': 'error',
     eqeqeq: ['error', 'always', { 'null': 'ignore' }],
+    'indent': ['error', 2],
     'jsx-quotes': ['error', 'prefer-single'],
+    'semi': ['error', 'always'],
     'no-case-declarations': 'off',
     'no-catch-shadow': 'error',
     'no-console': [
@@ -116,7 +120,6 @@ module.exports = {
     'react/jsx-uses-react': 'off', // not needed with new JSX transform
     'react/jsx-wrap-multilines': 'error',
     'react/no-deprecated': 'off',
-    'react/no-unknown-property': 'off',
     'react/react-in-jsx-scope': 'off', // not needed with new JSX transform
     'react/self-closing-comp': 'error',
 
@@ -192,6 +195,7 @@ module.exports = {
       'error',
       {
         devDependencies: [
+          '.eslintrc.js',
           'config/webpack/**',
           'app/javascript/mastodon/performance.js',
           'app/javascript/mastodon/test_setup.js',
@@ -235,19 +239,24 @@ module.exports = {
           },
           // Common React utilities
           {
-            pattern: '{classnames,react-helmet,react-router-dom}',
+            pattern: '{classnames,react-helmet,react-router,react-router-dom}',
             group: 'external',
             position: 'before',
           },
           // Immutable / Redux / data store
           {
-            pattern: '{immutable,react-redux,react-immutable-proptypes,react-immutable-pure-component,reselect}',
+            pattern: '{immutable,@reduxjs/toolkit,react-redux,react-immutable-proptypes,react-immutable-pure-component}',
             group: 'external',
             position: 'before',
           },
           // Internal packages
           {
-            pattern: '{mastodon/**,flavours/glitch-soc/**}',
+            pattern: '{mastodon/**}',
+            group: 'internal',
+            position: 'after',
+          },
+          {
+            pattern: '{flavours/glitch-soc/**}',
             group: 'internal',
             position: 'after',
           },
@@ -256,6 +265,18 @@ module.exports = {
       },
     ],
 
+    // Forbid imports from vanilla in glitch flavour
+    'import/no-restricted-paths': [
+      'error',
+      {
+        zones: [{
+          target: 'app/javascript/flavours/glitch/',
+          from: 'app/javascript/mastodon/',
+          message: 'Import from /flavours/glitch/ instead'
+        }]
+      }
+    ],
+
     'promise/always-return': 'off',
     'promise/catch-or-return': [
       'error',
@@ -279,7 +300,6 @@ module.exports = {
     'formatjs/no-id': 'off', // IDs are used for translation keys
     'formatjs/no-invalid-icu': 'error',
     'formatjs/no-literal-string-in-jsx': 'off', // Should be looked at, but mainly flagging punctuation outside of strings
-    'formatjs/no-multiple-plurals': 'off', // Only used by hashtag.jsx
     'formatjs/no-multiple-whitespaces': 'error',
     'formatjs/no-offset': 'error',
     'formatjs/no-useless-message': 'error',
@@ -298,6 +318,7 @@ module.exports = {
   overrides: [
     {
       files: [
+        '.eslintrc.js',
         '*.config.js',
         '.*rc.js',
         'ide-helper.js',
@@ -325,8 +346,8 @@ module.exports = {
 
       extends: [
         'eslint:recommended',
-        'plugin:@typescript-eslint/recommended',
+        'plugin:@typescript-eslint/strict-type-checked',
-        'plugin:@typescript-eslint/recommended-requiring-type-checking',
+        'plugin:@typescript-eslint/stylistic-type-checked',
         'plugin:react/recommended',
         'plugin:react-hooks/recommended',
         'plugin:jsx-a11y/recommended',
@@ -338,7 +359,7 @@ module.exports = {
       ],
 
       parserOptions: {
-        project: './tsconfig.json',
+        project: true,
         tsconfigRootDir: __dirname,
       },
 
@@ -348,7 +369,15 @@ module.exports = {
         '@typescript-eslint/consistent-type-definitions': ['warn', 'interface'],
         '@typescript-eslint/consistent-type-exports': 'error',
         '@typescript-eslint/consistent-type-imports': 'error',
-
+        "@typescript-eslint/prefer-nullish-coalescing": ['error', { ignorePrimitives: { boolean: true } }],
+        "@typescript-eslint/no-restricted-imports": [
+          "warn",
+          {
+            "name": "react-redux",
+            "importNames": ["useSelector", "useDispatch"],
+            "message": "Use typed hooks `useAppDispatch` and `useAppSelector` instead."
+          }
+        ],
         'jsdoc/require-jsdoc': 'off',
 
         // Those rules set stricter rules for TS files
@@ -370,14 +399,6 @@ module.exports = {
       env: {
         jest: true,
       },
-    },
+    }
-    {
-      files: [
-        'streaming/**/*',
-      ],
-      rules: {
-        'import/no-commonjs': 'off',
-      },
-    },
   ],
-};
+});

.github/FUNDING.yml

@@ -1,3 +0,0 @@
-patreon: mastodon
-open_collective: mastodon
-custom: https://sponsor.joinmastodon.org

.github/ISSUE_TEMPLATE/1.bug_report.yml

@@ -1,56 +0,0 @@
-name: Bug Report
-description: If something isn't working as expected
-labels: [bug]
-body:
-  - type: markdown
-    attributes:
-      value: |
-        Make sure that you are submitting a new bug that was not previously reported or already fixed.
-
-        Please use a concise and distinct title for the issue.
-  - type: textarea
-    attributes:
-      label: Steps to reproduce the problem
-      description: What were you trying to do?
-      value: |
-        1.
-        2.
-        3.
-        ...
-    validations:
-      required: true
-  - type: input
-    attributes:
-      label: Expected behaviour
-      description: What should have happened?
-    validations:
-      required: true
-  - type: input
-    attributes:
-      label: Actual behaviour
-      description: What happened?
-    validations:
-      required: true
-  - type: textarea
-    attributes:
-      label: Detailed description
-    validations:
-      required: false
-  - type: textarea
-    attributes:
-      label: Specifications
-      description: |
-        What version or commit hash of Mastodon did you find this bug in?
-
-        If a front-end issue, what browser and operating systems were you using?
-      placeholder: |
-        Mastodon 3.5.3 (or Edge)
-        Ruby 2.7.6 (or v3.1.2)
-        Node.js 16.18.0
-
-        Google Chrome 106.0.5249.119
-        Firefox 105.0.3
-
-        etc...
-    validations:
-      required: true

.github/ISSUE_TEMPLATE/1.web_bug_report.yml

@@ -0,0 +1,76 @@
+name: Bug Report (Web Interface)
+description: If you are using Mastodon's web interface and something is not working as expected
+labels: [bug, 'status/to triage', 'area/web interface']
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Make sure that you are submitting a new bug that was not previously reported or already fixed.
+
+        Please use a concise and distinct title for the issue.
+  - type: textarea
+    attributes:
+      label: Steps to reproduce the problem
+      description: What were you trying to do?
+      value: |
+        1.
+        2.
+        3.
+        ...
+    validations:
+      required: true
+  - type: input
+    attributes:
+      label: Expected behaviour
+      description: What should have happened?
+    validations:
+      required: true
+  - type: input
+    attributes:
+      label: Actual behaviour
+      description: What happened?
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: Detailed description
+    validations:
+      required: false
+  - type: input
+    attributes:
+      label: Mastodon instance
+      description: The address of the Mastodon instance where you experienced the issue
+      placeholder: mastodon.social
+    validations:
+      required: true
+  - type: input
+    attributes:
+      label: Mastodon version
+      description: |
+        This is displayed at the bottom of the About page, eg. `v4.1.2+nightly-20230627`
+      placeholder: v4.1.2
+    validations:
+      required: true
+  - type: input
+    attributes:
+      label: Browser name and version
+      description: |
+        What browser are you using when getting this bug? Please specify the version as well.
+      placeholder: Firefox 105.0.3
+    validations:
+      required: true
+  - type: input
+    attributes:
+      label: Operating system
+      description: |
+        What OS are you running? Please specify the version as well.
+      placeholder: macOS 13.4.1
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: Technical details
+      description: |
+        Any additional technical details you may have. This can include the full error log, inspector's output…
+    validations:
+      required: false

.github/ISSUE_TEMPLATE/2.server_bug_report.yml

@@ -0,0 +1,65 @@
+name: Bug Report (server / API)
+description: |
+  If something is not working as expected, but is not from using the web interface.
+labels: [bug, 'status/to triage']
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Make sure that you are submitting a new bug that was not previously reported or already fixed.
+
+        Please use a concise and distinct title for the issue.
+  - type: textarea
+    attributes:
+      label: Steps to reproduce the problem
+      description: What were you trying to do?
+      value: |
+        1.
+        2.
+        3.
+        ...
+    validations:
+      required: true
+  - type: input
+    attributes:
+      label: Expected behaviour
+      description: What should have happened?
+    validations:
+      required: true
+  - type: input
+    attributes:
+      label: Actual behaviour
+      description: What happened?
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: Detailed description
+    validations:
+      required: false
+  - type: input
+    attributes:
+      label: Mastodon instance
+      description: The address of the Mastodon instance where you experienced the issue
+      placeholder: mastodon.social
+    validations:
+      required: false
+  - type: input
+    attributes:
+      label: Mastodon version
+      description: |
+        This is displayed at the bottom of the About page, eg. `v4.1.2+nightly-20230627`
+      placeholder: v4.1.2
+    validations:
+      required: false
+  - type: textarea
+    attributes:
+      label: Technical details
+      description: |
+        Any additional technical details you may have, like logs or error traces
+      value: |
+        If this is happening on your own Mastodon server, please fill out those:
+        - Ruby version: (from `ruby --version`, eg. v3.1.2)
+        - Node.js version: (from `node --version`, eg. v18.16.0)
+    validations:
+      required: false

.github/actions/setup-javascript/action.yml

@@ -0,0 +1,42 @@
+name: 'Setup Javascript'
+description: 'Setup a Javascript environment ready to run the Mastodon code'
+inputs:
+  onlyProduction:
+    description: Only install production dependencies
+    default: 'false'
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Set up Node.js
+      uses: actions/setup-node@v4
+      with:
+        node-version-file: '.nvmrc'
+
+    # The following is needed because we can not use `cache: true` for `setup-node`, as it does not support Corepack yet and mess up with the cache location if ran after Node is installed
+    - name: Enable corepack
+      shell: bash
+      run: corepack enable
+
+    - name: Get yarn cache directory path
+      id: yarn-cache-dir-path
+      shell: bash
+      run: echo "dir=$(yarn config get cacheFolder)" >> $GITHUB_OUTPUT
+
+    - uses: actions/cache@v3
+      id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`)
+      with:
+        path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
+        key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
+        restore-keys: |
+          ${{ runner.os }}-yarn-
+
+    - name: Install all yarn packages
+      shell: bash
+      run: yarn install --immutable
+      if: inputs.onlyProduction == 'false'
+
+    - name: Install all production yarn packages
+      shell: bash
+      run: yarn workspaces focus --production
+      if: inputs.onlyProduction != 'false'

.github/actions/setup-ruby/action.yml

@@ -0,0 +1,23 @@
+name: 'Setup RUby'
+description: 'Setup a Ruby environment ready to run the Mastodon code'
+inputs:
+  ruby-version:
+    description: The Ruby version to install
+    default: '.ruby-version'
+  additional-system-dependencies:
+    description: 'Additional packages to install'
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Install system dependencies
+      shell: bash
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y libicu-dev libidn11-dev ${{ inputs.additional-system-dependencies }}
+
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ inputs.ruby-version }}
+        bundler-cache: true

.github/codecov.yml

@@ -0,0 +1,13 @@
+coverage:
+  status:
+    project:
+      default:
+        # Github status check is not blocking
+        informational: true
+    patch:
+      default:
+        # Github status check is not blocking
+        informational: true
+comment:
+  # Only write a comment in PR if there are changes
+  require_changes: true

.github/renovate.json5

@@ -1,25 +1,28 @@
 {
   $schema: 'https://docs.renovatebot.com/renovate-schema.json',
   extends: [
-    'config:base',
+    'config:recommended',
-    ':dependencyDashboard',
     ':labels(dependencies)',
-    ':maintainLockFilesMonthly', // update non-direct dependencies monthly
+    ':prConcurrentLimitNone', // Remove limit for open PRs at any time.
-    ':prConcurrentLimit10', // only 10 open PRs at the same time
+    ':prHourlyLimit2', // Rate limit PR creation to a maximum of two per hour.
   ],
-  stabilityDays: 3, // Wait 3 days after the package has been published before upgrading it
+  minimumReleaseAge: '3', // Wait 3 days after the package has been published before upgrading it
   // packageRules order is important, they are applied from top to bottom and are merged,
-  // so for example grouping rules needs to be at the bottom
+  // meaning the most important ones must be at the bottom, for example grouping rules
+  // If we do not want a package to be grouped with others, we need to set its groupName
+  // to `null` after any other rule set it to something.
+  dependencyDashboardHeader: 'This issue lists Renovate updates and detected dependencies. Read the [Dependency Dashboard](https://docs.renovatebot.com/key-concepts/dashboard/) docs to learn more. Before approving any upgrade: read the description and comments in the [`renovate.json5` file](https://github.com/mastodon/mastodon/blob/main/.github/renovate.json5).',
+  postUpdateOptions: ['yarnDedupeHighest'],
   packageRules: [
     {
-      // Ignore major version bumps for these node packages
+      // Require Dependency Dashboard Approval for major version bumps of these node packages
       matchManagers: ['npm'],
       matchPackageNames: [
-        '@rails/ujs', // Needs to match the major Rails version
         'tesseract.js', // Requires code changes
         'react-hotkeys', // Requires code changes
 
         // Requires Webpacker upgrade or replacement
+        '@svgr/webpack',
         '@types/webpack',
         'babel-loader',
         'compression-webpack-plugin',
@@ -40,26 +43,20 @@
         'react-router-dom',
       ],
       matchUpdateTypes: ['major'],
-      enabled: false,
+      dependencyDashboardApproval: true,
     },
     {
-      // Ignore major version bumps for these Ruby packages
+      // Require Dependency Dashboard Approval for major version bumps of these Ruby packages
       matchManagers: ['bundler'],
       matchPackageNames: [
-        'sprockets', // Requires manual upgrade https://github.com/rails/sprockets/blob/master/UPGRADING.md#guide-to-upgrading-from-sprockets-3x-to-4x
+        'rack', // Needs to be synced with Rails version
         'strong_migrations', // Requires manual upgrade
         'sidekiq', // Requires manual upgrade
         'sidekiq-unique-jobs', // Requires manual upgrades and sync with Sidekiq version
         'redis', // Requires manual upgrade and sync with Sidekiq version
-        'fog-openstack', // TODO: was ignored in https://github.com/mastodon/mastodon/pull/13964
-
-        // Needs major Rails version bump
-        'rack',
-        'rails',
-        'rails-i18n',
       ],
       matchUpdateTypes: ['major'],
-      enabled: false,
+      dependencyDashboardApproval: true,
     },
     {
       // Update Github Actions and Docker images weekly
@@ -67,36 +64,51 @@
       extends: ['schedule:weekly'],
     },
     {
-      // Ignore major & minor bumps for the ruby image, this needs to be synced with .ruby-version
+      // Require Dependency Dashboard Approval for major & minor bumps for the ruby image, this needs to be synced with .ruby-version
       matchManagers: ['dockerfile'],
       matchPackageNames: ['moritzheiber/ruby-jemalloc'],
       matchUpdateTypes: ['minor', 'major'],
-      enabled: false,
+      dependencyDashboardApproval: true,
     },
     {
-      // Ignore major bump for the node image, this needs to be synced with .nvmrc
+      // Require Dependency Dashboard Approval for major bumps for the node image, this needs to be synced with .nvmrc
       matchManagers: ['dockerfile'],
       matchPackageNames: ['node'],
       matchUpdateTypes: ['major'],
-      enabled: false,
+      dependencyDashboardApproval: true,
     },
     {
-      // Ignore major postgres bumps in the docker-compose file, as those break dev environments
+      // Require Dependency Dashboard Approval for major postgres bumps in the docker-compose file, as those break dev environments
       matchManagers: ['docker-compose'],
       matchPackageNames: ['postgres'],
       matchUpdateTypes: ['major'],
-      enabled: false,
+      dependencyDashboardApproval: true,
     },
     {
       // Update devDependencies every week, with one grouped PR
       matchDepTypes: 'devDependencies',
       matchUpdateTypes: ['patch', 'minor'],
-      excludePackageNames: [
-        'typescript', // Typescript has many changes in minor versions, needs to be checked every time
-      ],
       groupName: 'devDependencies (non-major)',
       extends: ['schedule:weekly'],
     },
+    {
+      // Group all eslint-related packages with `eslint` in the same PR
+      matchManagers: ['npm'],
+      matchPackageNames: ['eslint'],
+      matchPackagePrefixes: ['eslint-', '@typescript-eslint/'],
+      matchUpdateTypes: ['patch', 'minor'],
+      groupName: 'eslint (non-major)',
+    },
+    {
+      // Group actions/*-artifact in the same PR
+      matchManagers: ['github-actions'],
+      matchPackageNames: [
+        'actions/download-artifact',
+        'actions/upload-artifact',
+      ],
+      matchUpdateTypes: ['major'],
+      groupName: 'artifact actions (major)',
+    },
     {
       // Update @types/* packages every week, with one grouped PR
       matchPackagePrefixes: '@types/',
@@ -105,6 +117,14 @@
       extends: ['schedule:weekly'],
       addLabels: ['typescript'],
     },
+    {
+      // We want those packages to always have their own PR
+      matchManagers: ['npm'],
+      matchPackageNames: [
+        'typescript', // Typescript has code-impacting changes in minor versions
+      ],
+      groupName: null, // We dont want them to belong to any group
+    },
     // Add labels depending on package manager
     { matchManagers: ['npm', 'nvm'], addLabels: ['javascript'] },
     { matchManagers: ['bundler', 'ruby-version'], addLabels: ['ruby'] },

.github/workflows/build-container-image.yml

@@ -0,0 +1,102 @@
+on:
+  workflow_call:
+    inputs:
+      platforms:
+        required: true
+        type: string
+      cache:
+        type: boolean
+        default: true
+      use_native_arm64_builder:
+        type: boolean
+      push_to_images:
+        type: string
+      version_prerelease:
+        type: string
+      version_metadata:
+        type: string
+      flavor:
+        type: string
+      tags:
+        type: string
+      labels:
+        type: string
+      file_to_build:
+        type: string
+
+jobs:
+  build-image:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: docker/setup-qemu-action@v3
+        if: contains(inputs.platforms, 'linux/arm64') && !inputs.use_native_arm64_builder
+
+      - uses: docker/setup-buildx-action@v3
+        id: buildx
+        if: ${{ !(inputs.use_native_arm64_builder && contains(inputs.platforms, 'linux/arm64')) }}
+
+      - name: Start a local Docker Builder
+        if: inputs.use_native_arm64_builder && contains(inputs.platforms, 'linux/arm64')
+        run: |
+          docker run --rm -d --name buildkitd -p 1234:1234 --privileged moby/buildkit:latest --addr tcp://0.0.0.0:1234
+
+      - uses: docker/setup-buildx-action@v3
+        id: buildx-native
+        if: inputs.use_native_arm64_builder && contains(inputs.platforms, 'linux/arm64')
+        with:
+          driver: remote
+          endpoint: tcp://localhost:1234
+          platforms: linux/amd64
+          append: |
+            - endpoint: tcp://${{ vars.DOCKER_BUILDER_HETZNER_ARM64_01_HOST }}:13865
+              platforms: linux/arm64
+              name: mastodon-docker-builder-arm64-01
+              driver-opts:
+                - servername=mastodon-docker-builder-arm64-01
+        env:
+          BUILDER_NODE_1_AUTH_TLS_CACERT: ${{ secrets.DOCKER_BUILDER_HETZNER_ARM64_01_CACERT }}
+          BUILDER_NODE_1_AUTH_TLS_CERT: ${{ secrets.DOCKER_BUILDER_HETZNER_ARM64_01_CERT }}
+          BUILDER_NODE_1_AUTH_TLS_KEY: ${{ secrets.DOCKER_BUILDER_HETZNER_ARM64_01_KEY }}
+
+      - name: Log in to Docker Hub
+        if: contains(inputs.push_to_images, 'tootsuite')
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Log in to the Github Container registry
+        if: contains(inputs.push_to_images, 'ghcr.io')
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - uses: docker/metadata-action@v5
+        id: meta
+        if: ${{ inputs.push_to_images != '' }}
+        with:
+          images: ${{ inputs.push_to_images }}
+          flavor: ${{ inputs.flavor }}
+          tags: ${{ inputs.tags }}
+          labels: ${{ inputs.labels }}
+
+      - uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: ${{ inputs.file_to_build }}
+          build-args: |
+            MASTODON_VERSION_PRERELEASE=${{ inputs.version_prerelease }}
+            MASTODON_VERSION_METADATA=${{ inputs.version_metadata }}
+          platforms: ${{ inputs.platforms }}
+          provenance: false
+          builder: ${{ steps.buildx.outputs.name || steps.buildx-native.outputs.name }}
+          push: ${{ inputs.push_to_images != '' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: ${{ inputs.cache && 'type=gha' || '' }}
+          cache-to: ${{ inputs.cache && 'type=gha,mode=max' || '' }}

.github/workflows/build-image.yml

@@ -1,63 +0,0 @@
-name: Build container image
-on:
-  workflow_dispatch:
-  push:
-    branches:
-      - 'main'
-  pull_request:
-    paths:
-      - .github/workflows/build-image.yml
-      - Dockerfile
-permissions:
-  contents: read
-  packages: write
-
-jobs:
-  build-image:
-    runs-on: ubuntu-latest
-
-    concurrency:
-      group: ${{ github.workflow }}-${{ github.ref }}
-      cancel-in-progress: true
-
-    steps:
-      - uses: actions/checkout@v3
-      - uses: hadolint/hadolint-action@v3.1.0
-      - uses: docker/setup-qemu-action@v2
-      - uses: docker/setup-buildx-action@v2
-
-      - name: Log in to the Github Container registry
-        uses: docker/login-action@v2
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-        if: github.event_name != 'pull_request'
-
-      - uses: docker/metadata-action@v4
-        id: meta
-        with:
-          images: ghcr.io/${{ github.repository_owner }}/mastodon
-          tags: |
-            type=raw,value=latest,enable={{is_default_branch}}
-            type=edge,branch=main
-            type=sha,prefix=,format=long
-
-      - name: Generate version suffix
-        id: version_vars
-        if: github.repository == 'mastodon/mastodon' && github.event_name == 'push' && github.ref_name == 'main'
-        run: |
-          echo mastodon_version_suffix=+edge-$(git rev-parse --short HEAD) >> $GITHUB_OUTPUT
-
-      - uses: docker/build-push-action@v4
-        with:
-          context: .
-          build-args: MASTODON_VERSION_SUFFIX=${{ steps.version_vars.outputs.mastodon_version_suffix }}
-          platforms: linux/amd64,linux/arm64
-          provenance: false
-          builder: ${{ steps.buildx.outputs.name }}
-          push: ${{ github.event_name != 'pull_request' }}
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max

.github/workflows/build-nightly.yml

@@ -3,58 +3,62 @@ on:
   workflow_dispatch:
   schedule:
     - cron: '0 2 * * *' # run at 2 AM UTC
+
 permissions:
   contents: read
   packages: write
 
 jobs:
-  build-nightly-image:
+  compute-suffix:
     runs-on: ubuntu-latest
-
+    if: github.repository == 'glitch-soc/mastodon'
-    concurrency:
-      group: ${{ github.workflow }}-${{ github.ref }}
-      cancel-in-progress: true
-
     steps:
-      - uses: actions/checkout@v3
+      - id: version_vars
-      - uses: hadolint/hadolint-action@v3.1.0
+        env:
-      - uses: docker/setup-qemu-action@v2
+          TZ: Etc/UTC
-      - uses: docker/setup-buildx-action@v2
-
-      - name: Log in to the Github Container registry
-        uses: docker/login-action@v2
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - uses: docker/metadata-action@v4
-        id: meta
-        with:
-          images: |
-            ghcr.io/mastodon/mastodon
-          flavor: |
-            latest=auto
-          tags: |
-            type=raw,value=nightly
-            type=schedule,pattern=nightly-{{date 'YYYY-MM-DD' tz='Etc/UTC'}}
-          labels: |
-            org.opencontainers.image.description=Nightly build image used for testing purposes
-
-      - name: Generate version suffix
-        id: version_vars
         run: |
-          echo mastodon_version_suffix=+nightly-$(date +'%Y%m%d') >> $GITHUB_OUTPUT
+          echo mastodon_version_prerelease=nightly.$(date +'%Y-%m-%d')>> $GITHUB_OUTPUT
+    outputs:
+      prerelease: ${{ steps.version_vars.outputs.mastodon_version_prerelease }}
 
-      - uses: docker/build-push-action@v4
+  build-image:
-        with:
+    needs: compute-suffix
-          context: .
+    uses: ./.github/workflows/build-container-image.yml
-          build-args: MASTODON_VERSION_SUFFIX=${{ steps.version_vars.outputs.mastodon_version_suffix }}
+    with:
-          platforms: linux/amd64,linux/arm64
+      file_to_build: Dockerfile
-          provenance: false
+      platforms: linux/amd64,linux/arm64
-          builder: ${{ steps.buildx.outputs.name }}
+      use_native_arm64_builder: false
-          push: ${{ github.repository == 'mastodon/mastodon' && github.event_name != 'pull_request' }}
+      cache: false
-          tags: ${{ steps.meta.outputs.tags }}
+      push_to_images: |
-          labels: ${{ steps.meta.outputs.labels }}
+        ghcr.io/${{ github.repository_owner }}/mastodon
-          cache-from: type=gha
+      version_prerelease: ${{ needs.compute-suffix.outputs.prerelease }}
-          cache-to: type=gha,mode=max
+      labels: |
+        org.opencontainers.image.description=Nightly build image used for testing purposes
+      flavor: |
+        latest=true
+      tags: |
+        type=raw,value=edge
+        type=raw,value=nightly
+        type=schedule,pattern=${{ needs.compute-suffix.outputs.prerelease }}
+    secrets: inherit
+
+  build-image-streaming:
+    needs: compute-suffix
+    uses: ./.github/workflows/build-container-image.yml
+    with:
+      file_to_build: streaming/Dockerfile
+      platforms: linux/amd64,linux/arm64
+      use_native_arm64_builder: false
+      cache: false
+      push_to_images: |
+        ghcr.io/${{ github.repository_owner }}/mastodon-streaming
+      version_prerelease: ${{ needs.compute-suffix.outputs.prerelease }}
+      labels: |
+        org.opencontainers.image.description=Nightly build image used for testing purposes
+      flavor: |
+        latest=true
+      tags: |
+        type=raw,value=edge
+        type=raw,value=nightly
+        type=schedule,pattern=${{ needs.compute-suffix.outputs.prerelease }}
+    secrets: inherit

.github/workflows/build-push-pr.yml

@@ -0,0 +1,58 @@
+name: Build container image for PR
+on:
+  pull_request:
+    types: [labeled, synchronize, reopened, ready_for_review, opened]
+
+permissions:
+  contents: read
+  packages: write
+
+jobs:
+  compute-suffix:
+    runs-on: ubuntu-latest
+    # This is only allowed to run if:
+    # - the PR branch is in the `mastodon/mastodon` repository
+    # - the PR is not a draft
+    # - the PR has the "build-image" label
+    if: ${{ github.event.pull_request.head.repo.full_name == github.repository && !github.event.pull_request.draft && contains(github.event.pull_request.labels.*.name, 'build-image') }}
+    steps:
+      # Repository needs to be cloned so `git rev-parse` below works
+      - name: Clone repository
+        uses: actions/checkout@v4
+      - id: version_vars
+        run: |
+          echo mastodon_version_metadata=pr-${{ github.event.pull_request.number }}-$(git rev-parse --short HEAD) >> $GITHUB_OUTPUT
+    outputs:
+      metadata: ${{ steps.version_vars.outputs.mastodon_version_metadata }}
+
+  build-image:
+    needs: compute-suffix
+    uses: ./.github/workflows/build-container-image.yml
+    with:
+      file_to_build: Dockerfile
+      platforms: linux/amd64,linux/arm64
+      use_native_arm64_builder: false
+      push_to_images: |
+        ghcr.io/${{ github.repository_owner }}/mastodon
+      version_metadata: ${{ needs.compute-suffix.outputs.metadata }}
+      flavor: |
+        latest=auto
+      tags: |
+        type=ref,event=pr
+    secrets: inherit
+
+  build-image-streaming:
+    needs: compute-suffix
+    uses: ./.github/workflows/build-container-image.yml
+    with:
+      file_to_build: streaming/Dockerfile
+      platforms: linux/amd64,linux/arm64
+      use_native_arm64_builder: false
+      push_to_images: |
+        ghcr.io/${{ github.repository_owner }}/mastodon-streaming
+      version_metadata: ${{ needs.compute-suffix.outputs.metadata }}
+      flavor: |
+        latest=auto
+      tags: |
+        type=ref,event=pr
+    secrets: inherit

.github/workflows/build-releases.yml

@@ -0,0 +1,49 @@
+name: Build container release images
+on:
+  push:
+    tags:
+      - '*'
+
+permissions:
+  contents: read
+  packages: write
+
+jobs:
+  build-image:
+    uses: ./.github/workflows/build-container-image.yml
+    with:
+      file_to_build: Dockerfile
+      platforms: linux/amd64,linux/arm64
+      use_native_arm64_builder: false
+      push_to_images: |
+        ghcr.io/${{ github.repository_owner }}/mastodon
+      # Do not use cache when building releases, so apt update is always ran and the release always contain the latest packages
+      cache: false
+      # Only tag with latest when ran against the latest stable branch
+      # This needs to be updated after each minor version release
+      flavor: |
+        latest=${{ startsWith(github.ref, 'refs/tags/v4.2.') }}
+      tags: |
+        type=pep440,pattern={{raw}}
+        type=pep440,pattern=v{{major}}.{{minor}}
+    secrets: inherit
+
+  build-image-streaming:
+    if: startsWith(github.ref, 'refs/tags/v4.3.')
+    uses: ./.github/workflows/build-container-image.yml
+    with:
+      file_to_build: streaming/Dockerfile
+      platforms: linux/amd64,linux/arm64
+      use_native_arm64_builder: false
+      push_to_images: |
+        ghcr.io/${{ github.repository_owner }}/mastodon-streaming
+      # Do not use cache when building releases, so apt update is always ran and the release always contain the latest packages
+      cache: false
+      # Only tag with latest when ran against the latest stable branch
+      # This needs to be updated after each minor version release
+      flavor: |
+        latest=${{ startsWith(github.ref, 'refs/tags/v4.3.') }}
+      tags: |
+        type=pep440,pattern={{raw}}
+        type=pep440,pattern=v{{major}}.{{minor}}
+    secrets: inherit

.github/workflows/bundler-audit.yml

@@ -0,0 +1,34 @@
+name: Bundler Audit
+on:
+  push:
+    branches-ignore:
+      - 'dependabot/**'
+    paths:
+      - 'Gemfile*'
+      - '.ruby-version'
+      - '.bundler-audit.yml'
+      - '.github/workflows/bundler-audit.yml'
+
+  pull_request:
+    paths:
+      - 'Gemfile*'
+      - '.ruby-version'
+      - '.bundler-audit.yml'
+      - '.github/workflows/bundler-audit.yml'
+
+  schedule:
+    - cron: '0 5 * * 1'
+
+jobs:
+  security:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Clone repository
+        uses: actions/checkout@v4
+
+      - name: Set up Ruby environment
+        uses: ./.github/actions/setup-ruby
+
+      - name: Run bundler-audit
+        run: bundle exec bundler-audit

.github/workflows/check-i18n.yml

@@ -17,27 +17,13 @@ jobs:
     runs-on: ubuntu-22.04
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
-      - name: Install system dependencies
+      - name: Set up Ruby environment
-        run: |
+        uses: ./.github/actions/setup-ruby
-          sudo apt-get update
-          sudo apt-get install -y libicu-dev libidn11-dev
 
-      - name: Set up Ruby
+      - name: Set up Javascript environment
-        uses: ruby/setup-ruby@v1
+        uses: ./.github/actions/setup-javascript
-        with:
-          ruby-version: .ruby-version
-          bundler-cache: true
-
-      - name: Set up Node.js
-        uses: actions/setup-node@v3
-        with:
-          cache: yarn
-          node-version-file: '.nvmrc'
-
-      - name: Install all yarn packages
-        run: yarn --frozen-lockfile
 
       - name: Check for missing strings in English JSON
         run: |

.github/workflows/codeql.yml

@@ -27,11 +27,11 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@v2
+        uses: github/codeql-action/init@v3
         with:
           languages: ${{ matrix.language }}
           # If you wish to specify custom queries, you can do so here or in a config file.
@@ -44,7 +44,7 @@ jobs:
       # Autobuild attempts to build any compiled languages  (C/C++, C#, Go, or Java).
       # If this step fails, then you should remove it and run the build manually (see below)
       - name: Autobuild
-        uses: github/codeql-action/autobuild@v2
+        uses: github/codeql-action/autobuild@v3
 
       # ℹ️ Command-line programs to run using the OS shell.
       # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
@@ -57,6 +57,6 @@ jobs:
       #   ./location_of_script_within_repo/buildscript.sh
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v2
+        uses: github/codeql-action/analyze@v3
         with:
           category: '/language:${{matrix.language}}'

.github/workflows/crowdin-download.yml

@@ -0,0 +1,72 @@
+name: Crowdin / Download translations
+on:
+  schedule:
+    - cron: '17 4 * * *' # Every day
+  workflow_dispatch:
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  download-translations:
+    runs-on: ubuntu-latest
+    if: github.repository == 'glitch-soc/mastodon'
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Increase Git http.postBuffer
+        # This is needed due to a bug in Ubuntu's cURL version?
+        # See https://github.com/orgs/community/discussions/55820
+        run: |
+          git config --global http.version HTTP/1.1
+          git config --global http.postBuffer 157286400
+
+      # Download the translation files from Crowdin
+      - name: crowdin action
+        uses: crowdin/github-action@v1
+        with:
+          config: crowdin-glitch.yml
+          upload_sources: false
+          upload_translations: false
+          download_translations: true
+          crowdin_branch_name: main
+          push_translations: false
+          create_pull_request: false
+        env:
+          CROWDIN_PROJECT_ID: ${{ vars.CROWDIN_PROJECT_ID }}
+          CROWDIN_PERSONAL_TOKEN: ${{ secrets.CROWDIN_PERSONAL_TOKEN }}
+
+      # As the files are extracted from a Docker container, they belong to root:root
+      # We need to fix this before the next steps
+      - name: Fix file permissions
+        run: sudo chown -R runner:docker .
+
+      # This is needed to run the normalize step
+      - name: Set up Ruby environment
+        uses: ./.github/actions/setup-ruby
+
+      - name: Run i18n normalize task
+        run: bundle exec i18n-tasks normalize
+
+      # Create or update the pull request
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v5.0.2
+        with:
+          commit-message: 'New Crowdin translations'
+          title: 'New Crowdin Translations (automated)'
+          author: 'GitHub Actions <noreply@github.com>'
+          body: |
+            New Crowdin translations, automated with Github Actions
+
+            See `.github/workflows/crowdin-download.yml`
+
+            This PR will be updated every day with new translations.
+
+            Due to a limitation in Github Actions, checks are not running on this PR without manual action.
+            If you want to run the checks, then close and re-open it.
+          branch: i18n/crowdin/translations
+          base: main
+          labels: i18n

.github/workflows/crowdin-upload.yml

@@ -0,0 +1,36 @@
+name: Crowdin / Upload translations
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - crowdin.yml
+      - app/javascript/mastodon/locales/en.json
+      - config/locales/en.yml
+      - config/locales/simple_form.en.yml
+      - config/locales/activerecord.en.yml
+      - config/locales/devise.en.yml
+      - config/locales/doorkeeper.en.yml
+      - .github/workflows/crowdin-upload.yml
+
+jobs:
+  upload-translations:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: crowdin action
+        uses: crowdin/github-action@v1
+        with:
+          config: crowdin-glitch.yml
+          upload_sources: true
+          upload_translations: false
+          download_translations: false
+          crowdin_branch_name: main
+
+        env:
+          CROWDIN_PROJECT_ID: ${{ vars.CROWDIN_PROJECT_ID }}
+          CROWDIN_PERSONAL_TOKEN: ${{ secrets.CROWDIN_PERSONAL_TOKEN }}

.github/workflows/lint-css.yml

@@ -33,16 +33,10 @@ jobs:
 
     steps:
       - name: Clone repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
-      - name: Set up Node.js
+      - name: Set up Javascript environment
-        uses: actions/setup-node@v3
+        uses: ./.github/actions/setup-javascript
-        with:
-          cache: yarn
-          node-version-file: '.nvmrc'
-
-      - name: Install all yarn packages
-        run: yarn --frozen-lockfile
 
       - uses: xt0rted/stylelint-problem-matcher@v1
 

.github/workflows/lint-haml.yml

@@ -28,18 +28,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Clone repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
-      - name: Install native Ruby dependencies
+      - name: Set up Ruby environment
-        run: |
+        uses: ./.github/actions/setup-ruby
-          sudo apt-get update
-          sudo apt-get install -y libicu-dev libidn11-dev
-
-      - name: Set up Ruby
-        uses: ruby/setup-ruby@v1
-        with:
-          ruby-version: .ruby-version
-          bundler-cache: true
 
       - name: Run haml-lint
         run: |

.github/workflows/lint-js.yml

@@ -37,16 +37,10 @@ jobs:
 
     steps:
       - name: Clone repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
-      - name: Set up Node.js
+      - name: Set up Javascript environment
-        uses: actions/setup-node@v3
+        uses: ./.github/actions/setup-javascript
-        with:
-          cache: yarn
-          node-version-file: '.nvmrc'
-
-      - name: Install all yarn packages
-        run: yarn --frozen-lockfile
 
       - name: ESLint
         run: yarn lint:js --max-warnings 0

.github/workflows/lint-json.yml

@@ -29,16 +29,10 @@ jobs:
 
     steps:
       - name: Clone repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
-      - name: Set up Node.js
+      - name: Set up Javascript environment
-        uses: actions/setup-node@v3
+        uses: ./.github/actions/setup-javascript
-        with:
-          cache: yarn
-          node-version-file: '.nvmrc'
-
-      - name: Install all yarn packages
-        run: yarn --frozen-lockfile
 
       - name: Prettier
         run: yarn lint:json

.github/workflows/lint-md.yml

@@ -29,16 +29,10 @@ jobs:
 
     steps:
       - name: Clone repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
-      - name: Set up Node.js
+      - name: Set up Javascript environment
-        uses: actions/setup-node@v3
+        uses: ./.github/actions/setup-javascript
-        with:
-          cache: yarn
-          node-version-file: '.nvmrc'
-
-      - name: Install all yarn packages
-        run: yarn --frozen-lockfile
 
       - name: Prettier
         run: yarn lint:md

.github/workflows/lint-ruby.yml

@@ -8,7 +8,7 @@ on:
       - 'Gemfile*'
       - '.rubocop*.yml'
       - '.ruby-version'
-      - '.bundler-audit.yml'
+      - 'config/brakeman.ignore'
       - '**/*.rb'
       - '**/*.rake'
       - '.github/workflows/lint-ruby.yml'
@@ -18,7 +18,7 @@ on:
       - 'Gemfile*'
       - '.rubocop*.yml'
       - '.ruby-version'
-      - '.bundler-audit.yml'
+      - 'config/brakeman.ignore'
       - '**/*.rb'
       - '**/*.rake'
       - '.github/workflows/lint-ruby.yml'
@@ -29,16 +29,10 @@ jobs:
 
     steps:
       - name: Clone repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
-      - name: Install native Ruby dependencies
+      - name: Set up Ruby environment
-        run: sudo apt-get install -y libicu-dev libidn11-dev
+        uses: ./.github/actions/setup-ruby
-
-      - name: Set up Ruby
-        uses: ruby/setup-ruby@v1
-        with:
-          ruby-version: .ruby-version
-          bundler-cache: true
 
       - name: Set-up RuboCop Problem Matcher
         uses: r7kamura/rubocop-problem-matchers-action@v1
@@ -46,5 +40,6 @@ jobs:
       - name: Run rubocop
         run: bundle exec rubocop
 
-      - name: Run bundler-audit
+      - name: Run brakeman
-        run: bundle exec bundler-audit
+        if: always() # Run both checks, even if the first failed
+        run: bundle exec brakeman

.github/workflows/lint-yml.yml

@@ -31,16 +31,10 @@ jobs:
 
     steps:
       - name: Clone repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
-      - name: Set up Node.js
+      - name: Set up Javascript environment
-        uses: actions/setup-node@v3
+        uses: ./.github/actions/setup-javascript
-        with:
-          cache: yarn
-          node-version-file: '.nvmrc'
-
-      - name: Install all yarn packages
-        run: yarn --frozen-lockfile
 
       - name: Prettier
         run: yarn lint:yml

.github/workflows/rebase-needed.yml

@@ -1,17 +1,8 @@
 name: PR Needs Rebase
 
 on:
-  push:
+  schedule:
-    branches-ignore:
+    - cron: '0 * * * *'
-      - 'dependabot/**'
-      - 'renovate/**'
-      - 'l10n_main'
-  pull_request_target:
-    branches-ignore:
-      - 'dependabot/**'
-      - 'renovate/**'
-      - 'l10n_main'
-    types: [synchronize]
 
 permissions:
   pull-requests: write
@@ -32,5 +23,5 @@ jobs:
           repoToken: '${{ secrets.GITHUB_TOKEN }}'
           commentOnClean: This pull request has resolved merge conflicts and is ready for review.
           commentOnDirty: This pull request has merge conflicts that must be resolved before it can be merged.
-          retryMax: 10
+          retryMax: 30
           continueOnMissingPermissions: false

.github/workflows/test-image-build.yml

@@ -0,0 +1,35 @@
+name: Test container image build
+on:
+  pull_request:
+    paths:
+      - .github/workflows/build-nightly.yml
+      - .github/workflows/build-push-pr.yml
+      - .github/workflows/build-releases.yml
+      - .github/workflows/test-image-build.yml
+      - Dockerfile
+      - streaming/Dockerfile
+permissions:
+  contents: read
+
+jobs:
+  build-image:
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}
+      cancel-in-progress: true
+
+    uses: ./.github/workflows/build-container-image.yml
+    with:
+      file_to_build: Dockerfile
+      platforms: linux/amd64 # Testing only on native platform so it is performant
+      cache: true
+
+  build-image-streaming:
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}-streaming
+      cancel-in-progress: true
+
+    uses: ./.github/workflows/build-container-image.yml
+    with:
+      file_to_build: streaming/Dockerfile
+      platforms: linux/amd64 # Testing only on native platform so it is performant
+      cache: true

.github/workflows/test-js.yml

@@ -33,16 +33,10 @@ jobs:
 
     steps:
       - name: Clone repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
-      - name: Set up Node.js
+      - name: Set up Javascript environment
-        uses: actions/setup-node@v3
+        uses: ./.github/actions/setup-javascript
-        with:
-          cache: yarn
-          node-version-file: '.nvmrc'
-
-      - name: Install all yarn packages
-        run: yarn --frozen-lockfile
 
       - name: Jest testing
         run: yarn jest --reporters github-actions summary

.github/workflows/test-migrations-one-step.yml

@@ -70,39 +70,16 @@ jobs:
       BUNDLE_RETRY: 3
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
-      - name: Install native Ruby dependencies
+      - name: Set up Ruby environment
-        run: |
+        uses: ./.github/actions/setup-ruby
-          sudo apt-get update
-          sudo apt-get install -y libicu-dev libidn11-dev
-
-      - name: Set up bundler cache
-        uses: ruby/setup-ruby@v1
-        with:
-          ruby-version: .ruby-version
-          bundler-cache: true
 
       - name: Create database
         run: './bin/rails db:create'
 
-      - name: Run migrations up to v2.0.0
+      - name: Run historical migrations with data population
-        run: './bin/rails db:migrate VERSION=20171010025614'
+        run: './bin/rails tests:migrations:prepare_database'
-
-      - name: Populate database with test data
-        run: './bin/rails tests:migrations:populate_v2'
-
-      - name: Run migrations up to v2.4.0
-        run: './bin/rails db:migrate VERSION=20180514140000'
-
-      - name: Populate database with test data
-        run: './bin/rails tests:migrations:populate_v2_4'
-
-      - name: Run migrations up to v2.4.3
-        run: './bin/rails db:migrate VERSION=20180707154237'
-
-      - name: Populate database with test data
-        run: './bin/rails tests:migrations:populate_v2_4_3'
 
       - name: Run all remaining migrations
         run: './bin/rails db:migrate'

.github/workflows/test-migrations-two-step.yml

@@ -45,6 +45,7 @@ jobs:
           --health-retries 5
         ports:
           - 5432:5432
+
       redis:
         image: redis:7-alpine
         options: >-
@@ -69,44 +70,19 @@ jobs:
       BUNDLE_RETRY: 3
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
-      - name: Install native Ruby dependencies
+      - name: Set up Ruby environment
-        run: |
+        uses: ./.github/actions/setup-ruby
-          sudo apt-get update
-          sudo apt-get install -y libicu-dev libidn11-dev
-
-      - name: Set up bundler cache
-        uses: ruby/setup-ruby@v1
-        with:
-          ruby-version: .ruby-version
-          bundler-cache: true
 
       - name: Create database
         run: './bin/rails db:create'
 
-      - name: Run migrations up to v2.0.0
+      - name: Run historical migrations with data population
-        run: './bin/rails db:migrate VERSION=20171010025614'
+        run: './bin/rails tests:migrations:prepare_database'
-
-      - name: Populate database with test data
-        run: './bin/rails tests:migrations:populate_v2'
-
-      - name: Run pre-deployment migrations up to v2.4.0
-        run: './bin/rails db:migrate VERSION=20180514140000'
         env:
           SKIP_POST_DEPLOYMENT_MIGRATIONS: true
 
-      - name: Populate database with test data
-        run: './bin/rails tests:migrations:populate_v2_4'
-
-      - name: Run migrations up to v2.4.3
-        run: './bin/rails db:migrate VERSION=20180707154237'
-        env:
-          SKIP_POST_DEPLOYMENT_MIGRATIONS: true
-
-      - name: Populate database with test data
-        run: './bin/rails tests:migrations:populate_v2_4_3'
-
       - name: Run all remaining pre-deployment migrations
         run: './bin/rails db:migrate'
         env:

.github/workflows/test-ruby.yml

@@ -32,38 +32,31 @@ jobs:
       SECRET_KEY_BASE: precompile_placeholder
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
-      - name: Set up Node.js
+      - name: Set up Ruby environment
-        uses: actions/setup-node@v3
+        uses: ./.github/actions/setup-ruby
+
+      - name: Set up Javascript environment
+        uses: ./.github/actions/setup-javascript
         with:
-          cache: yarn
+          onlyProduction: 'true'
-          node-version-file: '.nvmrc'
 
-      - name: Install native Ruby dependencies
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y libicu-dev libidn11-dev
-
-      - name: Set up bundler cache
-        uses: ruby/setup-ruby@v1
-        with:
-          ruby-version: .ruby-version
-          bundler-cache: true
-
-      - run: yarn --frozen-lockfile --production
       - name: Precompile assets
         # Previously had set this, but it's not supported
         # export NODE_OPTIONS=--openssl-legacy-provider
         run: |-
           ./bin/rails assets:precompile
 
-      - uses: actions/upload-artifact@v3
+      - name: Archive asset artifacts
+        run: |
+          tar --exclude={"*.br","*.gz"} -zcf artifacts.tar.gz public/assets public/packs*
+
+      - uses: actions/upload-artifact@v4
         if: matrix.mode == 'test'
         with:
           path: |-
-            ./public/assets
+            ./artifacts.tar.gz
-            ./public/packs-test
           name: ${{ github.sha }}
           retention-days: 0
 
@@ -101,14 +94,18 @@ jobs:
       DB_HOST: localhost
       DB_USER: postgres
       DB_PASS: postgres
-      DISABLE_SIMPLECOV: true
+      DISABLE_SIMPLECOV: ${{ matrix.ruby-version != '.ruby-version' }}
       RAILS_ENV: test
       ALLOW_NOPAM: true
       PAM_ENABLED: true
       PAM_DEFAULT_SERVICE: pam_test
       PAM_CONTROLLED_SERVICE: pam_test_controlled
+      OIDC_ENABLED: true
+      OIDC_SCOPE: read
+      SAML_ENABLED: true
+      CAS_ENABLED: true
       BUNDLE_WITH: 'pam_authentication test'
-      CI_JOBS: ${{ matrix.ci_job }}/4
+      GITHUB_RSPEC: ${{ matrix.ruby-version == '.ruby-version' && github.event.pull_request && 'true' }}
 
     strategy:
       fail-fast: false
@@ -117,35 +114,218 @@ jobs:
           - '3.0'
           - '3.1'
           - '.ruby-version'
-        ci_job:
-          - 1
-          - 2
-          - 3
-          - 4
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
-          path: './public'
+          path: './'
           name: ${{ github.sha }}
 
-      - name: Update package index
+      - name: Expand archived asset artifacts
-        run: sudo apt-get update
+        run: |
+          tar xvzf artifacts.tar.gz
 
-      - name: Install native Ruby dependencies
+      - name: Set up Ruby environment
-        run: sudo apt-get install -y libicu-dev libidn11-dev
+        uses: ./.github/actions/setup-ruby
-
-      - name: Install additional system dependencies
-        run: sudo apt-get install -y ffmpeg imagemagick libpam-dev
-
-      - name: Set up bundler cache
-        uses: ruby/setup-ruby@v1
         with:
           ruby-version: ${{ matrix.ruby-version}}
-          bundler-cache: true
+          additional-system-dependencies: ffmpeg imagemagick libpam-dev
 
       - name: Load database schema
         run: './bin/rails db:create db:schema:load db:seed'
 
-      - run: bundle exec rake rspec_chunked
+      - run: bin/rspec
+
+      - name: Upload coverage reports to Codecov
+        if: matrix.ruby-version == '.ruby-version'
+        uses: codecov/codecov-action@v3
+        with:
+          files: coverage/lcov/mastodon.lcov
+
+  test-e2e:
+    name: End to End testing
+    runs-on: ubuntu-latest
+
+    needs:
+      - build
+
+    services:
+      postgres:
+        image: postgres:14-alpine
+        env:
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_USER: postgres
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+
+      redis:
+        image: redis:7-alpine
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 6379:6379
+
+    env:
+      DB_HOST: localhost
+      DB_USER: postgres
+      DB_PASS: postgres
+      DISABLE_SIMPLECOV: true
+      RAILS_ENV: test
+      BUNDLE_WITH: test
+
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby-version:
+          - '3.0'
+          - '3.1'
+          - '.ruby-version'
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/download-artifact@v4
+        with:
+          path: './public'
+          name: ${{ github.sha }}
+
+      - name: Set up Ruby environment
+        uses: ./.github/actions/setup-ruby
+        with:
+          ruby-version: ${{ matrix.ruby-version}}
+          additional-system-dependencies: ffmpeg imagemagick
+
+      - name: Set up Javascript environment
+        uses: ./.github/actions/setup-javascript
+
+      - name: Load database schema
+        run: './bin/rails db:create db:schema:load db:seed'
+
+      - run: bundle exec rake spec:system
+
+      - name: Archive logs
+        uses: actions/upload-artifact@v4
+        if: failure()
+        with:
+          name: e2e-logs-${{ matrix.ruby-version }}
+          path: log/
+
+      - name: Archive test screenshots
+        uses: actions/upload-artifact@v4
+        if: failure()
+        with:
+          name: e2e-screenshots
+          path: tmp/screenshots/
+
+  test-search:
+    name: Elastic Search integration testing
+    runs-on: ubuntu-latest
+
+    needs:
+      - build
+
+    services:
+      postgres:
+        image: postgres:14-alpine
+        env:
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_USER: postgres
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+
+      redis:
+        image: redis:7-alpine
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 6379:6379
+
+      search:
+        image: ${{ matrix.search-image }}
+        env:
+          discovery.type: single-node
+          xpack.security.enabled: false
+        options: >-
+          --health-cmd "curl http://localhost:9200/_cluster/health"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 10
+        ports:
+          - 9200:9200
+
+    env:
+      DB_HOST: localhost
+      DB_USER: postgres
+      DB_PASS: postgres
+      DISABLE_SIMPLECOV: true
+      RAILS_ENV: test
+      BUNDLE_WITH: test
+      ES_ENABLED: true
+      ES_HOST: localhost
+      ES_PORT: 9200
+
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby-version:
+          - '3.0'
+          - '3.1'
+          - '.ruby-version'
+        search-image:
+          - docker.elastic.co/elasticsearch/elasticsearch:7.17.13
+        include:
+          - ruby-version: '.ruby-version'
+            search-image: docker.elastic.co/elasticsearch/elasticsearch:8.10.2
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/download-artifact@v4
+        with:
+          path: './public'
+          name: ${{ github.sha }}
+
+      - name: Set up Ruby environment
+        uses: ./.github/actions/setup-ruby
+        with:
+          ruby-version: ${{ matrix.ruby-version}}
+          additional-system-dependencies: ffmpeg imagemagick
+
+      - name: Set up Javascript environment
+        uses: ./.github/actions/setup-javascript
+
+      - name: Load database schema
+        run: './bin/rails db:create db:schema:load db:seed'
+
+      - run: bin/rspec --tag search
+
+      - name: Archive logs
+        uses: actions/upload-artifact@v4
+        if: failure()
+        with:
+          name: test-search-logs-${{ matrix.ruby-version }}
+          path: log/
+
+      - name: Archive test screenshots
+        uses: actions/upload-artifact@v4
+        if: failure()
+        with:
+          name: test-search-screenshots
+          path: tmp/screenshots/

.gitignore

@@ -31,9 +31,6 @@
 # Ignore Vagrant files
 .vagrant/
 
-# Ignore Capistrano customizations
-/config/deploy/*
-
 # Ignore IDE files
 .vscode/
 .idea/
@@ -58,6 +55,15 @@ npm-debug.log
 yarn-error.log
 yarn-debug.log
 
+# From https://yarnpkg.com/getting-started/qa#which-files-should-be-gitignored
+.pnp.*
+.yarn/*
+!.yarn/patches
+!.yarn/plugins
+!.yarn/releases
+!.yarn/sdks
+!.yarn/versions
+
 # Ignore vagrant log files
 *-cloudimg-console.log
 

.haml-lint.yml

@@ -12,3 +12,5 @@ linters:
     enabled: true
   MiddleDot:
     enabled: true
+  LineLength:
+    max: 320

.haml-lint_todo.yml

@@ -1,106 +1,13 @@
 # This configuration was generated by
 # `haml-lint --auto-gen-config`
-# on 2023-03-15 00:55:01 -0400 using Haml-Lint version 0.45.0.
+# on 2024-01-09 11:30:07 -0500 using Haml-Lint version 0.53.0.
 # The point is for the user to remove these configuration records
 # one by one as the lints are removed from the code base.
 # Note that changes in the inspected code, or installation of new
 # versions of Haml-Lint, may require this file to be generated again.
 
 linters:
-  # Offense count: 63
+  # Offense count: 1
-  RuboCop:
-    exclude:
-      - 'app/views/accounts/_og.html.haml'
-      - 'app/views/admin/account_warnings/_account_warning.html.haml'
-      - 'app/views/admin/accounts/index.html.haml'
-      - 'app/views/admin/accounts/show.html.haml'
-      - 'app/views/admin/announcements/edit.html.haml'
-      - 'app/views/admin/announcements/new.html.haml'
-      - 'app/views/admin/disputes/appeals/_appeal.html.haml'
-      - 'app/views/admin/domain_blocks/edit.html.haml'
-      - 'app/views/admin/domain_blocks/new.html.haml'
-      - 'app/views/admin/ip_blocks/new.html.haml'
-      - 'app/views/admin/reports/actions/preview.html.haml'
-      - 'app/views/admin/reports/index.html.haml'
-      - 'app/views/admin/reports/show.html.haml'
-      - 'app/views/admin/roles/_form.html.haml'
-      - 'app/views/admin/settings/about/show.html.haml'
-      - 'app/views/admin/settings/appearance/show.html.haml'
-      - 'app/views/admin/settings/registrations/show.html.haml'
-      - 'app/views/admin/statuses/show.html.haml'
-      - 'app/views/auth/registrations/new.html.haml'
-      - 'app/views/disputes/strikes/show.html.haml'
-      - 'app/views/filters/_filter_fields.html.haml'
-      - 'app/views/invites/_form.html.haml'
-      - 'app/views/layouts/application.html.haml'
-      - 'app/views/layouts/error.html.haml'
-      - 'app/views/notification_mailer/_status.html.haml'
-      - 'app/views/settings/applications/_fields.html.haml'
-      - 'app/views/settings/imports/show.html.haml'
-      - 'app/views/settings/preferences/appearance/show.html.haml'
-      - 'app/views/settings/preferences/other/show.html.haml'
-      - 'app/views/statuses/_detailed_status.html.haml'
-      - 'app/views/statuses/_poll.html.haml'
-      - 'app/views/statuses/show.html.haml'
-      - 'app/views/statuses_cleanup/show.html.haml'
-      - 'app/views/user_mailer/warning.html.haml'
-
-  # Offense count: 913
   LineLength:
-    enabled: false
-
-  # Offense count: 22
-  UnnecessaryStringOutput:
     exclude:
-      - 'app/views/accounts/show.html.haml'
-      - 'app/views/admin/custom_emojis/_custom_emoji.html.haml'
-      - 'app/views/admin/relays/_relay.html.haml'
-      - 'app/views/admin/rules/_rule.html.haml'
-      - 'app/views/admin/statuses/index.html.haml'
-      - 'app/views/auth/registrations/_sessions.html.haml'
-      - 'app/views/disputes/strikes/show.html.haml'
-      - 'app/views/notification_mailer/_status.html.haml'
-      - 'app/views/settings/two_factor_authentication_methods/index.html.haml'
-      - 'app/views/statuses/_detailed_status.html.haml'
-      - 'app/views/statuses/_poll.html.haml'
-      - 'app/views/statuses/_simple_status.html.haml'
-      - 'app/views/user_mailer/suspicious_sign_in.html.haml'
-      - 'app/views/user_mailer/webauthn_credential_added.html.haml'
-      - 'app/views/user_mailer/webauthn_credential_deleted.html.haml'
-      - 'app/views/user_mailer/welcome.html.haml'
-
-  # Offense count: 3
-  ViewLength:
-    exclude:
-      - 'app/views/admin/accounts/show.html.haml'
-      - 'app/views/admin/reports/show.html.haml'
-      - 'app/views/disputes/strikes/show.html.haml'
-
-  # Offense count: 41
-  InstanceVariables:
-    exclude:
-      - 'app/views/admin/reports/_actions.html.haml'
       - 'app/views/admin/roles/_form.html.haml'
-      - 'app/views/admin/webhooks/_form.html.haml'
-      - 'app/views/auth/registrations/_sessions.html.haml'
-      - 'app/views/auth/registrations/_status.html.haml'
-      - 'app/views/auth/sessions/two_factor/_otp_authentication_form.html.haml'
-      - 'app/views/authorize_interactions/_post_follow_actions.html.haml'
-      - 'app/views/invites/_form.html.haml'
-      - 'app/views/relationships/_account.html.haml'
-      - 'app/views/shared/_og.html.haml'
-      - 'app/views/statuses/_status.html.haml'
-
-  # Offense count: 6
-  ConsecutiveSilentScripts:
-    exclude:
-      - 'app/views/admin/settings/shared/_links.html.haml'
-      - 'app/views/settings/login_activities/_login_activity.html.haml'
-      - 'app/views/statuses/_poll.html.haml'
-
-  # Offense count: 3
-  IdNames:
-    exclude:
-      - 'app/views/authorize_interactions/error.html.haml'
-      - 'app/views/oauth/authorizations/error.html.haml'
-      - 'app/views/shared/_error_messages.html.haml'

.nvmrc

@@ -1 +1 @@
-16.20
+20.11

.prettierignore

@@ -31,9 +31,6 @@
 # Ignore Vagrant files
 .vagrant/
 
-# Ignore Capistrano customizations
-/config/deploy/*
-
 # Ignore IDE files
 .vscode/
 .idea/
@@ -77,6 +74,8 @@ app/javascript/styles/mastodon/reset.scss
 # Ignore the generated AUTHORS.md
 AUTHORS.md
 
+!lint-staged.config.js
+
 # Ignore glitch-soc emoji map file
 /app/javascript/flavours/glitch/features/emoji/emoji_map.json
 

.rubocop.yml

@@ -24,11 +24,11 @@ AllCops:
   Exclude:
     - db/schema.rb
     - 'bin/*'
-    - 'Rakefile'
     - 'node_modules/**/*'
     - 'Vagrantfile'
     - 'vendor/**/*'
-    - 'lib/json_ld/*' # Generated files
+    - 'config/initializers/json_ld*' # Generated files
+    - 'lib/mastodon/migration_helpers.rb' # Vendored from GitLab
     - 'lib/templates/**/*'
 
 # Reason: Prefer Hashes without extreme indentation
@@ -39,14 +39,7 @@ Layout/FirstHashElementIndentation:
 # Reason: Currently disabled in .rubocop_todo.yml
 # https://docs.rubocop.org/rubocop/cops_layout.html#layoutlinelength
 Layout/LineLength:
-  AllowedPatterns:
+  Max: 320 # Default of 120 causes a duplicate entry in generated todo file
-    # Allow comments to be long lines
-    - !ruby/regexp / \# .*$/
-    - !ruby/regexp /^\# .*$/
-  Exclude:
-    - 'lib/mastodon/cli/*.rb'
-    - db/*migrate/**/*
-    - db/seeds/**/*
 
 # Reason:
 # https://docs.rubocop.org/rubocop/cops_lint.html#lintuselessaccessmodifier
@@ -79,13 +72,6 @@ Metrics/ModuleLength:
 # Reason: Currently disabled in .rubocop_todo.yml
 # https://docs.rubocop.org/rubocop/cops_metrics.html#metricsabcsize
 Metrics/AbcSize:
-  Exclude:
-    - 'lib/mastodon/cli/*.rb'
-    - db/*migrate/**/*
-
-# Reason:
-# https://docs.rubocop.org/rubocop/cops_metrics.html#metricsblocknesting
-Metrics/BlockNesting:
   Exclude:
     - 'lib/mastodon/cli/*.rb'
 
@@ -94,7 +80,6 @@ Metrics/BlockNesting:
 Metrics/CyclomaticComplexity:
   Exclude:
     - lib/mastodon/cli/*.rb
-    - db/*migrate/**/*
 
 # Reason:
 # https://docs.rubocop.org/rubocop/cops_metrics.html#metricsparameterlists
@@ -118,26 +103,42 @@ Rails/Exit:
     - 'config/boot.rb'
     - 'lib/mastodon/cli/*.rb'
 
-# Reason: Some single letter camel case files shouldn't be split
+# Reason: Conflicts with `Lint/UselessMethodDefinition` for inherited controller actions
+# https://docs.rubocop.org/rubocop-rails/cops_rails.html#railslexicallyscopedactionfilter
+Rails/LexicallyScopedActionFilter:
+  Exclude:
+    - 'app/controllers/auth/*'
+
+# Reason: These tasks are doing local work which do not need full env loaded
+# https://docs.rubocop.org/rubocop-rails/cops_rails.html#railsrakeenvironment
+Rails/RakeEnvironment:
+  Exclude:
+    - 'lib/tasks/auto_annotate_models.rake'
+    - 'lib/tasks/emojis.rake'
+    - 'lib/tasks/mastodon.rake'
+    - 'lib/tasks/repo.rake'
+    - 'lib/tasks/statistics.rake'
+
+# Reason: There are appropriate times to use these features
+# https://docs.rubocop.org/rubocop-rails/cops_rails.html#railsskipsmodelvalidations
+Rails/SkipsModelValidations:
+  Enabled: false
+
+# Reason: We want to preserve the ability to migrate from arbitrary old versions,
+# and cannot guarantee that every installation has run every migration as they upgrade.
+# https://docs.rubocop.org/rubocop-rails/cops_rails.html#railsunusedignoredcolumns
+Rails/UnusedIgnoredColumns:
+  Enabled: false
+
+# Reason: Prevailing style choice
+# https://docs.rubocop.org/rubocop-rails/cops_rails.html#railsnegateinclude
+Rails/NegateInclude:
+  Enabled: false
+
+# Reason: Deprecated cop, will be removed in 3.0, replaced by SpecFilePathFormat
 # https://docs.rubocop.org/rubocop-rspec/cops_rspec.html#rspecfilepath
 RSpec/FilePath:
-  CustomTransform:
+  Enabled: false
-    ActivityPub: activitypub # Ignore the snake_case due to the amount of files to rename
-    DeepL: deepl
-    FetchOEmbedService: fetch_oembed_service
-    JsonLdHelper: jsonld_helper
-    OEmbedController: oembed_controller
-    OStatus: ostatus
-    NodeInfoController: nodeinfo_controller # NodeInfo isn't snake_cased for any of the instances
-  Exclude:
-    - 'spec/config/initializers/rack_attack_spec.rb' # namespaces usually have separate folder
-    - 'spec/lib/sanitize_config_spec.rb' # namespaces usually have separate folder
-    - 'spec/controllers/concerns/account_controller_concern_spec.rb' # Concerns describe ApplicationController and don't fit naming
-    - 'spec/controllers/concerns/export_controller_concern_spec.rb'
-    - 'spec/controllers/concerns/localized_spec.rb'
-    - 'spec/controllers/concerns/rate_limit_headers_spec.rb'
-    - 'spec/controllers/concerns/signature_verification_spec.rb'
-    - 'spec/controllers/concerns/user_tracking_concern_spec.rb'
 
 # Reason:
 # https://docs.rubocop.org/rubocop-rspec/cops_rspec.html#rspecnamedsubject
@@ -154,6 +155,16 @@ RSpec/NotToNot:
 RSpec/Rails/HttpStatus:
   EnforcedStyle: numeric
 
+# Reason: Match overrides from Rspec/FilePath rule above
+# https://docs.rubocop.org/rubocop-rspec/cops_rspec.html#rspecspecfilepathformat
+RSpec/SpecFilePathFormat:
+  CustomTransform:
+    ActivityPub: activitypub
+    DeepL: deepl
+    FetchOEmbedService: fetch_oembed_service
+    OEmbedController: oembed_controller
+    OStatus: ostatus
+
 # Reason:
 # https://docs.rubocop.org/rubocop/cops_style.html#styleclassandmodulechildren
 Style/ClassAndModuleChildren:
@@ -192,6 +203,11 @@ Style/RedundantBegin:
 Style/RescueStandardError:
   EnforcedStyle: implicit
 
+# Reason: Simplify some spec layouts
+# https://docs.rubocop.org/rubocop/cops_style.html#stylesemicolon
+Style/Semicolon:
+  AllowAsExpressionSeparator: true
+
 # Reason: Originally disabled for CodeClimate, and no config consensus has been found
 # https://docs.rubocop.org/rubocop/cops_style.html#stylesymbolarray
 Style/SymbolArray:

.ruby-version

@@ -1 +1 @@
-3.2.2
+3.2.3

.simplecov

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+if ENV['CI']
+  require 'simplecov-lcov'
+  SimpleCov::Formatter::LcovFormatter.config.report_with_single_file = true
+  SimpleCov.formatter = SimpleCov::Formatter::LcovFormatter
+else
+  SimpleCov.formatter = SimpleCov::Formatter::HTMLFormatter
+end
+
+SimpleCov.start 'rails' do
+  enable_coverage :branch
+
+  add_filter 'lib/linter'
+
+  add_group 'Libraries', 'lib'
+  add_group 'Policies', 'app/policies'
+  add_group 'Presenters', 'app/presenters'
+  add_group 'Serializers', 'app/serializers'
+  add_group 'Services', 'app/services'
+  add_group 'Validators', 'app/validators'
+end

.watchmanconfig

@@ -0,0 +1,3 @@
+{
+  "ignore_dirs": ["node_modules/", "public/"]
+}

.yarn/patches/babel-plugin-lodash-npm-3.3.4-c7161075b6.patch

@@ -0,0 +1,13 @@
+diff --git a/lib/index.js b/lib/index.js
+index 16ed6be8be8f555cc99096c2ff60954b42dc313d..d009c069770d066ad0db7ad02de1ea473a29334e 100644
+--- a/lib/index.js
++++ b/lib/index.js
+@@ -99,7 +99,7 @@ function lodash(_ref) {
+ 
+         var node = _ref3;
+ 
+-        if ((0, _types.isModuleDeclaration)(node)) {
++        if ((0, _types.isImportDeclaration)(node)  || (0, _types.isExportDeclaration)(node)) {
+           isModule = true;
+           break;
+         }

.yarnclean

@@ -1,49 +0,0 @@
-# test directories
-__tests__
-test
-tests
-powered-test
-
-# asset directories
-docs
-doc
-website
-images
-# assets
-
-# examples
-example
-examples
-
-# code coverage directories
-coverage
-.nyc_output
-
-# build scripts
-Makefile
-Gulpfile.js
-Gruntfile.js
-
-# configs
-.tern-project
-.gitattributes
-.editorconfig
-.*ignore
-.eslintrc
-.jshintrc
-.flowconfig
-.documentup.json
-.yarn-metadata.json
-.*.yml
-*.yml
-
-# misc
-*.gz
-*.md
-
-# for specific ignore
-!.svgo.yml
-!sass-lint/**/*.yml
-
-# breaks lint-staged or generally anything using https://github.com/eemeli/yaml/issues/384
-!**/yaml/dist/**/doc

.yarnrc.yml

@@ -0,0 +1 @@
+nodeLinker: node-modules

CONTRIBUTING.md

@@ -50,6 +50,10 @@ You can contribute in the following ways:
 
 If your contributions are accepted into Mastodon, you can request to be paid through [our OpenCollective](https://opencollective.com/mastodon).
 
+## API Changes and Additions
+
+Please note that any changes or additions made to the API should have an accompanying pull request on [our documentation repository](https://github.com/mastodon/documentation).
+
 ## Bug reports
 
 Bug reports and feature suggestions must use descriptive and concise titles and be submitted to [GitHub Issues](https://github.com/mastodon/mastodon/issues). Please use the search function to make sure that you are not submitting duplicates, and that a similar report or request has not already been resolved or rejected.

Capfile

@@ -1,15 +0,0 @@
-# frozen_string_literal: true
-
-require 'capistrano/setup'
-require 'capistrano/deploy'
-require 'capistrano/scm/git'
-
-install_plugin Capistrano::SCM::Git
-
-require 'capistrano/rbenv'
-require 'capistrano/bundler'
-require 'capistrano/yarn'
-require 'capistrano/rails/assets'
-require 'capistrano/rails/migrations'
-
-Dir.glob('lib/capistrano/tasks/*.rake').each { |r| import r }

Dockerfile

@@ -1,104 +1,260 @@
 # syntax=docker/dockerfile:1.4
-# This needs to be bullseye-slim because the Ruby image is built on bullseye-slim
-ARG NODE_VERSION="16.20-bullseye-slim"
 
-FROM ghcr.io/moritzheiber/ruby-jemalloc:3.2.2-slim as ruby
+# Please see https://docs.docker.com/engine/reference/builder for information about
-FROM node:${NODE_VERSION} as build
+# the extended buildx capabilities used in this file.
+# Make sure multiarch TARGETPLATFORM is available for interpolation
+# See: https://docs.docker.com/build/building/multi-platform/
+ARG TARGETPLATFORM=${TARGETPLATFORM}
+ARG BUILDPLATFORM=${BUILDPLATFORM}
 
-COPY --link --from=ruby /opt/ruby /opt/ruby
+# Ruby image to use for base image, change with [--build-arg RUBY_VERSION="3.2.3"]
+ARG RUBY_VERSION="3.2.3"
+# # Node version to use in base image, change with [--build-arg NODE_MAJOR_VERSION="20"]
+ARG NODE_MAJOR_VERSION="20"
+# Debian image to use for base image, change with [--build-arg DEBIAN_VERSION="bookworm"]
+ARG DEBIAN_VERSION="bookworm"
+# Node image to use for base image based on combined variables (ex: 20-bookworm-slim)
+FROM docker.io/node:${NODE_MAJOR_VERSION}-${DEBIAN_VERSION}-slim as node
+# Ruby image to use for base image based on combined variables (ex: 3.2.3-slim-bookworm)
+FROM docker.io/ruby:${RUBY_VERSION}-slim-${DEBIAN_VERSION} as ruby
 
-ENV DEBIAN_FRONTEND="noninteractive" \
+# Resulting version string is vX.X.X-MASTODON_VERSION_PRERELEASE+MASTODON_VERSION_METADATA
-    PATH="${PATH}:/opt/ruby/bin"
+# Example: v4.2.0-nightly.2023.11.09+something
-
+# Overwrite existance of 'alpha.0' in version.rb [--build-arg MASTODON_VERSION_PRERELEASE="nightly.2023.11.09"]
-SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+ARG MASTODON_VERSION_PRERELEASE=""
-
+# Append build metadata or fork information to version.rb [--build-arg MASTODON_VERSION_METADATA="something"]
-WORKDIR /opt/mastodon
+ARG MASTODON_VERSION_METADATA=""
-COPY Gemfile* package.json yarn.lock /opt/mastodon/
-
-# hadolint ignore=DL3008
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends build-essential \
-        git \
-        libicu-dev \
-        libidn11-dev \
-        libpq-dev \
-        libjemalloc-dev \
-        zlib1g-dev \
-        libgdbm-dev \
-        libgmp-dev \
-        libssl-dev \
-        libyaml-0-2 \
-        ca-certificates \
-        libreadline8 \
-        python3 \
-        shared-mime-info && \
-    bundle config set --local deployment 'true' && \
-    bundle config set --local without 'development test' && \
-    bundle config set silence_root_warning true && \
-    bundle install -j"$(nproc)" && \
-    yarn install --pure-lockfile --production --network-timeout 600000 && \
-    yarn cache clean
-
-FROM node:${NODE_VERSION}
-
-# Use those args to specify your own version flags & suffixes
-ARG MASTODON_VERSION_FLAGS=""
-ARG MASTODON_VERSION_SUFFIX=""
 
+# Allow Ruby on Rails to serve static files
+# See: https://docs.joinmastodon.org/admin/config/#rails_serve_static_files
+ARG RAILS_SERVE_STATIC_FILES="true"
+# Allow to use YJIT compiler
+# See: https://github.com/ruby/ruby/blob/master/doc/yjit/yjit.md
+ARG RUBY_YJIT_ENABLE="1"
+# Timezone used by the Docker container and runtime, change with [--build-arg TZ=Europe/Berlin]
+ARG TZ="Etc/UTC"
+# Linux UID (user id) for the mastodon user, change with [--build-arg UID=1234]
 ARG UID="991"
+# Linux GID (group id) for the mastodon user, change with [--build-arg GID=1234]
 ARG GID="991"
 
-COPY --link --from=ruby /opt/ruby /opt/ruby
+# Apply Mastodon build options based on options above
+ENV \
+# Apply Mastodon version information
+  MASTODON_VERSION_PRERELEASE="${MASTODON_VERSION_PRERELEASE}" \
+  MASTODON_VERSION_METADATA="${MASTODON_VERSION_METADATA}" \
+# Apply Mastodon static files and YJIT options
+  RAILS_SERVE_STATIC_FILES=${RAILS_SERVE_STATIC_FILES} \
+  RUBY_YJIT_ENABLE=${RUBY_YJIT_ENABLE} \
+# Apply timezone
+  TZ=${TZ}
 
-SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+ENV \
+# Configure the IP to bind Mastodon to when serving traffic
+  BIND="0.0.0.0" \
+# Use production settings for Yarn, Node and related nodejs based tools
+  NODE_ENV="production" \
+# Use production settings for Ruby on Rails
+  RAILS_ENV="production" \
+# Add Ruby and Mastodon installation to the PATH
+  DEBIAN_FRONTEND="noninteractive" \
+  PATH="${PATH}:/opt/ruby/bin:/opt/mastodon/bin" \
+# Optimize jemalloc 5.x performance
+  MALLOC_CONF="narenas:2,background_thread:true,thp:never,dirty_decay_ms:1000,muzzy_decay_ms:0"
 
-ENV DEBIAN_FRONTEND="noninteractive" \
+# Set default shell used for running commands
-    PATH="${PATH}:/opt/ruby/bin:/opt/mastodon/bin"
+SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-c"]
 
-# Ignoring these here since we don't want to pin any versions and the Debian image removes apt-get content after use
+ARG TARGETPLATFORM
-# hadolint ignore=DL3008,DL3009
-RUN apt-get update && \
-    echo "Etc/UTC" > /etc/localtime && \
-    groupadd -g "${GID}" mastodon && \
-    useradd -l -u "$UID" -g "${GID}" -m -d /opt/mastodon mastodon && \
-    apt-get -y --no-install-recommends install whois \
-        wget \
-        procps \
-        libssl1.1 \
-        libpq5 \
-        imagemagick \
-        ffmpeg \
-        libjemalloc2 \
-        libicu67 \
-        libidn11 \
-        libyaml-0-2 \
-        file \
-        ca-certificates \
-        tzdata \
-        libreadline8 \
-        tini && \
-    ln -s /opt/mastodon /mastodon
 
-# Note: no, cleaning here since Debian does this automatically
+RUN echo "Target platform is $TARGETPLATFORM"
-# See the file /etc/apt/apt.conf.d/docker-clean within the Docker image's filesystem
 
-COPY --chown=mastodon:mastodon . /opt/mastodon
+RUN \
-COPY --chown=mastodon:mastodon --from=build /opt/mastodon /opt/mastodon
+# Remove automatic apt cache Docker cleanup scripts
+  rm -f /etc/apt/apt.conf.d/docker-clean; \
+# Sets timezone
+  echo "${TZ}" > /etc/localtime; \
+# Creates mastodon user/group and sets home directory
+  groupadd -g "${GID}" mastodon; \
+  useradd -l -u "${UID}" -g "${GID}" -m -d /opt/mastodon mastodon; \
+# Creates /mastodon symlink to /opt/mastodon
+  ln -s /opt/mastodon /mastodon;
 
-ENV RAILS_ENV="production" \
+# Set /opt/mastodon as working directory
-    NODE_ENV="production" \
-    RAILS_SERVE_STATIC_FILES="true" \
-    BIND="0.0.0.0" \
-    MASTODON_VERSION_FLAGS="${MASTODON_VERSION_FLAGS}" \
-    MASTODON_VERSION_SUFFIX="${MASTODON_VERSION_SUFFIX}"
-
-# Set the run user
-USER mastodon
 WORKDIR /opt/mastodon
 
-# Precompile assets
+# hadolint ignore=DL3008,DL3005
-RUN OTP_SECRET=precompile_placeholder SECRET_KEY_BASE=precompile_placeholder rails assets:precompile
+RUN \
+# Mount Apt cache and lib directories from Docker buildx caches
+--mount=type=cache,id=apt-cache-${TARGETPLATFORM},target=/var/cache/apt,sharing=locked \
+--mount=type=cache,id=apt-lib-${TARGETPLATFORM},target=/var/lib/apt,sharing=locked \
+# Apt update & upgrade to check for security updates to Debian image
+  apt-get update; \
+  apt-get dist-upgrade -yq; \
+# Install jemalloc, curl and other necessary components
+  apt-get install -y --no-install-recommends \
+    ca-certificates \
+    curl \
+    ffmpeg \
+    file \
+    imagemagick \
+    libjemalloc2 \
+    patchelf \
+    procps \
+    tini \
+    tzdata \
+    wget \
+  ; \
+# Patch Ruby to use jemalloc
+  patchelf --add-needed libjemalloc.so.2 /usr/local/bin/ruby; \
+# Discard patchelf after use
+  apt-get purge -y \
+    patchelf \
+  ;
 
-# Set the work dir and the container entry point
+# Create temporary build layer from base image
-ENTRYPOINT ["/usr/bin/tini", "--"]
+FROM ruby as build
-EXPOSE 3000 4000
+
+# Copy Node package configuration files into working directory
+COPY package.json yarn.lock .yarnrc.yml /opt/mastodon/
+COPY .yarn /opt/mastodon/.yarn
+
+COPY --from=node /usr/local/bin /usr/local/bin
+COPY --from=node /usr/local/lib /usr/local/lib
+
+ARG TARGETPLATFORM
+
+# hadolint ignore=DL3008
+RUN \
+# Mount Apt cache and lib directories from Docker buildx caches
+--mount=type=cache,id=apt-cache-${TARGETPLATFORM},target=/var/cache/apt,sharing=locked \
+--mount=type=cache,id=apt-lib-${TARGETPLATFORM},target=/var/lib/apt,sharing=locked \
+# Install build tools and bundler dependencies from APT
+  apt-get install -y --no-install-recommends \
+    g++ \
+    gcc \
+    git \
+    libgdbm-dev \
+    libgmp-dev \
+    libicu-dev \
+    libidn-dev \
+    libpq-dev \
+    libssl-dev \
+    make \
+    shared-mime-info \
+    zlib1g-dev \
+  ;
+
+RUN \
+# Configure Corepack
+  rm /usr/local/bin/yarn*; \
+  corepack enable; \
+  corepack prepare --activate;
+
+# Create temporary bundler specific build layer from build layer
+FROM build as bundler
+
+ARG TARGETPLATFORM
+
+# Copy Gemfile config into working directory
+COPY Gemfile* /opt/mastodon/
+
+RUN \
+# Mount Ruby Gem caches
+--mount=type=cache,id=gem-cache-${TARGETPLATFORM},target=/usr/local/bundle/cache/,sharing=locked \
+# Configure bundle to prevent changes to Gemfile and Gemfile.lock
+  bundle config set --global frozen "true"; \
+# Configure bundle to not cache downloaded Gems
+  bundle config set --global cache_all "false"; \
+# Configure bundle to only process production Gems
+  bundle config set --local without "development test"; \
+# Configure bundle to not warn about root user
+  bundle config set silence_root_warning "true"; \
+# Download and install required Gems
+  bundle install -j"$(nproc)";
+
+# Create temporary node specific build layer from build layer
+FROM build as yarn
+
+ARG TARGETPLATFORM
+
+# Copy Node package configuration files into working directory
+COPY package.json yarn.lock .yarnrc.yml /opt/mastodon/
+COPY streaming/package.json /opt/mastodon/streaming/
+COPY .yarn /opt/mastodon/.yarn
+
+# hadolint ignore=DL3008
+RUN \
+--mount=type=cache,id=corepack-cache-${TARGETPLATFORM},target=/usr/local/share/.cache/corepack,sharing=locked \
+--mount=type=cache,id=yarn-cache-${TARGETPLATFORM},target=/usr/local/share/.cache/yarn,sharing=locked \
+# Install Node packages
+  yarn workspaces focus --production @mastodon/mastodon;
+
+# Create temporary assets build layer from build layer
+FROM build as precompiler
+
+# Copy Mastodon sources into precompiler layer
+COPY . /opt/mastodon/
+
+# Copy bundler and node packages from build layer to container
+COPY --from=yarn /opt/mastodon /opt/mastodon/
+COPY --from=bundler /opt/mastodon /opt/mastodon/
+COPY --from=bundler /usr/local/bundle/ /usr/local/bundle/
+
+ARG TARGETPLATFORM
+
+RUN \
+# Use Ruby on Rails to create Mastodon assets
+  OTP_SECRET=precompile_placeholder SECRET_KEY_BASE=precompile_placeholder bundle exec rails assets:precompile; \
+# Cleanup temporary files
+  rm -fr /opt/mastodon/tmp;
+
+# Prep final Mastodon Ruby layer
+FROM ruby as mastodon
+
+ARG TARGETPLATFORM
+
+# hadolint ignore=DL3008
+RUN \
+# Mount Apt cache and lib directories from Docker buildx caches
+--mount=type=cache,id=apt-cache-${TARGETPLATFORM},target=/var/cache/apt,sharing=locked \
+--mount=type=cache,id=apt-lib-${TARGETPLATFORM},target=/var/lib/apt,sharing=locked \
+# Mount Corepack and Yarn caches from Docker buildx caches
+--mount=type=cache,id=corepack-cache-${TARGETPLATFORM},target=/usr/local/share/.cache/corepack,sharing=locked \
+--mount=type=cache,id=yarn-cache-${TARGETPLATFORM},target=/usr/local/share/.cache/yarn,sharing=locked \
+# Apt update install non-dev versions of necessary components
+  apt-get install -y --no-install-recommends \
+    libssl3 \
+    libpq5 \
+    libicu72 \
+    libidn12 \
+    libreadline8 \
+    libyaml-0-2 \
+  ;
+
+# Copy Mastodon sources into final layer
+COPY . /opt/mastodon/
+
+# Copy compiled assets to layer
+COPY --from=precompiler /opt/mastodon/public/packs /opt/mastodon/public/packs
+COPY --from=precompiler /opt/mastodon/public/assets /opt/mastodon/public/assets
+# Copy bundler components to layer
+COPY --from=bundler /usr/local/bundle/ /usr/local/bundle/
+
+RUN \
+# Precompile bootsnap code for faster Rails startup
+  bundle exec bootsnap precompile --gemfile app/ lib/;
+
+RUN \
+# Pre-create and chown system volume to Mastodon user
+  mkdir -p /opt/mastodon/public/system; \
+  chown mastodon:mastodon /opt/mastodon/public/system; \
+# Set Mastodon user as owner of tmp folder
+  chown -R mastodon:mastodon /opt/mastodon/tmp;
+
+# Set the running user for resulting container
+USER mastodon
+# Expose default Puma ports
+EXPOSE 3000
+# Set container tini as default entry point
+ENTRYPOINT ["/usr/bin/tini", "--"]

FEDERATION.md

@@ -1,19 +1,35 @@
-## ActivityPub federation in Mastodon
+# Federation
+
+## Supported federation protocols and standards
+
+- [ActivityPub](https://www.w3.org/TR/activitypub/) (Server-to-Server)
+- [WebFinger](https://webfinger.net/)
+- [Http Signatures](https://datatracker.ietf.org/doc/html/draft-cavage-http-signatures)
+- [NodeInfo](https://nodeinfo.diaspora.software/)
+
+## Supported FEPs
+
+- [FEP-67ff: FEDERATION.md](https://codeberg.org/fediverse/fep/src/branch/main/fep/67ff/fep-67ff.md)
+- [FEP-f1d5: NodeInfo in Fediverse Software](https://codeberg.org/fediverse/fep/src/branch/main/fep/f1d5/fep-f1d5.md)
+- [FEP-8fcf: Followers collection synchronization across servers](https://codeberg.org/fediverse/fep/src/branch/main/fep/8fcf/fep-8fcf.md)
+- [FEP-5feb: Search indexing consent for actors](https://codeberg.org/fediverse/fep/src/branch/main/fep/5feb/fep-5feb.md)
+
+## ActivityPub in Mastodon
 
 Mastodon largely follows the ActivityPub server-to-server specification but it makes uses of some non-standard extensions, some of which are required for interacting with Mastodon at all.
 
-Supported vocabulary: https://docs.joinmastodon.org/spec/activitypub/
+- [Supported ActivityPub vocabulary](https://docs.joinmastodon.org/spec/activitypub/)
 
 ### Required extensions
 
-#### Webfinger
+#### WebFinger
 
 In Mastodon, users are identified by a `username` and `domain` pair (e.g., `Gargron@mastodon.social`).
 This is used both for discovery and for unambiguously mentioning users across the fediverse. Furthermore, this is part of Mastodon's database design from its very beginnings.
 
 As a result, Mastodon requires that each ActivityPub actor uniquely maps back to an `acct:` URI that can be resolved via WebFinger.
 
-More information and examples are available at: https://docs.joinmastodon.org/spec/webfinger/
+- [WebFinger information and examples](https://docs.joinmastodon.org/spec/webfinger/)
 
 #### HTTP Signatures
 
@@ -21,10 +37,13 @@ In order to authenticate activities, Mastodon relies on HTTP Signatures, signing
 
 Mastodon requires all `POST` requests to be signed, and MAY require `GET` requests to be signed, depending on the configuration of the Mastodon server.
 
-More information on HTTP Signatures, as well as examples, can be found here: https://docs.joinmastodon.org/spec/security/#http
+- [HTTP Signatures information and examples](https://docs.joinmastodon.org/spec/security/#http)
 
 ### Optional extensions
 
-- Linked-Data Signatures: https://docs.joinmastodon.org/spec/security/#ld
+- [Linked-Data Signatures](https://docs.joinmastodon.org/spec/security/#ld)
-- Bearcaps: https://docs.joinmastodon.org/spec/bearcaps/
+- [Bearcaps](https://docs.joinmastodon.org/spec/bearcaps/)
-- Followers collection synchronization: https://git.activitypub.dev/ActivityPubDev/Fediverse-Enhancement-Proposals/src/branch/main/feps/fep-8fcf.md
+
+### Additional documentation
+
+- [Mastodon documentation](https://docs.joinmastodon.org/)

Gemfile

@@ -4,26 +4,29 @@ source 'https://rubygems.org'
 ruby '>= 3.0.0'
 
 gem 'puma', '~> 6.3'
-gem 'rails', '~> 6.1.7'
+gem 'rails', '~> 7.1.1'
-gem 'sprockets', '~> 3.7.2'
+gem 'propshaft'
 gem 'thor', '~> 1.2'
 gem 'rack', '~> 2.2.7'
 
+# For why irb is in the Gemfile, see: https://ruby.social/@st0012/111444685161478182
+gem 'irb', '~> 1.8'
+
 gem 'haml-rails', '~>2.0'
 gem 'pg', '~> 1.5'
-gem 'makara', '~> 0.5'
 gem 'pghero'
 gem 'dotenv-rails', '~> 2.8'
 
 gem 'aws-sdk-s3', '~> 1.123', require: false
 gem 'fog-core', '<= 2.4.0'
-gem 'fog-openstack', '~> 0.3', require: false
+gem 'fog-openstack', '~> 1.0', require: false
 gem 'kt-paperclip', '~> 7.2'
+gem 'md-paperclip-azure', '~> 2.2', require: false
 gem 'blurhash', '~> 0.1'
 
 gem 'active_model_serializers', '~> 0.10'
 gem 'addressable', '~> 2.8'
-gem 'bootsnap', '~> 1.16.0', require: false
+gem 'bootsnap', '~> 1.17.0', require: false
 gem 'browser'
 gem 'charlock_holmes', '~> 0.7.7'
 gem 'chewy', '~> 7.3'
@@ -35,13 +38,15 @@ group :pam_authentication, optional: true do
 end
 
 gem 'net-ldap', '~> 0.18'
-gem 'omniauth-cas', '~> 2.0'
+
-gem 'omniauth-saml', '~> 1.10'
+gem 'omniauth-cas', '~> 3.0.0.beta.1'
+gem 'omniauth-saml', '~> 2.0'
 gem 'omniauth_openid_connect', '~> 0.6.1'
-gem 'omniauth', '~> 1.9'
+gem 'omniauth', '~> 2.0'
-gem 'omniauth-rails_csrf_protection', '~> 0.1'
+gem 'omniauth-rails_csrf_protection', '~> 1.0'
 
 gem 'color_diff', '~> 0.1'
+gem 'csv', '~> 3.2'
 gem 'discard', '~> 1.2'
 gem 'doorkeeper', '~> 5.6'
 gem 'ed25519', '~> 1.3'
@@ -56,8 +61,9 @@ gem 'httplog', '~> 1.6.2'
 gem 'idn-ruby', require: 'idn'
 gem 'kaminari', '~> 1.2'
 gem 'link_header', '~> 0.0'
-gem 'mime-types', '~> 3.4.1', require: 'mime/types/columnar'
+gem 'mime-types', '~> 3.5.0', require: 'mime/types/columnar'
 gem 'nokogiri', '~> 1.15'
+gem 'nsa', github: 'jhawthorn/nsa', ref: 'e020fcc3a54d993ab45b7194d89ab720296c111b'
 gem 'oj', '~> 3.14'
 gem 'ox', '~> 2.14'
 gem 'parslet'
@@ -67,8 +73,7 @@ gem 'pundit', '~> 2.3'
 gem 'premailer-rails'
 gem 'rack-attack', '~> 6.6'
 gem 'rack-cors', '~> 2.0', require: 'rack/cors'
-gem 'rails-i18n', '~> 6.0'
+gem 'rails-i18n', '~> 7.0'
-gem 'rails-settings-cached', '~> 0.6', git: 'https://github.com/mastodon/rails-settings-cached.git', branch: 'v0.6.6-aliases-true'
 gem 'redcarpet', '~> 3.6'
 gem 'redis', '~> 4.5', require: ['redis', 'redis/connection/hiredis']
 gem 'mario-redis-lock', '~> 1.2', require: 'redis_lock'
@@ -82,9 +87,8 @@ gem 'sidekiq-unique-jobs', '~> 7.1'
 gem 'sidekiq-bulk', '~> 0.2.0'
 gem 'simple-navigation', '~> 4.4'
 gem 'simple_form', '~> 5.2'
-gem 'sprockets-rails', '~> 3.4', require: 'sprockets/railtie'
 gem 'stoplight', '~> 3.0.1'
-gem 'strong_migrations', '~> 0.8'
+gem 'strong_migrations', '1.7.0'
 gem 'tty-prompt', '~> 0.23', require: false
 gem 'twitter-text', '~> 3.1.0'
 gem 'tzinfo-data', '~> 1.2023'
@@ -99,23 +103,27 @@ gem 'rdf-normalize', '~> 0.5'
 gem 'private_address_check', '~> 0.5'
 
 group :test do
-  # RSpec runner for rails
+  # Adds RSpec Error/Warning annotations to GitHub PRs on the Files tab
-  gem 'rspec-rails', '~> 6.0'
+  gem 'rspec-github', '~> 2.4', require: false
-
-  # Used to split testing into chunks in CI
-  gem 'rspec_chunked', '~> 0.6'
 
   # RSpec progress bar formatter
   gem 'fuubar', '~> 2.5'
 
+  # RSpec helpers for email specs
+  gem 'email_spec'
+
   # Extra RSpec extenion methods and helpers for sidekiq
-  gem 'rspec-sidekiq', '~> 3.1'
+  gem 'rspec-sidekiq', '~> 4.0'
 
   # Browser integration testing
   gem 'capybara', '~> 3.39'
+  gem 'selenium-webdriver'
+
+  # Used to reset the database between system tests
+  gem 'database_cleaner-active_record'
 
   # Used to mock environment variables
-  gem 'climate_control', '~> 0.2'
+  gem 'climate_control'
 
   # Generating fake data for specs
   gem 'faker', '~> 3.2'
@@ -134,6 +142,7 @@ group :test do
 
   # Coverage formatter for RSpec test if DISABLE_SIMPLECOV is false
   gem 'simplecov', '~> 0.22', require: false
+  gem 'simplecov-lcov', '~> 0.8', require: false
 
   # Stub web requests for specs
   gem 'webmock', '~> 3.18'
@@ -159,24 +168,28 @@ group :development do
   gem 'letter_opener_web', '~> 2.0'
 
   # Security analysis CLI tools
-  gem 'brakeman', '~> 5.4', require: false
+  gem 'brakeman', '~> 6.0', require: false
   gem 'bundler-audit', '~> 0.9', require: false
 
   # Linter CLI for HAML files
   gem 'haml_lint', require: false
 
-  # Deployment automation
-  gem 'capistrano', '~> 3.17'
-  gem 'capistrano-rails', '~> 1.6'
-  gem 'capistrano-rbenv', '~> 2.2'
-  gem 'capistrano-yarn', '~> 2.0'
-
   # Validate missing i18n keys
   gem 'i18n-tasks', '~> 1.0', require: false
+end
+
+group :development, :test do
+  # Interactive Debugging tools
+  gem 'debug', '~> 1.8'
 
   # Profiling tools
   gem 'memory_profiler', require: false
+  gem 'ruby-prof', require: false
   gem 'stackprof', require: false
+  gem 'test-prof'
+
+  # RSpec runner for rails
+  gem 'rspec-rails', '~> 6.0'
 end
 
 group :production do
@@ -189,7 +202,7 @@ gem 'xorcist', '~> 1.1'
 
 gem 'cocoon', '~> 1.2'
 
-gem 'net-http', '~> 0.3.2'
+gem 'net-http', '~> 0.4.0'
 gem 'rubyzip', '~> 2.3'
 
 gem 'hcaptcha', '~> 7.1'

Procfile.dev

@@ -1,4 +1,4 @@
 web: env PORT=3000 RAILS_ENV=development bundle exec puma -C config/puma.rb
 sidekiq: env PORT=3000 RAILS_ENV=development bundle exec sidekiq
-stream: env PORT=4000 yarn run start
+stream: env PORT=4000 yarn workspace @mastodon/streaming start
-webpack: ./bin/webpack-dev-server --listen-host 0.0.0.0
+webpack: bin/webpack-dev-server

Rakefile

@@ -1,6 +1,8 @@
+# frozen_string_literal: true
+
 # Add your own tasks in files placed in lib/tasks ending in .rake,
 # for example lib/tasks/capistrano.rake, and they will automatically be available to Rake.
 
-require File.expand_path('../config/application', __FILE__)
+require File.expand_path('config/application', __dir__)
 
 Rails.application.load_tasks

SECURITY.md

@@ -1,8 +1,11 @@
 # Security Policy
 
-If you believe you've identified a security vulnerability in Mastodon (a bug that allows something to happen that shouldn't be possible), you can reach us at <security@joinmastodon.org>.
+If you believe you've identified a security vulnerability in Mastodon (a bug that allows something to happen that shouldn't be possible), you can either:
 
-You should _not_ report such issues on GitHub or in other public spaces to give us time to publish a fix for the issue without exposing Mastodon's users to increased risk.
+- open a [Github security issue on the Mastodon project](https://github.com/mastodon/mastodon/security/advisories/new)
+- reach us at <security@joinmastodon.org>
+
+You should _not_ report such issues on public GitHub issues or in other public spaces to give us time to publish a fix for the issue without exposing Mastodon's users to increased risk.
 
 ## Scope
 
@@ -12,7 +15,6 @@ A "vulnerability in Mastodon" is a vulnerability in the code distributed through
 
 | Version | Supported |
 | ------- | --------- |
+| 4.2.x   | Yes       |
 | 4.1.x   | Yes       |
-| 4.0.x   | Yes       |
+| < 4.1   | No        |
-| 3.5.x   | Yes       |
-| < 3.5   | No        |

Vagrantfile

@@ -10,7 +10,11 @@ curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add -
 sudo apt-add-repository 'deb https://dl.yarnpkg.com/debian/ stable main'
 
 # Add repo for NodeJS
-curl -sL https://deb.nodesource.com/setup_16.x | sudo bash -
+sudo mkdir -p /etc/apt/keyrings
+curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | sudo gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg
+NODE_MAJOR=20
+echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | sudo tee /etc/apt/sources.list.d/nodesource.list
+sudo apt-get update
 
 # Add firewall rule to redirect 80 to PORT and save
 sudo iptables -t nat -A PREROUTING -p tcp --dport 80 -j REDIRECT --to-port #{ENV["PORT"]}
@@ -60,6 +64,38 @@ sudo usermod -a -G rvm $USER
 
 SCRIPT
 
+$provisionElasticsearch = <<SCRIPT
+# Install Elastic Search
+sudo apt install openjdk-17-jre-headless -y
+sudo wget -O /usr/share/keyrings/elasticsearch.asc https://artifacts.elastic.co/GPG-KEY-elasticsearch
+sudo sh -c 'echo "deb [signed-by=/usr/share/keyrings/elasticsearch.asc] https://artifacts.elastic.co/packages/7.x/apt stable main" > /etc/apt/sources.list.d/elastic-7.x.list'
+sudo apt update
+sudo apt install elasticsearch -y
+
+sudo systemctl daemon-reload
+sudo systemctl enable --now elasticsearch
+
+echo 'path.data: /var/lib/elasticsearch
+path.logs: /var/log/elasticsearch
+network.host: 0.0.0.0
+http.port: 9200
+discovery.seed_hosts: ["localhost"]
+cluster.initial_master_nodes: ["node-1"]
+xpack.security.enabled: false' > /etc/elasticsearch/elasticsearch.yml
+
+sudo systemctl restart elasticsearch
+
+# Install Kibana
+sudo apt install kibana -y
+sudo systemctl enable --now kibana
+
+echo 'server.host: "0.0.0.0"
+elasticsearch.hosts: ["http://localhost:9200"]' > /etc/kibana/kibana.yml
+
+sudo systemctl restart kibana
+
+SCRIPT
+
 $provisionB = <<SCRIPT
 
 source "/etc/profile.d/rvm.sh"
@@ -80,11 +116,11 @@ bundle install
 
 # Install node modules
 sudo corepack enable
-yarn set version classic
+corepack prepare
 yarn install
 
 # Build Mastodon
-export RAILS_ENV=development 
+export RAILS_ENV=development
 export $(cat ".env.vagrant" | xargs)
 bundle exec rails db:setup
 
@@ -102,10 +138,8 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
 
   config.vm.provider :virtualbox do |vb|
     vb.name = "mastodon"
-    vb.customize ["modifyvm", :id, "--memory", "4096"]
+    vb.customize ["modifyvm", :id, "--memory", "8192"]
-    # Increase the number of CPUs. Uncomment and adjust to
+    vb.customize ["modifyvm", :id, "--cpus", "3"]
-    # increase performance
-    # vb.customize ["modifyvm", :id, "--cpus", "3"]
 
     # Disable VirtualBox DNS proxy to skip long-delay IPv6 resolutions.
     # https://github.com/mitchellh/vagrant/issues/1172
@@ -141,9 +175,15 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   config.vm.network :forwarded_port, guest: 3000, host: 3000
   config.vm.network :forwarded_port, guest: 4000, host: 4000
   config.vm.network :forwarded_port, guest: 8080, host: 8080
+  config.vm.network :forwarded_port, guest: 9200, host: 9200
+  config.vm.network :forwarded_port, guest: 9300, host: 9300
+  config.vm.network :forwarded_port, guest: 9243, host: 9243
+  config.vm.network :forwarded_port, guest: 5601, host: 5601
 
   # Full provisioning script, only runs on first 'vagrant up' or with 'vagrant provision'
   config.vm.provision :shell, inline: $provisionA, privileged: false, reset: true
+  # Run with elevated privileges for Elasticsearch installation
+  config.vm.provision :shell, inline: $provisionElasticsearch, privileged: true
   config.vm.provision :shell, inline: $provisionB, privileged: false
 
   config.vm.post_up_message = <<MESSAGE

app/chewy/accounts_index.rb

@@ -1,7 +1,9 @@
 # frozen_string_literal: true
 
 class AccountsIndex < Chewy::Index
-  settings index: { refresh_interval: '30s' }, analysis: {
+  include DatetimeClampingConcern
+
+  settings index: index_preset(refresh_interval: '30s'), analysis: {
     filter: {
       english_stop: {
         type: 'stop',
@@ -21,19 +23,20 @@ class AccountsIndex < Chewy::Index
 
     analyzer: {
       natural: {
-        tokenizer: 'uax_url_email',
+        tokenizer: 'standard',
         filter: %w(
-          english_possessive_stemmer
           lowercase
           asciifolding
           cjk_width
+          elision
+          english_possessive_stemmer
           english_stop
           english_stemmer
         ),
       },
 
       verbatim: {
-        tokenizer: 'whitespace',
+        tokenizer: 'standard',
         filter: %w(lowercase asciifolding cjk_width),
       },
 
@@ -59,9 +62,9 @@ class AccountsIndex < Chewy::Index
     field(:following_count, type: 'long')
     field(:followers_count, type: 'long')
     field(:properties, type: 'keyword', value: ->(account) { account.searchable_properties })
-    field(:last_status_at, type: 'date', value: ->(account) { account.last_status_at || account.created_at })
+    field(:last_status_at, type: 'date', value: ->(account) { clamp_date(account.last_status_at || account.created_at) })
     field(:display_name, type: 'text', analyzer: 'verbatim') { field :edge_ngram, type: 'text', analyzer: 'edge_ngram', search_analyzer: 'verbatim' }
     field(:username, type: 'text', analyzer: 'verbatim', value: ->(account) { [account.username, account.domain].compact.join('@') }) { field :edge_ngram, type: 'text', analyzer: 'edge_ngram', search_analyzer: 'verbatim' }
-    field(:text, type: 'text', value: ->(account) { account.searchable_text }) { field :stemmed, type: 'text', analyzer: 'natural' }
+    field(:text, type: 'text', analyzer: 'verbatim', value: ->(account) { account.searchable_text }) { field :stemmed, type: 'text', analyzer: 'natural' }
   end
 end

app/chewy/concerns/datetime_clamping_concern.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module DatetimeClampingConcern
+  extend ActiveSupport::Concern
+
+  MIN_ISO8601_DATETIME = '0000-01-01T00:00:00Z'.to_datetime.freeze
+  MAX_ISO8601_DATETIME = '9999-12-31T23:59:59Z'.to_datetime.freeze
+
+  class_methods do
+    def clamp_date(datetime)
+      datetime.clamp(MIN_ISO8601_DATETIME, MAX_ISO8601_DATETIME)
+    end
+  end
+end

app/chewy/instances_index.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+class InstancesIndex < Chewy::Index
+  settings index: index_preset(refresh_interval: '30s')
+
+  index_scope ::Instance.searchable
+
+  root date_detection: false do
+    field :domain, type: 'text', index_prefixes: { min_chars: 1, max_chars: 5 }
+    field :accounts_count, type: 'long'
+  end
+end

app/chewy/public_statuses_index.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+class PublicStatusesIndex < Chewy::Index
+  include DatetimeClampingConcern
+
+  settings index: index_preset(refresh_interval: '30s', number_of_shards: 5), analysis: {
+    filter: {
+      english_stop: {
+        type: 'stop',
+        stopwords: '_english_',
+      },
+
+      english_stemmer: {
+        type: 'stemmer',
+        language: 'english',
+      },
+
+      english_possessive_stemmer: {
+        type: 'stemmer',
+        language: 'possessive_english',
+      },
+    },
+
+    analyzer: {
+      verbatim: {
+        tokenizer: 'uax_url_email',
+        filter: %w(lowercase),
+      },
+
+      content: {
+        tokenizer: 'standard',
+        filter: %w(
+          lowercase
+          asciifolding
+          cjk_width
+          elision
+          english_possessive_stemmer
+          english_stop
+          english_stemmer
+        ),
+      },
+
+      hashtag: {
+        tokenizer: 'keyword',
+        filter: %w(
+          word_delimiter_graph
+          lowercase
+          asciifolding
+          cjk_width
+        ),
+      },
+    },
+  }
+
+  index_scope ::Status.unscoped
+                      .kept
+                      .indexable
+                      .includes(:media_attachments, :preloadable_poll, :tags, preview_cards_status: :preview_card)
+
+  root date_detection: false do
+    field(:id, type: 'long')
+    field(:account_id, type: 'long')
+    field(:text, type: 'text', analyzer: 'verbatim', value: ->(status) { status.searchable_text }) { field(:stemmed, type: 'text', analyzer: 'content') }
+    field(:tags, type: 'text', analyzer: 'hashtag', value: ->(status) { status.tags.map(&:display_name) })
+    field(:language, type: 'keyword')
+    field(:properties, type: 'keyword', value: ->(status) { status.searchable_properties })
+    field(:created_at, type: 'date', value: ->(status) { clamp_date(status.created_at) })
+  end
+end

app/chewy/statuses_index.rb

@@ -1,75 +1,67 @@
 # frozen_string_literal: true
 
 class StatusesIndex < Chewy::Index
-  include FormattingHelper
+  include DatetimeClampingConcern
 
-  settings index: { refresh_interval: '30s' }, analysis: {
+  settings index: index_preset(refresh_interval: '30s', number_of_shards: 5), analysis: {
     filter: {
       english_stop: {
         type: 'stop',
         stopwords: '_english_',
       },
+
       english_stemmer: {
         type: 'stemmer',
         language: 'english',
       },
+
       english_possessive_stemmer: {
         type: 'stemmer',
         language: 'possessive_english',
       },
     },
+
     analyzer: {
-      content: {
+      verbatim: {
         tokenizer: 'uax_url_email',
+        filter: %w(lowercase),
+      },
+
+      content: {
+        tokenizer: 'standard',
         filter: %w(
-          english_possessive_stemmer
           lowercase
           asciifolding
           cjk_width
+          elision
+          english_possessive_stemmer
           english_stop
           english_stemmer
         ),
       },
+
+      hashtag: {
+        tokenizer: 'keyword',
+        filter: %w(
+          word_delimiter_graph
+          lowercase
+          asciifolding
+          cjk_width
+        ),
+      },
     },
   }
 
-  # We do not use delete_if option here because it would call a method that we
+  index_scope ::Status.unscoped.kept.without_reblogs.includes(:media_attachments, :local_mentioned, :local_favorited, :local_reblogged, :local_bookmarked, :tags, preview_cards_status: :preview_card, preloadable_poll: :local_voters), delete_if: ->(status) { status.searchable_by.empty? }
-  # expect to be called with crutches without crutches, causing n+1 queries
-  index_scope ::Status.unscoped.kept.without_reblogs.includes(:media_attachments, :preloadable_poll)
-
-  crutch :mentions do |collection|
-    data = ::Mention.where(status_id: collection.map(&:id)).where(account: Account.local, silent: false).pluck(:status_id, :account_id)
-    data.each.with_object({}) { |(id, name), result| (result[id] ||= []).push(name) }
-  end
-
-  crutch :favourites do |collection|
-    data = ::Favourite.where(status_id: collection.map(&:id)).where(account: Account.local).pluck(:status_id, :account_id)
-    data.each.with_object({}) { |(id, name), result| (result[id] ||= []).push(name) }
-  end
-
-  crutch :reblogs do |collection|
-    data = ::Status.where(reblog_of_id: collection.map(&:id)).where(account: Account.local).pluck(:reblog_of_id, :account_id)
-    data.each.with_object({}) { |(id, name), result| (result[id] ||= []).push(name) }
-  end
-
-  crutch :bookmarks do |collection|
-    data = ::Bookmark.where(status_id: collection.map(&:id)).where(account: Account.local).pluck(:status_id, :account_id)
-    data.each.with_object({}) { |(id, name), result| (result[id] ||= []).push(name) }
-  end
-
-  crutch :votes do |collection|
-    data = ::PollVote.joins(:poll).where(poll: { status_id: collection.map(&:id) }).where(account: Account.local).pluck(:status_id, :account_id)
-    data.each.with_object({}) { |(id, name), result| (result[id] ||= []).push(name) }
-  end
 
   root date_detection: false do
-    field :id, type: 'long'
+    field(:id, type: 'long')
-    field :account_id, type: 'long'
+    field(:account_id, type: 'long')
-
+    field(:text, type: 'text', analyzer: 'verbatim', value: ->(status) { status.searchable_text }) { field(:stemmed, type: 'text', analyzer: 'content') }
-    field :text, type: 'text', value: ->(status) { status.searchable_text } do
+    field(:tags, type: 'text', analyzer: 'hashtag',  value: ->(status) { status.tags.map(&:display_name) })
-      field :stemmed, type: 'text', analyzer: 'content'
+    field(:searchable_by, type: 'long', value: ->(status) { status.searchable_by })
-    end
+    field(:language, type: 'keyword')
-
+    field(:properties, type: 'keyword', value: ->(status) { status.searchable_properties })
-    field :searchable_by, type: 'long', value: ->(status, crutches) { status.searchable_by(crutches) }
+    field(:created_at, type: 'date', value: ->(status) { clamp_date(status.created_at) })
   end
 end

app/chewy/tags_index.rb

@@ -1,16 +1,27 @@
 # frozen_string_literal: true
 
 class TagsIndex < Chewy::Index
-  settings index: { refresh_interval: '30s' }, analysis: {
+  include DatetimeClampingConcern
+
+  settings index: index_preset(refresh_interval: '30s'), analysis: {
     analyzer: {
       content: {
         tokenizer: 'keyword',
-        filter: %w(lowercase asciifolding cjk_width),
+        filter: %w(
+          word_delimiter_graph
+          lowercase
+          asciifolding
+          cjk_width
+        ),
       },
 
       edge_ngram: {
         tokenizer: 'edge_ngram',
-        filter: %w(lowercase asciifolding cjk_width),
+        filter: %w(
+          lowercase
+          asciifolding
+          cjk_width
+        ),
       },
     },
 
@@ -30,12 +41,9 @@ class TagsIndex < Chewy::Index
   end
 
   root date_detection: false do
-    field :name, type: 'text', analyzer: 'content' do
+    field(:name, type: 'text', analyzer: 'content', value: :display_name) { field(:edge_ngram, type: 'text', analyzer: 'edge_ngram', search_analyzer: 'content') }
-      field :edge_ngram, type: 'text', analyzer: 'edge_ngram', search_analyzer: 'content'
+    field(:reviewed, type: 'boolean', value: ->(tag) { tag.reviewed? })
-    end
+    field(:usage, type: 'long', value: ->(tag, crutches) { tag.history.aggregate(crutches.time_period).accounts })
-
+    field(:last_status_at, type: 'date', value: ->(tag) { clamp_date(tag.last_status_at || tag.created_at) })
-    field :reviewed, type: 'boolean', value: ->(tag) { tag.reviewed? }
-    field :usage, type: 'long', value: ->(tag, crutches) { tag.history.aggregate(crutches.time_period).accounts }
-    field :last_status_at, type: 'date', value: ->(tag) { tag.last_status_at || tag.created_at }
   end
 end

app/controllers/about_controller.rb

@@ -5,15 +5,7 @@ class AboutController < ApplicationController
 
   skip_before_action :require_functional!
 
-  before_action :set_instance_presenter
-
   def show
     expires_in(15.seconds, public: true, stale_while_revalidate: 30.seconds, stale_if_error: 1.day) unless user_signed_in?
   end
-
-  private
-
-  def set_instance_presenter
-    @instance_presenter = InstancePresenter.new
-  end
 end

app/controllers/accounts_controller.rb

@@ -12,14 +12,12 @@ class AccountsController < ApplicationController
   before_action :require_account_signature!, if: -> { request.format == :json && authorized_fetch_mode? }
 
   skip_around_action :set_locale, if: -> { [:json, :rss].include?(request.format&.to_sym) }
-  skip_before_action :require_functional!, unless: :whitelist_mode?
+  skip_before_action :require_functional!, unless: :limited_federation_mode?
 
   def show
     respond_to do |format|
       format.html do
         expires_in(15.seconds, public: true, stale_while_revalidate: 30.seconds, stale_if_error: 1.hour) unless user_signed_in?
-
-        @rss_url = rss_url
       end
 
       format.rss do
@@ -52,7 +50,7 @@ class AccountsController < ApplicationController
   end
 
   def only_media_scope
-    Status.joins(:media_attachments).merge(@account.media_attachments.reorder(nil)).group(:id)
+    Status.joins(:media_attachments).merge(@account.media_attachments).group(:id)
   end
 
   def no_replies_scope
@@ -84,29 +82,21 @@ class AccountsController < ApplicationController
       short_account_url(@account, format: 'rss')
     end
   end
+  helper_method :rss_url
 
   def media_requested?
-    request.path.split('.').first.end_with?('/media') && !tag_requested?
+    path_without_format.end_with?('/media') && !tag_requested?
   end
 
   def replies_requested?
-    request.path.split('.').first.end_with?('/with_replies') && !tag_requested?
+    path_without_format.end_with?('/with_replies') && !tag_requested?
   end
 
   def tag_requested?
-    request.path.split('.').first.end_with?(Addressable::URI.parse("/tagged/#{params[:tag]}").normalize)
+    path_without_format.end_with?(Addressable::URI.parse("/tagged/#{params[:tag]}").normalize)
   end
 
-  def cached_filtered_status_page
+  def path_without_format
-    cache_collection_paginated_by_id(
+    request.path.split('.').first
-      filtered_statuses,
-      Status,
-      PAGE_SIZE,
-      params_slice(:max_id, :min_id, :since_id)
-    )
-  end
-
-  def params_slice(*keys)
-    params.slice(*keys).permit(*keys)
   end
 end

app/controllers/activitypub/followers_synchronizations_controller.rb

@@ -24,7 +24,7 @@ class ActivityPub::FollowersSynchronizationsController < ActivityPub::BaseContro
   end
 
   def set_items
-    @items = @account.followers.where(Account.arel_table[:uri].matches("#{Account.sanitize_sql_like(uri_prefix)}/%", false, true)).or(@account.followers.where(uri: uri_prefix)).pluck(:uri)
+    @items = @account.followers.matches_uri_prefix(uri_prefix).pluck(:uri)
   end
 
   def collection_presenter

app/controllers/activitypub/inboxes_controller.rb

@@ -24,7 +24,7 @@ class ActivityPub::InboxesController < ActivityPub::BaseController
 
   def unknown_affected_account?
     json = Oj.load(body, mode: :strict)
-    json.is_a?(Hash) && %w(Delete Update).include?(json['type']) && json['actor'].present? && json['actor'] == value_or_id(json['object']) && !Account.where(uri: json['actor']).exists?
+    json.is_a?(Hash) && %w(Delete Update).include?(json['type']) && json['actor'].present? && json['actor'] == value_or_id(json['object']) && !Account.exists?(uri: json['actor'])
   rescue Oj::ParseError
     false
   end

app/controllers/admin/account_actions_controller.rb

@@ -21,7 +21,7 @@ module Admin
       account_action.save!
 
       if account_action.with_report?
-        redirect_to admin_reports_path, notice: I18n.t('admin.reports.processed_msg', id: params[:report_id])
+        redirect_to admin_reports_path, notice: I18n.t('admin.reports.processed_msg', id: resource_params[:report_id])
       else
         redirect_to admin_account_path(@account.id)
       end

app/controllers/admin/account_moderation_notes_controller.rb

@@ -16,7 +16,7 @@ module Admin
         @moderation_notes = @account.targeted_moderation_notes.latest
         @warnings         = @account.strikes.custom.latest
 
-        render template: 'admin/accounts/show'
+        render 'admin/accounts/show'
       end
     end
 

app/controllers/admin/action_logs_controller.rb

@@ -6,7 +6,7 @@ module Admin
 
     def index
       authorize :audit_log, :index?
-      @auditable_accounts = Account.where(id: Admin::ActionLog.reorder(nil).select('distinct account_id')).select(:id, :username)
+      @auditable_accounts = Account.auditable.select(:id, :username)
     end
 
     private

app/controllers/admin/confirmations_controller.rb

@@ -3,11 +3,11 @@
 module Admin
   class ConfirmationsController < BaseController
     before_action :set_user
-    before_action :check_confirmation, only: [:resend]
+    before_action :redirect_confirmed_user, only: [:resend], if: :user_confirmed?
 
     def create
       authorize @user, :confirm?
-      @user.confirm!
+      @user.mark_email_as_confirmed!
       log_action :confirm, @user
       redirect_to admin_accounts_path
     end
@@ -25,11 +25,13 @@ module Admin
 
     private
 
-    def check_confirmation
+    def redirect_confirmed_user
-      if @user.confirmed?
+      flash[:error] = I18n.t('admin.accounts.resend_confirmation.already_confirmed')
-        flash[:error] = I18n.t('admin.accounts.resend_confirmation.already_confirmed')
+      redirect_to admin_accounts_path
-        redirect_to admin_accounts_path
+    end
-      end
+
+    def user_confirmed?
+      @user.confirmed?
     end
   end
 end

app/controllers/admin/disputes/appeals_controller.rb

@@ -20,7 +20,7 @@ class Admin::Disputes::AppealsController < Admin::BaseController
     authorize @appeal, :approve?
     log_action :reject, @appeal
     @appeal.reject!(current_account)
-    UserMailer.appeal_rejected(@appeal.account.user, @appeal)
+    UserMailer.appeal_rejected(@appeal.account.user, @appeal).deliver_later
     redirect_to disputes_strike_path(@appeal.strike)
   end
 

app/controllers/admin/domain_blocks_controller.rb

@@ -33,14 +33,14 @@ module Admin
 
       # Disallow accidentally downgrading a domain block
       if existing_domain_block.present? && !@domain_block.stricter_than?(existing_domain_block)
-        @domain_block.save
+        @domain_block.validate
         flash.now[:alert] = I18n.t('admin.domain_blocks.existing_domain_block_html', name: existing_domain_block.domain, unblock_url: admin_domain_block_path(existing_domain_block)).html_safe
         @domain_block.errors.delete(:domain)
         return render :new
       end
 
       # Allow transparently upgrading a domain block
-      if existing_domain_block.present?
+      if existing_domain_block.present? && existing_domain_block.domain == TagManager.instance.normalize_domain(@domain_block.domain.strip)
         @domain_block = existing_domain_block
         @domain_block.assign_attributes(resource_params)
       end

app/controllers/admin/email_domain_blocks_controller.rb

@@ -38,9 +38,9 @@ module Admin
           log_action :create, @email_domain_block
 
           (@email_domain_block.other_domains || []).uniq.each do |domain|
-            next if EmailDomainBlock.where(domain: domain).exists?
+            next if EmailDomainBlock.exists?(domain: domain)
 
-            other_email_domain_block = EmailDomainBlock.create!(domain: domain, parent: @email_domain_block)
+            other_email_domain_block = EmailDomainBlock.create!(domain: domain, allow_with_approval: @email_domain_block.allow_with_approval, parent: @email_domain_block)
             log_action :create, other_email_domain_block
           end
         end
@@ -65,7 +65,7 @@ module Admin
     end
 
     def resource_params
-      params.require(:email_domain_block).permit(:domain, other_domains: [])
+      params.require(:email_domain_block).permit(:domain, :allow_with_approval, other_domains: [])
     end
 
     def form_email_domain_block_batch_params

app/controllers/admin/export_domain_allows_controller.rb

@@ -4,7 +4,7 @@ require 'csv'
 
 module Admin
   class ExportDomainAllowsController < BaseController
-    include AdminExportControllerConcern
+    include Admin::ExportControllerConcern
 
     before_action :set_dummy_import!, only: [:new]
 

app/controllers/admin/export_domain_blocks_controller.rb

@@ -4,7 +4,7 @@ require 'csv'
 
 module Admin
   class ExportDomainBlocksController < BaseController
-    include AdminExportControllerConcern
+    include Admin::ExportControllerConcern
 
     before_action :set_dummy_import!, only: [:new]
 
@@ -49,7 +49,7 @@ module Admin
         next
       end
 
-      @warning_domains = Instance.where(domain: @domain_blocks.map(&:domain)).where('EXISTS (SELECT 1 FROM follows JOIN accounts ON follows.account_id = accounts.id OR follows.target_account_id = accounts.id WHERE accounts.domain = instances.domain)').pluck(:domain)
+      @warning_domains = instances_from_imported_blocks.pluck(:domain)
     rescue ActionController::ParameterMissing
       flash.now[:alert] = I18n.t('admin.export_domain_blocks.no_file')
       set_dummy_import!
@@ -58,6 +58,10 @@ module Admin
 
     private
 
+    def instances_from_imported_blocks
+      Instance.with_domain_follows(@domain_blocks.map(&:domain))
+    end
+
     def export_filename
       'domain_blocks.csv'
     end
@@ -68,7 +72,7 @@ module Admin
 
     def export_data
       CSV.generate(headers: export_headers, write_headers: true) do |content|
-        DomainBlock.with_limitations.each do |instance|
+        DomainBlock.with_limitations.order(id: :asc).each do |instance|
           content << [instance.domain, instance.severity, instance.reject_media, instance.reject_reports, instance.public_comment, instance.obfuscate]
         end
       end

app/controllers/admin/follow_recommendations_controller.rb

@@ -8,7 +8,7 @@ module Admin
       authorize :follow_recommendation, :show?
 
       @form     = Form::AccountBatch.new
-      @accounts = filtered_follow_recommendations
+      @accounts = filtered_follow_recommendations.page(params[:page])
     end
 
     def update

app/controllers/admin/instances_controller.rb

@@ -49,7 +49,7 @@ module Admin
     private
 
     def set_instance
-      @instance = Instance.find(TagManager.instance.normalize_domain(params[:id]&.strip))
+      @instance = Instance.find_or_initialize_by(domain: TagManager.instance.normalize_domain(params[:id]&.strip))
     end
 
     def set_instances
@@ -65,7 +65,7 @@ module Admin
     end
 
     def filtered_instances
-      InstanceFilter.new(whitelist_mode? ? { allowed: true } : filter_params).results
+      InstanceFilter.new(limited_federation_mode? ? { allowed: true } : filter_params).results
     end
 
     def filter_params

app/controllers/admin/relays_controller.rb

@@ -24,7 +24,7 @@ module Admin
         @relay.enable!
         redirect_to admin_relays_path
       else
-        render action: :new
+        render :new
       end
     end
 

app/controllers/admin/report_notes_controller.rb

@@ -26,7 +26,7 @@ module Admin
         @form         = Admin::StatusBatchAction.new
         @statuses     = @report.statuses.with_includes
 
-        render template: 'admin/reports/show'
+        render 'admin/reports/show'
       end
     end
 

app/controllers/admin/software_updates_controller.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Admin
+  class SoftwareUpdatesController < BaseController
+    before_action :check_enabled!
+
+    def index
+      authorize :software_update, :index?
+      @software_updates = SoftwareUpdate.all.sort_by(&:gem_version)
+    end
+
+    private
+
+    def check_enabled!
+      not_found unless SoftwareUpdate.check_enabled?
+    end
+  end
+end

app/controllers/admin/statuses_controller.rb

@@ -31,6 +31,11 @@ module Admin
 
     private
 
+    def batched_ordered_status_edits
+      @status.edits.includes(:account, status: [:account]).find_each(order: :asc)
+    end
+    helper_method :batched_ordered_status_edits
+
     def admin_status_batch_action_params
       params.require(:admin_status_batch_action).permit(status_ids: [])
     end

app/controllers/api/base_controller.rb

@@ -4,11 +4,12 @@ class Api::BaseController < ApplicationController
   DEFAULT_STATUSES_LIMIT = 20
   DEFAULT_ACCOUNTS_LIMIT = 40
 
-  include RateLimitHeaders
+  include Api::RateLimitHeaders
-  include AccessTokenTrackingConcern
+  include Api::AccessTokenTrackingConcern
-  include ApiCachingConcern
+  include Api::CachingConcern
+  include Api::ContentSecurityPolicy
 
-  skip_before_action :require_functional!, unless: :whitelist_mode?
+  skip_before_action :require_functional!, unless: :limited_federation_mode?
 
   before_action :require_authenticated_user!, if: :disallow_unauthenticated_api_access?
   before_action :require_not_suspended!
@@ -17,26 +18,6 @@ class Api::BaseController < ApplicationController
 
   protect_from_forgery with: :null_session
 
-  content_security_policy do |p|
-    # Set every directive that does not have a fallback
-    p.default_src :none
-    p.frame_ancestors :none
-    p.form_action :none
-
-    # Disable every directive with a fallback to cut on response size
-    p.base_uri false
-    p.font_src false
-    p.img_src false
-    p.style_src false
-    p.media_src false
-    p.frame_src false
-    p.manifest_src false
-    p.connect_src false
-    p.script_src false
-    p.child_src false
-    p.worker_src false
-  end
-
   rescue_from ActiveRecord::RecordInvalid, Mastodon::ValidationError do |e|
     render json: { error: e.to_s }, status: 422
   end
@@ -83,7 +64,7 @@ class Api::BaseController < ApplicationController
   end
 
   def doorkeeper_unauthorized_render_options(error: nil)
-    { json: { error: (error.try(:description) || 'Not authorized') } }
+    { json: { error: error.try(:description) || 'Not authorized' } }
   end
 
   def doorkeeper_forbidden_render_options(*)
@@ -124,7 +105,11 @@ class Api::BaseController < ApplicationController
   end
 
   def require_not_suspended!
-    render json: { error: 'Your login is currently disabled' }, status: 403 if current_user&.account&.suspended?
+    render json: { error: 'Your login is currently disabled' }, status: 403 if current_user&.account&.unavailable?
+  end
+
+  def require_valid_pagination_options!
+    render json: { error: 'Pagination values for `offset` and `limit` must be positive' }, status: 400 if pagination_options_invalid?
   end
 
   def require_user!
@@ -150,11 +135,15 @@ class Api::BaseController < ApplicationController
   end
 
   def disallow_unauthenticated_api_access?
-    ENV['DISALLOW_UNAUTHENTICATED_API_ACCESS'] == 'true' || Rails.configuration.x.whitelist_mode
+    ENV['DISALLOW_UNAUTHENTICATED_API_ACCESS'] == 'true' || Rails.configuration.x.limited_federation_mode
   end
 
   private
 
+  def pagination_options_invalid?
+    params.slice(:limit, :offset).values.map(&:to_i).any?(&:negative?)
+  end
+
   def respond_with_error(code)
     render json: { error: Rack::Utils::HTTP_STATUS_CODES[code] }, status: code
   end

app/controllers/api/v1/accounts/credentials_controller.rb

@@ -16,6 +16,8 @@ class Api::V1::Accounts::CredentialsController < Api::BaseController
     current_user.update(user_params) if user_params
     ActivityPub::UpdateDistributionWorker.perform_async(@account.id)
     render json: @account, serializer: REST::CredentialAccountSerializer
+  rescue ActiveRecord::RecordInvalid => e
+    render json: ValidationErrorFormatter.new(e).as_json, status: 422
   end
 
   private
@@ -30,6 +32,7 @@ class Api::V1::Accounts::CredentialsController < Api::BaseController
       :bot,
       :discoverable,
       :hide_collections,
+      :indexable,
       fields_attributes: [:name, :value]
     )
   end

app/controllers/api/v1/accounts/familiar_followers_controller.rb

@@ -12,7 +12,7 @@ class Api::V1::Accounts::FamiliarFollowersController < Api::BaseController
   private
 
   def set_accounts
-    @accounts = Account.without_suspended.where(id: account_ids).select('id, hide_collections').index_by(&:id).values_at(*account_ids).compact
+    @accounts = Account.without_suspended.where(id: account_ids).select('id, hide_collections')
   end
 
   def familiar_followers

app/controllers/api/v1/accounts/follower_accounts_controller.rb

@@ -21,16 +21,16 @@ class Api::V1::Accounts::FollowerAccountsController < Api::BaseController
     return [] if hide_results?
 
     scope = default_accounts
-    scope = scope.where.not(id: current_account.excluded_from_timeline_account_ids) unless current_account.nil? || current_account.id == @account.id
+    scope = scope.not_excluded_by_account(current_account) unless current_account.nil? || current_account.id == @account.id
     scope.merge(paginated_follows).to_a
   end
 
   def hide_results?
-    @account.suspended? || (@account.hides_followers? && current_account&.id != @account.id) || (current_account && @account.blocking?(current_account))
+    @account.unavailable? || (@account.hides_followers? && current_account&.id != @account.id) || (current_account && @account.blocking?(current_account))
   end
 
   def default_accounts
-    Account.includes(:active_relationships, :account_stat).references(:active_relationships)
+    Account.includes(:active_relationships, :account_stat, :user).references(:active_relationships)
   end
 
   def paginated_follows

app/controllers/api/v1/accounts/following_accounts_controller.rb

@@ -21,16 +21,16 @@ class Api::V1::Accounts::FollowingAccountsController < Api::BaseController
     return [] if hide_results?
 
     scope = default_accounts
-    scope = scope.where.not(id: current_account.excluded_from_timeline_account_ids) unless current_account.nil? || current_account.id == @account.id
+    scope = scope.not_excluded_by_account(current_account) unless current_account.nil? || current_account.id == @account.id
     scope.merge(paginated_follows).to_a
   end
 
   def hide_results?
-    @account.suspended? || (@account.hides_following? && current_account&.id != @account.id) || (current_account && @account.blocking?(current_account))
+    @account.unavailable? || (@account.hides_following? && current_account&.id != @account.id) || (current_account && @account.blocking?(current_account))
   end
 
   def default_accounts
-    Account.includes(:passive_relationships, :account_stat).references(:passive_relationships)
+    Account.includes(:passive_relationships, :account_stat, :user).references(:passive_relationships)
   end
 
   def paginated_follows

app/controllers/api/v1/accounts/notes_controller.rb

@@ -25,6 +25,6 @@ class Api::V1::Accounts::NotesController < Api::BaseController
   end
 
   def relationships_presenter
-    AccountRelationshipsPresenter.new([@account.id], current_user.account_id)
+    AccountRelationshipsPresenter.new([@account], current_user.account_id)
   end
 end

app/controllers/api/v1/accounts/pins_controller.rb

@@ -25,6 +25,6 @@ class Api::V1::Accounts::PinsController < Api::BaseController
   end
 
   def relationships_presenter
-    AccountRelationshipsPresenter.new([@account.id], current_user.account_id)
+    AccountRelationshipsPresenter.new([@account], current_user.account_id)
   end
 end