Merge branch 'mastodon:main' into BirdUI

.devcontainer/docker-compose.yml

@@ -2,6 +2,7 @@ version: '3'
 
 services:
   app:
+    working_dir: /workspaces/mastodon/
     build:
       context: .
       dockerfile: Dockerfile

.env.test

@@ -4,7 +4,8 @@ NODE_ENV=production
 LOCAL_DOMAIN=cb6e6126.ngrok.io
 LOCAL_HTTPS=true
 
-# Required by ActiveRecord encryption feature
-ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY=fkSxKD2bF396kdQbrP1EJ7WbU7ZgNokR
-ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT=r0hvVmzBVsjxC7AMlwhOzmtc36ZCOS1E
-ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY=PhdFyyfy5xJ7WVd2lWBpcPScRQHzRTNr
+# Secret values required by ActiveRecord encryption feature
+# Use `bin/rails db:encryption:init` to generate fresh secrets
+ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY=test_determinist_key_DO_NOT_USE_IN_PRODUCTION
+ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT=test_salt_DO_NOT_USE_IN_PRODUCTION
+ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY=test_primary_key_DO_NOT_USE_IN_PRODUCTION

.github/renovate.json5

@@ -141,6 +141,13 @@
       matchUpdateTypes: ['patch', 'minor'],
       groupName: 'RSpec (non-major)',
     },
+    {
+      // Group all opentelemetry-ruby packages in the same PR
+      matchManagers: ['bundler'],
+      matchPackagePrefixes: ['opentelemetry-'],
+      matchUpdateTypes: ['patch', 'minor'],
+      groupName: 'opentelemetry-ruby (non-major)',
+    },
     // Add labels depending on package manager
     { matchManagers: ['npm', 'nvm'], addLabels: ['javascript'] },
     { matchManagers: ['bundler', 'ruby-version'], addLabels: ['ruby'] },

.github/workflows/lint-css.yml

@@ -38,9 +38,5 @@ jobs:
       - name: Set up Javascript environment
         uses: ./.github/actions/setup-javascript
 
-      - uses: xt0rted/stylelint-problem-matcher@v1
-
-      - run: echo "::add-matcher::.github/stylelint-matcher.json"
-
       - name: Stylelint
-        run: yarn lint:css
+        run: yarn lint:css -f github

.github/workflows/test-ruby.yml

@@ -145,6 +145,8 @@ jobs:
         uses: codecov/codecov-action@v4
         with:
           files: coverage/lcov/mastodon.lcov
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
 
   test-e2e:
     name: End to End testing
@@ -184,6 +186,8 @@ jobs:
       DISABLE_SIMPLECOV: true
       RAILS_ENV: test
       BUNDLE_WITH: test
+      LOCAL_DOMAIN: localhost:3000
+      LOCAL_HTTPS: false
 
     strategy:
       fail-fast: false
@@ -213,7 +217,7 @@ jobs:
       - name: Load database schema
         run: './bin/rails db:create db:schema:load db:seed'
 
-      - run: bundle exec rake spec:system
+      - run: bin/rspec spec/system --tag streaming --tag js
 
       - name: Archive logs
         uses: actions/upload-artifact@v4
@@ -260,8 +264,8 @@ jobs:
         ports:
           - 6379:6379
 
-      search:
-        image: ${{ matrix.search-image }}
+      elasticsearch:
+        image: ${{ contains(matrix.search-image, 'elasticsearch') && matrix.search-image || '' }}
         env:
           discovery.type: single-node
           xpack.security.enabled: false
@@ -273,6 +277,20 @@ jobs:
         ports:
           - 9200:9200
 
+      opensearch:
+        image: ${{ contains(matrix.search-image, 'opensearch') && matrix.search-image || '' }}
+        env:
+          discovery.type: single-node
+          DISABLE_INSTALL_DEMO_CONFIG: true
+          DISABLE_SECURITY_PLUGIN: true
+        options: >-
+          --health-cmd "curl http://localhost:9200/_cluster/health"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 10
+        ports:
+          - 9200:9200
+
     env:
       DB_HOST: localhost
       DB_USER: postgres
@@ -296,6 +314,8 @@ jobs:
         include:
           - ruby-version: '.ruby-version'
             search-image: docker.elastic.co/elasticsearch/elasticsearch:8.10.2
+          - ruby-version: '.ruby-version'
+            search-image: opensearchproject/opensearch:2
 
     steps:
       - uses: actions/checkout@v4

.nvmrc

@@ -1 +1 @@
-20.12
+20.14

.rubocop.yml

@@ -211,6 +211,11 @@ Style/PercentLiteralDelimiters:
 Style/RedundantBegin:
   Enabled: false
 
+# Reason: Prevailing style choice
+# https://docs.rubocop.org/rubocop/cops_style.html#styleredundantfetchblock
+Style/RedundantFetchBlock:
+  Enabled: false
+
 # Reason: Overridden to reduce implicit StandardError rescues
 # https://docs.rubocop.org/rubocop/cops_style.html#stylerescuestandarderror
 Style/RescueStandardError:

.rubocop_todo.yml

@@ -1,6 +1,6 @@
 # This configuration was generated by
 # `rubocop --auto-gen-config --auto-gen-only-exclude --no-exclude-limit --no-offense-counts --no-auto-gen-timestamp`
-# using RuboCop version 1.62.1.
+# using RuboCop version 1.63.5.
 # The point is for the user to remove these configuration records
 # one by one as the offenses are removed from the code base.
 # Note that changes in the inspected code, or installation of new
@@ -42,14 +42,6 @@ RSpec/MultipleMemoizedHelpers:
 RSpec/NestedGroups:
   Max: 6
 
-# Configuration parameters: Include.
-# Include: app/models/**/*.rb
-Rails/HasAndBelongsToMany:
-  Exclude:
-    - 'app/models/concerns/account/associations.rb'
-    - 'app/models/status.rb'
-    - 'app/models/tag.rb'
-
 Rails/OutputSafety:
   Exclude:
     - 'config/initializers/simple_form.rb'
@@ -62,10 +54,6 @@ Style/ClassEqualityComparison:
     - 'app/helpers/jsonld_helper.rb'
     - 'app/serializers/activitypub/outbox_serializer.rb'
 
-Style/ClassVars:
-  Exclude:
-    - 'config/initializers/devise.rb'
-
 # This cop supports safe autocorrection (--autocorrect).
 # Configuration parameters: AllowedVars.
 Style/FetchEnvVar:
@@ -82,7 +70,7 @@ Style/FetchEnvVar:
     - 'config/initializers/vapid.rb'
     - 'lib/mastodon/redis_config.rb'
     - 'lib/tasks/repo.rake'
-    - 'spec/features/profile_spec.rb'
+    - 'spec/system/profile_spec.rb'
 
 # This cop supports safe autocorrection (--autocorrect).
 # Configuration parameters: EnforcedStyle, MaxUnannotatedPlaceholdersAllowed, AllowedMethods, AllowedPatterns.
@@ -134,13 +122,6 @@ Style/HashTransformValues:
     - 'app/serializers/rest/web_push_subscription_serializer.rb'
     - 'app/services/import_service.rb'
 
-# This cop supports safe autocorrection (--autocorrect).
-Style/IfUnlessModifier:
-  Exclude:
-    - 'config/environments/production.rb'
-    - 'config/initializers/devise.rb'
-    - 'config/initializers/ffmpeg.rb'
-
 # This cop supports unsafe autocorrection (--autocorrect-all).
 Style/MapToHash:
   Exclude:
@@ -188,29 +169,13 @@ Style/RedundantConstantBase:
     - 'config/environments/production.rb'
     - 'config/initializers/sidekiq.rb'
 
-# This cop supports unsafe autocorrection (--autocorrect-all).
-# Configuration parameters: SafeForConstants.
-Style/RedundantFetchBlock:
-  Exclude:
-    - 'config/initializers/1_hosts.rb'
-    - 'config/initializers/chewy.rb'
-    - 'config/initializers/devise.rb'
-    - 'config/initializers/paperclip.rb'
-    - 'config/puma.rb'
-
 # This cop supports unsafe autocorrection (--autocorrect-all).
 # Configuration parameters: ConvertCodeThatCanStartToReturnNil, AllowedMethods, MaxChainLength.
 # AllowedMethods: present?, blank?, presence, try, try!
 Style/SafeNavigation:
   Exclude:
     - 'app/models/concerns/account/finder_concern.rb'
 
-# This cop supports unsafe autocorrection (--autocorrect-all).
-# Configuration parameters: Mode.
-Style/StringConcatenation:
-  Exclude:
-    - 'config/initializers/paperclip.rb'
-
 # This cop supports safe autocorrection (--autocorrect).
 # Configuration parameters: WordRegex.
 # SupportedStyles: percent, brackets

.ruby-version

@@ -1 +1 @@
-3.3.1
+3.3.2

CHANGELOG.md

@@ -2,6 +2,61 @@
 
 All notable changes to this project will be documented in this file.
 
+## [4.2.9] - 2024-05-30
+
+### Security
+
+- Update dependencies
+- Fix private mention filtering ([GHSA-5fq7-3p3j-9vrf](https://github.com/mastodon/mastodon/security/advisories/GHSA-5fq7-3p3j-9vrf))
+- Fix password change endpoint not being rate-limited ([GHSA-q3rg-xx5v-4mxh](https://github.com/mastodon/mastodon/security/advisories/GHSA-q3rg-xx5v-4mxh))
+- Add hardening around rate-limit bypass ([GHSA-c2r5-cfqr-c553](https://github.com/mastodon/mastodon/security/advisories/GHSA-c2r5-cfqr-c553))
+
+### Added
+
+- Add rate-limit on OAuth application registration ([ThisIsMissEm](https://github.com/mastodon/mastodon/pull/30316))
+- Add fallback redirection when getting a webfinger query `WEB_DOMAIN@WEB_DOMAIN` ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/28592))
+- Add `digest` attribute to `Admin::DomainBlock` entity in REST API ([ThisIsMissEm](https://github.com/mastodon/mastodon/pull/29092))
+
+### Removed
+
+- Remove superfluous application-level caching in some controllers ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/29862))
+- Remove aggressive OAuth application vacuuming ([ThisIsMissEm](https://github.com/mastodon/mastodon/pull/30316))
+
+### Fixed
+
+- Fix leaking Elasticsearch connections in Sidekiq processes ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/30450))
+- Fix language of remote posts not being recognized when using unusual casing ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/30403))
+- Fix off-by-one in `tootctl media` commands ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/30306))
+- Fix removal of allowed domains (in `LIMITED_FEDERATION_MODE`) not being recorded in the audit log ([ThisIsMissEm](https://github.com/mastodon/mastodon/pull/30125))
+- Fix not being able to block a subdomain of an already-blocked domain through the API ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/30119))
+- Fix `Idempotency-Key` being ignored when scheduling a post ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/30084))
+- Fix crash when supplying the `FFMPEG_BINARY` environment variable ([timothyjrogers](https://github.com/mastodon/mastodon/pull/30022))
+- Fix improper email address validation ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/29838))
+- Fix results/query in `api/v1/featured_tags/suggestions` ([mjankowski](https://github.com/mastodon/mastodon/pull/29597))
+- Fix unblocking internationalized domain names under certain conditions ([tribela](https://github.com/mastodon/mastodon/pull/29530))
+- Fix admin account created by `mastodon:setup` not being auto-approved ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/29379))
+- Fix reference to non-existent var in CLI maintenance command ([mjankowski](https://github.com/mastodon/mastodon/pull/28363))
+
+## [4.2.8] - 2024-02-23
+
+### Added
+
+- Add hourly task to automatically require approval for new registrations in the absence of moderators ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/29318), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/29355))
+  In order to prevent future abandoned Mastodon servers from being used for spam, harassment and other malicious activity, Mastodon will now automatically switch new user registrations to require moderator approval whenever they are left open and no activity (including non-moderation actions from apps) from any logged-in user with permission to access moderation reports has been detected in a full week.
+  When this happens, users with the permission to change server settings will receive an email notification.
+  This feature is disabled when `EMAIL_DOMAIN_ALLOWLIST` is used, and can also be disabled with `DISABLE_AUTOMATIC_SWITCHING_TO_APPROVED_REGISTRATIONS=true`.
+
+### Changed
+
+- Change registrations to be closed by default on new installations ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/29280))
+  If you are running a server and never changed your registrations mode from the default, updating will automatically close your registrations.
+  Simply re-enable them through the administration interface or using `tootctl settings registrations open` if you want to enable them again.
+
+### Fixed
+
+- Fix processing of remote ActivityPub actors making use of `Link` objects as `Image` `url` ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/29335))
+- Fix link verifications when page size exceeds 1MB ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/29358))
+
 ## [4.2.7] - 2024-02-16
 
 ### Fixed

Gemfile

@@ -31,7 +31,7 @@ gem 'browser'
 gem 'charlock_holmes', '~> 0.7.7'
 gem 'chewy', '~> 7.3'
 gem 'devise', '~> 4.9'
-gem 'devise-two-factor', '~> 4.1'
+gem 'devise-two-factor'
 
 group :pam_authentication, optional: true do
   gem 'devise_pam_authenticatable2', '~> 9.2'
@@ -57,7 +57,7 @@ gem 'htmlentities', '~> 4.3'
 gem 'http', '~> 5.2.0'
 gem 'http_accept_language', '~> 2.1'
 gem 'httplog', '~> 1.6.2'
-gem 'i18n', '1.14.1' # TODO: Remove version when resolved: https://github.com/glebm/i18n-tasks/issues/552 / https://github.com/ruby-i18n/i18n/pull/688
+gem 'i18n'
 gem 'idn-ruby', require: 'idn'
 gem 'inline_svg'
 gem 'kaminari', '~> 1.2'
@@ -103,6 +103,26 @@ gem 'rdf-normalize', '~> 0.5'
 
 gem 'private_address_check', '~> 0.5'
 
+gem 'opentelemetry-api', '~> 1.2.5'
+
+group :opentelemetry do
+  gem 'opentelemetry-exporter-otlp', '~> 0.26.3', require: false
+  gem 'opentelemetry-instrumentation-active_job', '~> 0.7.1', require: false
+  gem 'opentelemetry-instrumentation-active_model_serializers', '~> 0.20.1', require: false
+  gem 'opentelemetry-instrumentation-concurrent_ruby', '~> 0.21.2', require: false
+  gem 'opentelemetry-instrumentation-excon', '~> 0.22.0', require: false
+  gem 'opentelemetry-instrumentation-faraday', '~> 0.24.1', require: false
+  gem 'opentelemetry-instrumentation-http', '~> 0.23.2', require: false
+  gem 'opentelemetry-instrumentation-http_client', '~> 0.22.3', require: false
+  gem 'opentelemetry-instrumentation-net_http', '~> 0.22.4', require: false
+  gem 'opentelemetry-instrumentation-pg', '~> 0.27.1', require: false
+  gem 'opentelemetry-instrumentation-rack', '~> 0.24.1', require: false
+  gem 'opentelemetry-instrumentation-rails', '~> 0.30.0', require: false
+  gem 'opentelemetry-instrumentation-redis', '~> 0.25.3', require: false
+  gem 'opentelemetry-instrumentation-sidekiq', '~> 0.25.2', require: false
+  gem 'opentelemetry-sdk', '~> 1.4', require: false
+end
+
 group :test do
   # Adds RSpec Error/Warning annotations to GitHub PRs on the Files tab
   gem 'rspec-github', '~> 2.4', require: false
@@ -114,7 +134,7 @@ group :test do
   gem 'email_spec'
 
   # Extra RSpec extension methods and helpers for sidekiq
-  gem 'rspec-sidekiq', '~> 4.0'
+  gem 'rspec-sidekiq', '~> 5.0'
 
   # Browser integration testing
   gem 'capybara', '~> 3.39'
@@ -160,7 +180,7 @@ group :development do
 
   # Preview mail in the browser
   gem 'letter_opener', '~> 1.8'
-  gem 'letter_opener_web', '~> 2.0'
+  gem 'letter_opener_web', '~> 3.0'
 
   # Security analysis CLI tools
   gem 'brakeman', '~> 6.0', require: false