Merge pull request #9 from dftba-club/2024-06-07-1

2024-06-07-2

.devcontainer/Dockerfile

@@ -1,5 +1,5 @@
 # For details, see https://github.com/devcontainers/images/tree/main/src/ruby
-FROM mcr.microsoft.com/devcontainers/ruby:1-3.2-bullseye
+FROM mcr.microsoft.com/devcontainers/ruby:1-3.3-bookworm
 
 # Install Rails
 # RUN gem install rails webdrivers
@@ -9,7 +9,7 @@ RUN su vscode -c "source /usr/local/share/nvm/nvm.sh && nvm install ${NODE_VERSI
 
 # [Optional] Uncomment this section to install additional OS packages.
 RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
-    && apt-get -y install --no-install-recommends libicu-dev libidn11-dev ffmpeg imagemagick libpam-dev
+    && apt-get -y install --no-install-recommends libicu-dev libidn11-dev ffmpeg imagemagick libvips42 libpam-dev
 
 # [Optional] Uncomment this line to install additional gems.
 RUN gem install foreman

.devcontainer/codespaces/devcontainer.json

@@ -37,7 +37,7 @@
   },
 
   "onCreateCommand": "git config --global --add safe.directory ${containerWorkspaceFolder}",
-  "postCreateCommand": ".devcontainer/post-create.sh",
+  "postCreateCommand": "bin/setup",
   "waitFor": "postCreateCommand",
 
   "customizations": {

.devcontainer/devcontainer.json

@@ -23,12 +23,14 @@
     }
   },
 
+  "remoteUser": "root",
+
   "otherPortsAttributes": {
     "onAutoForward": "silent"
   },
 
   "onCreateCommand": "git config --global --add safe.directory ${containerWorkspaceFolder}",
-  "postCreateCommand": ".devcontainer/post-create.sh",
+  "postCreateCommand": "bin/setup",
   "waitFor": "postCreateCommand",
 
   "customizations": {

.devcontainer/docker-compose.yml

@@ -1,7 +1,6 @@
-version: '3'
-
 services:
   app:
+    working_dir: /workspaces/mastodon/
     build:
       context: .
       dockerfile: Dockerfile

.github/actions/setup-ruby/action.yml

@@ -14,7 +14,7 @@ runs:
       shell: bash
       run: |
         sudo apt-get update
-        sudo apt-get install -y libicu-dev libidn11-dev ${{ inputs.additional-system-dependencies }}
+        sudo apt-get install -y libicu-dev libidn11-dev libvips42 ${{ inputs.additional-system-dependencies }}
 
     - name: Set up Ruby
       uses: ruby/setup-ruby@v1

.github/workflows/test-ruby.yml

@@ -133,7 +133,7 @@ jobs:
         uses: ./.github/actions/setup-ruby
         with:
           ruby-version: ${{ matrix.ruby-version}}
-          additional-system-dependencies: ffmpeg imagemagick libpam-dev
+          additional-system-dependencies: ffmpeg libpam-dev
 
       - name: Load database schema
         run: './bin/rails db:create db:schema:load db:seed'
@@ -148,6 +148,93 @@ jobs:
         env:
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
 
+  test-libvips:
+    name: Libvips tests
+    runs-on: ubuntu-24.04
+
+    needs:
+      - build
+
+    services:
+      postgres:
+        image: postgres:14-alpine
+        env:
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_USER: postgres
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+
+      redis:
+        image: redis:7-alpine
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 6379:6379
+
+    env:
+      DB_HOST: localhost
+      DB_USER: postgres
+      DB_PASS: postgres
+      DISABLE_SIMPLECOV: ${{ matrix.ruby-version != '.ruby-version' }}
+      RAILS_ENV: test
+      ALLOW_NOPAM: true
+      PAM_ENABLED: true
+      PAM_DEFAULT_SERVICE: pam_test
+      PAM_CONTROLLED_SERVICE: pam_test_controlled
+      OIDC_ENABLED: true
+      OIDC_SCOPE: read
+      SAML_ENABLED: true
+      CAS_ENABLED: true
+      BUNDLE_WITH: 'pam_authentication test'
+      GITHUB_RSPEC: ${{ matrix.ruby-version == '.ruby-version' && github.event.pull_request && 'true' }}
+      MASTODON_USE_LIBVIPS: true
+
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby-version:
+          - '3.1'
+          - '3.2'
+          - '.ruby-version'
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/download-artifact@v4
+        with:
+          path: './'
+          name: ${{ github.sha }}
+
+      - name: Expand archived asset artifacts
+        run: |
+          tar xvzf artifacts.tar.gz
+
+      - name: Set up Ruby environment
+        uses: ./.github/actions/setup-ruby
+        with:
+          ruby-version: ${{ matrix.ruby-version}}
+          additional-system-dependencies: ffmpeg libpam-dev libyaml-dev
+
+      - name: Load database schema
+        run: './bin/rails db:create db:schema:load db:seed'
+
+      - run: bin/rspec --tag paperclip_processing
+
+      - name: Upload coverage reports to Codecov
+        if: matrix.ruby-version == '.ruby-version'
+        uses: codecov/codecov-action@v4
+        with:
+          files: coverage/lcov/mastodon.lcov
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
   test-e2e:
     name: End to End testing
     runs-on: ubuntu-latest
@@ -209,7 +296,7 @@ jobs:
         uses: ./.github/actions/setup-ruby
         with:
           ruby-version: ${{ matrix.ruby-version}}
-          additional-system-dependencies: ffmpeg imagemagick
+          additional-system-dependencies: ffmpeg
 
       - name: Set up Javascript environment
         uses: ./.github/actions/setup-javascript
@@ -329,7 +416,7 @@ jobs:
         uses: ./.github/actions/setup-ruby
         with:
           ruby-version: ${{ matrix.ruby-version}}
-          additional-system-dependencies: ffmpeg imagemagick
+          additional-system-dependencies: ffmpeg
 
       - name: Set up Javascript environment
         uses: ./.github/actions/setup-javascript

.nvmrc

@@ -1 +1 @@
-20.13
+20.14

.rubocop_todo.yml

@@ -176,12 +176,6 @@ Style/SafeNavigation:
   Exclude:
     - 'app/models/concerns/account/finder_concern.rb'
 
-# This cop supports unsafe autocorrection (--autocorrect-all).
-# Configuration parameters: Mode.
-Style/StringConcatenation:
-  Exclude:
-    - 'config/initializers/paperclip.rb'
-
 # This cop supports safe autocorrection (--autocorrect).
 # Configuration parameters: WordRegex.
 # SupportedStyles: percent, brackets

.ruby-version

@@ -1 +1 @@
-3.3.1
+3.3.2

CHANGELOG.md

@@ -2,6 +2,61 @@
 
 All notable changes to this project will be documented in this file.
 
+## [4.2.9] - 2024-05-30
+
+### Security
+
+- Update dependencies
+- Fix private mention filtering ([GHSA-5fq7-3p3j-9vrf](https://github.com/mastodon/mastodon/security/advisories/GHSA-5fq7-3p3j-9vrf))
+- Fix password change endpoint not being rate-limited ([GHSA-q3rg-xx5v-4mxh](https://github.com/mastodon/mastodon/security/advisories/GHSA-q3rg-xx5v-4mxh))
+- Add hardening around rate-limit bypass ([GHSA-c2r5-cfqr-c553](https://github.com/mastodon/mastodon/security/advisories/GHSA-c2r5-cfqr-c553))
+
+### Added
+
+- Add rate-limit on OAuth application registration ([ThisIsMissEm](https://github.com/mastodon/mastodon/pull/30316))
+- Add fallback redirection when getting a webfinger query `WEB_DOMAIN@WEB_DOMAIN` ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/28592))
+- Add `digest` attribute to `Admin::DomainBlock` entity in REST API ([ThisIsMissEm](https://github.com/mastodon/mastodon/pull/29092))
+
+### Removed
+
+- Remove superfluous application-level caching in some controllers ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/29862))
+- Remove aggressive OAuth application vacuuming ([ThisIsMissEm](https://github.com/mastodon/mastodon/pull/30316))
+
+### Fixed
+
+- Fix leaking Elasticsearch connections in Sidekiq processes ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/30450))
+- Fix language of remote posts not being recognized when using unusual casing ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/30403))
+- Fix off-by-one in `tootctl media` commands ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/30306))
+- Fix removal of allowed domains (in `LIMITED_FEDERATION_MODE`) not being recorded in the audit log ([ThisIsMissEm](https://github.com/mastodon/mastodon/pull/30125))
+- Fix not being able to block a subdomain of an already-blocked domain through the API ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/30119))
+- Fix `Idempotency-Key` being ignored when scheduling a post ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/30084))
+- Fix crash when supplying the `FFMPEG_BINARY` environment variable ([timothyjrogers](https://github.com/mastodon/mastodon/pull/30022))
+- Fix improper email address validation ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/29838))
+- Fix results/query in `api/v1/featured_tags/suggestions` ([mjankowski](https://github.com/mastodon/mastodon/pull/29597))
+- Fix unblocking internationalized domain names under certain conditions ([tribela](https://github.com/mastodon/mastodon/pull/29530))
+- Fix admin account created by `mastodon:setup` not being auto-approved ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/29379))
+- Fix reference to non-existent var in CLI maintenance command ([mjankowski](https://github.com/mastodon/mastodon/pull/28363))
+
+## [4.2.8] - 2024-02-23
+
+### Added
+
+- Add hourly task to automatically require approval for new registrations in the absence of moderators ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/29318), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/29355))
+  In order to prevent future abandoned Mastodon servers from being used for spam, harassment and other malicious activity, Mastodon will now automatically switch new user registrations to require moderator approval whenever they are left open and no activity (including non-moderation actions from apps) from any logged-in user with permission to access moderation reports has been detected in a full week.
+  When this happens, users with the permission to change server settings will receive an email notification.
+  This feature is disabled when `EMAIL_DOMAIN_ALLOWLIST` is used, and can also be disabled with `DISABLE_AUTOMATIC_SWITCHING_TO_APPROVED_REGISTRATIONS=true`.
+
+### Changed
+
+- Change registrations to be closed by default on new installations ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/29280))
+  If you are running a server and never changed your registrations mode from the default, updating will automatically close your registrations.
+  Simply re-enable them through the administration interface or using `tootctl settings registrations open` if you want to enable them again.
+
+### Fixed
+
+- Fix processing of remote ActivityPub actors making use of `Link` objects as `Image` `url` ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/29335))
+- Fix link verifications when page size exceeds 1MB ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/29358))
+
 ## [4.2.7] - 2024-02-16
 
 ### Fixed

Dockerfile

@@ -1,28 +1,31 @@
 # syntax=docker/dockerfile:1.7
 
+# This file is designed for production server deployment, not local development work
+# For a containerized local dev environment, see: https://github.com/mastodon/mastodon/blob/main/README.md#docker
+
 # Please see https://docs.docker.com/engine/reference/builder for information about
 # the extended buildx capabilities used in this file.
 # Make sure multiarch TARGETPLATFORM is available for interpolation
 # See: https://docs.docker.com/build/building/multi-platform/
 ARG TARGETPLATFORM=${TARGETPLATFORM}
 ARG BUILDPLATFORM=${BUILDPLATFORM}
 
-# Ruby image to use for base image, change with [--build-arg RUBY_VERSION="3.3.1"]
-ARG RUBY_VERSION="3.3.1"
+# Ruby image to use for base image, change with [--build-arg RUBY_VERSION="3.3.x"]
+ARG RUBY_VERSION="3.3.2"
 # # Node version to use in base image, change with [--build-arg NODE_MAJOR_VERSION="20"]
 ARG NODE_MAJOR_VERSION="20"
 # Debian image to use for base image, change with [--build-arg DEBIAN_VERSION="bookworm"]
 ARG DEBIAN_VERSION="bookworm"
 # Node image to use for base image based on combined variables (ex: 20-bookworm-slim)
 FROM docker.io/node:${NODE_MAJOR_VERSION}-${DEBIAN_VERSION}-slim as node
-# Ruby image to use for base image based on combined variables (ex: 3.3.1-slim-bookworm)
+# Ruby image to use for base image based on combined variables (ex: 3.3.x-slim-bookworm)
 FROM docker.io/ruby:${RUBY_VERSION}-slim-${DEBIAN_VERSION} as ruby
 
 # Resulting version string is vX.X.X-MASTODON_VERSION_PRERELEASE+MASTODON_VERSION_METADATA
 # Example: v4.2.0-nightly.2023.11.09+something
 # Overwrite existence of 'alpha.0' in version.rb [--build-arg MASTODON_VERSION_PRERELEASE="nightly.2023.11.09"]
 ARG MASTODON_VERSION_PRERELEASE=""
-# Append build metadata or fork information to version.rb [--build-arg MASTODON_VERSION_METADATA="something"]
+# Append build metadata or fork information to version.rb [--build-arg MASTODON_VERSION_METADATA="pr-12345"]
 ARG MASTODON_VERSION_METADATA=""
 
 # Allow Ruby on Rails to serve static files
@@ -43,6 +46,8 @@ ENV \
 # Apply Mastodon version information
   MASTODON_VERSION_PRERELEASE="${MASTODON_VERSION_PRERELEASE}" \
   MASTODON_VERSION_METADATA="${MASTODON_VERSION_METADATA}" \
+# Enable libvips
+  MASTODON_USE_LIBVIPS=true \
 # Apply Mastodon static files and YJIT options
   RAILS_SERVE_STATIC_FILES=${RAILS_SERVE_STATIC_FILES} \
   RUBY_YJIT_ENABLE=${RUBY_YJIT_ENABLE} \
@@ -97,7 +102,7 @@ RUN \
     curl \
     ffmpeg \
     file \
-    imagemagick \
+    libvips42 \
     libjemalloc2 \
     patchelf \
     procps \

Gemfile

@@ -23,6 +23,7 @@ gem 'fog-core', '<= 2.4.0'
 gem 'fog-openstack', '~> 1.0', require: false
 gem 'kt-paperclip', '~> 7.2'
 gem 'md-paperclip-azure', '~> 2.2', require: false
+gem 'ruby-vips', '~> 2.2', require: false
 
 gem 'active_model_serializers', '~> 0.10'
 gem 'addressable', '~> 2.8'
@@ -103,6 +104,8 @@ gem 'rdf-normalize', '~> 0.5'
 
 gem 'private_address_check', '~> 0.5'
 
+gem 'opentelemetry-api', '~> 1.2.5'
+
 group :opentelemetry do
   gem 'opentelemetry-exporter-otlp', '~> 0.26.3', require: false
   gem 'opentelemetry-instrumentation-active_job', '~> 0.7.1', require: false