diff --git a/.github/workflows/prepare-release.yml b/.github/workflows/prepare-release.yml index fabc79a2b739d..186f121baeb4d 100644 --- a/.github/workflows/prepare-release.yml +++ b/.github/workflows/prepare-release.yml @@ -83,4 +83,5 @@ jobs: files: | docker/docker-compose.yml docker/example.env + docker/hwaccel.yml *.apk diff --git a/cli/src/api/open-api/api.ts b/cli/src/api/open-api/api.ts index 6d10c4eb2ac59..4564402d9e782 100644 --- a/cli/src/api/open-api/api.ts +++ b/cli/src/api/open-api/api.ts @@ -704,13 +704,13 @@ export interface AssetStatsResponseDto { * @type {number} * @memberof AssetStatsResponseDto */ - 'total': number; + 'videos': number; /** * * @type {number} * @memberof AssetStatsResponseDto */ - 'videos': number; + 'total': number; } /** * @@ -2548,6 +2548,12 @@ export interface SystemConfigDto { * @interface SystemConfigFFmpegDto */ export interface SystemConfigFFmpegDto { + /** + * + * @type {TranscodeHWAccel} + * @memberof SystemConfigFFmpegDto + */ + 'accel': TranscodeHWAccel; /** * * @type {number} @@ -2896,6 +2902,22 @@ export const TimeGroupEnum = { export type TimeGroupEnum = typeof TimeGroupEnum[keyof typeof TimeGroupEnum]; +/** + * + * @export + * @enum {string} + */ + +export const TranscodeHWAccel = { + Nvenc: 'nvenc', + Qsv: 'qsv', + Vaapi: 'vaapi', + Disabled: 'disabled' +} as const; + +export type TranscodeHWAccel = typeof TranscodeHWAccel[keyof typeof TranscodeHWAccel]; + + /** * * @export diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml index 0f509ad51d800..c8b69da0e4cde 100644 --- a/docker/docker-compose.dev.yml +++ b/docker/docker-compose.dev.yml @@ -47,6 +47,9 @@ services: immich-microservices: container_name: immich_microservices image: immich-microservices:latest + # extends: + # file: hwaccel.yml + # service: hwaccel build: context: ../server dockerfile: Dockerfile diff --git a/docker/docker-compose.prod.yml b/docker/docker-compose.prod.yml index bf225d93129c6..8e4a6ca613268 100644 --- a/docker/docker-compose.prod.yml +++ b/docker/docker-compose.prod.yml @@ -33,6 +33,9 @@ services: immich-microservices: container_name: immich_microservices image: immich-microservices:latest + # extends: + # file: hwaccel.yml + # service: hwaccel build: context: ../server dockerfile: Dockerfile diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index b5f6a47b1c3ee..c7e3be9934067 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -18,6 +18,9 @@ services: immich-microservices: container_name: immich_microservices image: ghcr.io/immich-app/immich-server:${IMMICH_VERSION:-release} + # extends: + # file: hwaccel.yml + # service: hwaccel command: [ "start.sh", "microservices" ] volumes: - ${UPLOAD_LOCATION}:/usr/src/app/upload diff --git a/docker/hwaccel.yml b/docker/hwaccel.yml new file mode 100644 index 0000000000000..fb6554248fb92 --- /dev/null +++ b/docker/hwaccel.yml @@ -0,0 +1,23 @@ +version: "3.8" + +# Hardware acceleration for transcoding - Optional +# This is only needed if you want to use hardware acceleration for transcoding. +# Depending on your hardware, you should uncomment the relevant lines below. + +services: + hwaccel: + # devices: + # - /dev/dri:/dev/dri # If using Intel QuickSync or VAAPI + # volumes: + # - /usr/lib/wsl:/usr/lib/wsl # If using VAAPI in WSL2 + # environment: + # - NVIDIA_DRIVER_CAPABILITIES=all # If using NVIDIA GPU + # - LD_LIBRARY_PATH=/usr/lib/wsl/lib # If using VAAPI in WSL2 + # - LIBVA_DRIVER_NAME=d3d12 # If using VAAPI in WSL2 + # deploy: # Uncomment this section if using NVIDIA GPU + # resources: + # reservations: + # devices: + # - driver: nvidia + # count: 1 + # capabilities: [gpu] diff --git a/docs/docs/features/hardware-transcoding.md b/docs/docs/features/hardware-transcoding.md new file mode 100644 index 0000000000000..2f4a3e478576f --- /dev/null +++ b/docs/docs/features/hardware-transcoding.md @@ -0,0 +1,60 @@ +# Hardware Transcoding [Experimental] + +This feature allows you to use a GPU or Intel Quick Sync to accelerate transcoding and reduce CPU load. +Note that hardware transcoding is much less efficient for file sizes. +As this is a new feature, it is still experimental and may not work on all systems. + +## Supported APIs + +- NVENC + - NVIDIA GPUs +- Quick Sync + - Intel CPUs +- VAAPI + - GPUs + +## Limitations + +- The instructions and configurations here are specific to Docker Compose. Other container engines may require different configuration. +- Only Linux and Windows (through WSL2) servers are supported. +- WSL2 does not support Quick Sync. +- Raspberry Pi is currently not supported. +- Two-pass mode is only supported for NVENC. Other APIs will ignore this setting. +- Only encoding is currently hardware accelerated, so the CPU is still used for software decoding. + - This is mainly because the original video may not be hardware-decodable. +- Hardware dependent + - Codec support varies, but H.264 and HEVC are usually supported. + - Notably, NVIDIA and AMD GPUs do not support VP9 encoding. + - Newer devices tend to have higher transcoding quality. + +## Prerequisites + +#### NVENC + +- You must have the official NVIDIA driver installed on the server. +- On Linux (except for WSL2), you also need to have [NVIDIA Container Runtime][nvcr] installed. + +#### QSV + +- For VP9 to work: + - You must have a 9th gen Intel CPU or newer + - If you have an 11th gen CPU or older, then you may need to follow [these][jellyfin-lp] instructions as Low-Power mode is required + - Additionally, if the server specifically has an 11th gen CPU and is running kernel 5.15 (shipped with Ubuntu 22.04 LTS), then you will need to upgrade this kernel (from [Jellyfin docs][jellyfin-kernel-bug]) + +## Setup + +1. If you do not already have it, download the latest [`hwaccel.yml`][hw-file] file and ensure it's in the same folder as the `docker-compose.yml`. +2. Uncomment the lines that apply to your system and desired usage. +3. In the `docker-compose.yml` under `immich-microservices`, uncomment the lines relating to the `hwaccel.yml` file. +4. Redeploy the `immich-microservices` container with these updated settings. +5. In the Admin page under `FFmpeg settings`, change the hardware acceleration setting to the appropriate option and save. + +## Tips + +- You may want to choose a slower preset than for software transcoding to maintain quality and efficiency +- While you can use VAAPI with Nvidia GPUs and Intel CPUs, prefer the more specific APIs since they're more optimized for their respective devices + +[hw-file]: https://github.com/immich-app/immich/releases/latest/download/hwaccel.yml +[nvcr]: https://github.com/NVIDIA/nvidia-container-runtime/ +[jellyfin-lp]: https://jellyfin.org/docs/general/administration/hardware-acceleration/intel/#configure-and-verify-lp-mode-on-linux +[jellyfin-kernel-bug]: https://jellyfin.org/docs/general/administration/hardware-acceleration/intel/#known-issues-and-limitations diff --git a/docs/docs/install/docker-compose.md b/docs/docs/install/docker-compose.md index 046fe68285d42..dca8c0211c7c8 100644 --- a/docs/docs/install/docker-compose.md +++ b/docs/docs/install/docker-compose.md @@ -25,10 +25,18 @@ wget https://github.com/immich-app/immich/releases/latest/download/docker-compos wget -O .env https://github.com/immich-app/immich/releases/latest/download/example.env ``` +```bash title="(Optional) Get hwaccel.yml file" +wget https://github.com/immich-app/immich/releases/latest/download/hwaccel.yml +``` + or by downloading from your browser and moving the files to the directory that you created. Note: If you downloaded the files from your browser, also ensure that you rename `example.env` to `.env`. +:::info +Optionally, you can use the [`hwaccel.yml`][hw-file] file to enable hardware acceleration for transcoding. See the [Hardware Transcoding](/docs/features/hardware-transcoding.md) guide for info on how to set this up. +::: + ### Step 2 - Populate the .env file with custom values
@@ -186,4 +194,5 @@ Immich is currently under heavy development, which means you can expect breaking [compose-file]: https://github.com/immich-app/immich/releases/latest/download/docker-compose.yml [env-file]: https://github.com/immich-app/immich/releases/latest/download/example.env +[hw-file]: https://github.com/immich-app/immich/releases/latest/download/hwaccel.yml [watchtower]: https://containrrr.dev/watchtower/ diff --git a/mobile/openapi/.openapi-generator/FILES b/mobile/openapi/.openapi-generator/FILES index ad445090a25ec..91ea108da9772 100644 --- a/mobile/openapi/.openapi-generator/FILES +++ b/mobile/openapi/.openapi-generator/FILES @@ -115,6 +115,7 @@ doc/TagResponseDto.md doc/TagTypeEnum.md doc/ThumbnailFormat.md doc/TimeGroupEnum.md +doc/TranscodeHWAccel.md doc/TranscodePolicy.md doc/UpdateAlbumDto.md doc/UpdateAssetDto.md @@ -249,6 +250,7 @@ lib/model/tag_response_dto.dart lib/model/tag_type_enum.dart lib/model/thumbnail_format.dart lib/model/time_group_enum.dart +lib/model/transcode_hw_accel.dart lib/model/transcode_policy.dart lib/model/update_album_dto.dart lib/model/update_asset_dto.dart @@ -372,6 +374,7 @@ test/tag_response_dto_test.dart test/tag_type_enum_test.dart test/thumbnail_format_test.dart test/time_group_enum_test.dart +test/transcode_hw_accel_test.dart test/transcode_policy_test.dart test/update_album_dto_test.dart test/update_asset_dto_test.dart diff --git a/mobile/openapi/README.md b/mobile/openapi/README.md index 38780bd67a98f..ff4175755d245 100644 --- a/mobile/openapi/README.md +++ b/mobile/openapi/README.md @@ -277,6 +277,7 @@ Class | Method | HTTP request | Description - [TagTypeEnum](doc//TagTypeEnum.md) - [ThumbnailFormat](doc//ThumbnailFormat.md) - [TimeGroupEnum](doc//TimeGroupEnum.md) + - [TranscodeHWAccel](doc//TranscodeHWAccel.md) - [TranscodePolicy](doc//TranscodePolicy.md) - [UpdateAlbumDto](doc//UpdateAlbumDto.md) - [UpdateAssetDto](doc//UpdateAssetDto.md) diff --git a/mobile/openapi/doc/AssetStatsResponseDto.md b/mobile/openapi/doc/AssetStatsResponseDto.md index 370b7c059b0eb..d7937a7eda44a 100644 --- a/mobile/openapi/doc/AssetStatsResponseDto.md +++ b/mobile/openapi/doc/AssetStatsResponseDto.md @@ -9,8 +9,8 @@ import 'package:openapi/api.dart'; Name | Type | Description | Notes ------------ | ------------- | ------------- | ------------- **images** | **int** | | -**total** | **int** | | **videos** | **int** | | +**total** | **int** | | [[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) diff --git a/mobile/openapi/doc/SystemConfigFFmpegDto.md b/mobile/openapi/doc/SystemConfigFFmpegDto.md index a23815e9dcd05..334d268d82b68 100644 --- a/mobile/openapi/doc/SystemConfigFFmpegDto.md +++ b/mobile/openapi/doc/SystemConfigFFmpegDto.md @@ -8,6 +8,7 @@ import 'package:openapi/api.dart'; ## Properties Name | Type | Description | Notes ------------ | ------------- | ------------- | ------------- +**accel** | [**TranscodeHWAccel**](TranscodeHWAccel.md) | | **crf** | **int** | | **maxBitrate** | **String** | | **preset** | **String** | | diff --git a/mobile/openapi/doc/TranscodeHWAccel.md b/mobile/openapi/doc/TranscodeHWAccel.md new file mode 100644 index 0000000000000..c03f561660c0c --- /dev/null +++ b/mobile/openapi/doc/TranscodeHWAccel.md @@ -0,0 +1,14 @@ +# openapi.model.TranscodeHWAccel + +## Load the model package +```dart +import 'package:openapi/api.dart'; +``` + +## Properties +Name | Type | Description | Notes +------------ | ------------- | ------------- | ------------- + +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) + + diff --git a/mobile/openapi/lib/api.dart b/mobile/openapi/lib/api.dart index 36415d353ae97..b61dd82babf64 100644 --- a/mobile/openapi/lib/api.dart +++ b/mobile/openapi/lib/api.dart @@ -142,6 +142,7 @@ part 'model/tag_response_dto.dart'; part 'model/tag_type_enum.dart'; part 'model/thumbnail_format.dart'; part 'model/time_group_enum.dart'; +part 'model/transcode_hw_accel.dart'; part 'model/transcode_policy.dart'; part 'model/update_album_dto.dart'; part 'model/update_asset_dto.dart'; diff --git a/mobile/openapi/lib/api_client.dart b/mobile/openapi/lib/api_client.dart index 000629e1c8072..9da7faa6657c2 100644 --- a/mobile/openapi/lib/api_client.dart +++ b/mobile/openapi/lib/api_client.dart @@ -379,6 +379,8 @@ class ApiClient { return ThumbnailFormatTypeTransformer().decode(value); case 'TimeGroupEnum': return TimeGroupEnumTypeTransformer().decode(value); + case 'TranscodeHWAccel': + return TranscodeHWAccelTypeTransformer().decode(value); case 'TranscodePolicy': return TranscodePolicyTypeTransformer().decode(value); case 'UpdateAlbumDto': diff --git a/mobile/openapi/lib/api_helper.dart b/mobile/openapi/lib/api_helper.dart index 9e7f5c3bead0d..c41cea12b6075 100644 --- a/mobile/openapi/lib/api_helper.dart +++ b/mobile/openapi/lib/api_helper.dart @@ -82,6 +82,9 @@ String parameterToString(dynamic value) { if (value is TimeGroupEnum) { return TimeGroupEnumTypeTransformer().encode(value).toString(); } + if (value is TranscodeHWAccel) { + return TranscodeHWAccelTypeTransformer().encode(value).toString(); + } if (value is TranscodePolicy) { return TranscodePolicyTypeTransformer().encode(value).toString(); } diff --git a/mobile/openapi/lib/model/asset_stats_response_dto.dart b/mobile/openapi/lib/model/asset_stats_response_dto.dart index d910917745d12..1221712d89c1c 100644 --- a/mobile/openapi/lib/model/asset_stats_response_dto.dart +++ b/mobile/openapi/lib/model/asset_stats_response_dto.dart @@ -14,37 +14,37 @@ class AssetStatsResponseDto { /// Returns a new [AssetStatsResponseDto] instance. AssetStatsResponseDto({ required this.images, - required this.total, required this.videos, + required this.total, }); int images; - int total; - int videos; + int total; + @override bool operator ==(Object other) => identical(this, other) || other is AssetStatsResponseDto && other.images == images && - other.total == total && - other.videos == videos; + other.videos == videos && + other.total == total; @override int get hashCode => // ignore: unnecessary_parenthesis (images.hashCode) + - (total.hashCode) + - (videos.hashCode); + (videos.hashCode) + + (total.hashCode); @override - String toString() => 'AssetStatsResponseDto[images=$images, total=$total, videos=$videos]'; + String toString() => 'AssetStatsResponseDto[images=$images, videos=$videos, total=$total]'; Map toJson() { final json = {}; json[r'images'] = this.images; - json[r'total'] = this.total; json[r'videos'] = this.videos; + json[r'total'] = this.total; return json; } @@ -57,8 +57,8 @@ class AssetStatsResponseDto { return AssetStatsResponseDto( images: mapValueOfType(json, r'images')!, - total: mapValueOfType(json, r'total')!, videos: mapValueOfType(json, r'videos')!, + total: mapValueOfType(json, r'total')!, ); } return null; @@ -107,8 +107,8 @@ class AssetStatsResponseDto { /// The list of required keys that must be present in a JSON. static const requiredKeys = { 'images', - 'total', 'videos', + 'total', }; } diff --git a/mobile/openapi/lib/model/system_config_f_fmpeg_dto.dart b/mobile/openapi/lib/model/system_config_f_fmpeg_dto.dart index 003d98ca14bd8..4dbc471c8cd30 100644 --- a/mobile/openapi/lib/model/system_config_f_fmpeg_dto.dart +++ b/mobile/openapi/lib/model/system_config_f_fmpeg_dto.dart @@ -13,6 +13,7 @@ part of openapi.api; class SystemConfigFFmpegDto { /// Returns a new [SystemConfigFFmpegDto] instance. SystemConfigFFmpegDto({ + required this.accel, required this.crf, required this.maxBitrate, required this.preset, @@ -24,6 +25,8 @@ class SystemConfigFFmpegDto { required this.twoPass, }); + TranscodeHWAccel accel; + int crf; String maxBitrate; @@ -44,6 +47,7 @@ class SystemConfigFFmpegDto { @override bool operator ==(Object other) => identical(this, other) || other is SystemConfigFFmpegDto && + other.accel == accel && other.crf == crf && other.maxBitrate == maxBitrate && other.preset == preset && @@ -57,6 +61,7 @@ class SystemConfigFFmpegDto { @override int get hashCode => // ignore: unnecessary_parenthesis + (accel.hashCode) + (crf.hashCode) + (maxBitrate.hashCode) + (preset.hashCode) + @@ -68,10 +73,11 @@ class SystemConfigFFmpegDto { (twoPass.hashCode); @override - String toString() => 'SystemConfigFFmpegDto[crf=$crf, maxBitrate=$maxBitrate, preset=$preset, targetAudioCodec=$targetAudioCodec, targetResolution=$targetResolution, targetVideoCodec=$targetVideoCodec, threads=$threads, transcode=$transcode, twoPass=$twoPass]'; + String toString() => 'SystemConfigFFmpegDto[accel=$accel, crf=$crf, maxBitrate=$maxBitrate, preset=$preset, targetAudioCodec=$targetAudioCodec, targetResolution=$targetResolution, targetVideoCodec=$targetVideoCodec, threads=$threads, transcode=$transcode, twoPass=$twoPass]'; Map toJson() { final json = {}; + json[r'accel'] = this.accel; json[r'crf'] = this.crf; json[r'maxBitrate'] = this.maxBitrate; json[r'preset'] = this.preset; @@ -92,6 +98,7 @@ class SystemConfigFFmpegDto { final json = value.cast(); return SystemConfigFFmpegDto( + accel: TranscodeHWAccel.fromJson(json[r'accel'])!, crf: mapValueOfType(json, r'crf')!, maxBitrate: mapValueOfType(json, r'maxBitrate')!, preset: mapValueOfType(json, r'preset')!, @@ -148,6 +155,7 @@ class SystemConfigFFmpegDto { /// The list of required keys that must be present in a JSON. static const requiredKeys = { + 'accel', 'crf', 'maxBitrate', 'preset', diff --git a/mobile/openapi/lib/model/transcode_hw_accel.dart b/mobile/openapi/lib/model/transcode_hw_accel.dart new file mode 100644 index 0000000000000..5db18bb70ec2f --- /dev/null +++ b/mobile/openapi/lib/model/transcode_hw_accel.dart @@ -0,0 +1,91 @@ +// +// AUTO-GENERATED FILE, DO NOT MODIFY! +// +// @dart=2.12 + +// ignore_for_file: unused_element, unused_import +// ignore_for_file: always_put_required_named_parameters_first +// ignore_for_file: constant_identifier_names +// ignore_for_file: lines_longer_than_80_chars + +part of openapi.api; + + +class TranscodeHWAccel { + /// Instantiate a new enum with the provided [value]. + const TranscodeHWAccel._(this.value); + + /// The underlying value of this enum member. + final String value; + + @override + String toString() => value; + + String toJson() => value; + + static const nvenc = TranscodeHWAccel._(r'nvenc'); + static const qsv = TranscodeHWAccel._(r'qsv'); + static const vaapi = TranscodeHWAccel._(r'vaapi'); + static const disabled = TranscodeHWAccel._(r'disabled'); + + /// List of all possible values in this [enum][TranscodeHWAccel]. + static const values = [ + nvenc, + qsv, + vaapi, + disabled, + ]; + + static TranscodeHWAccel? fromJson(dynamic value) => TranscodeHWAccelTypeTransformer().decode(value); + + static List? listFromJson(dynamic json, {bool growable = false,}) { + final result = []; + if (json is List && json.isNotEmpty) { + for (final row in json) { + final value = TranscodeHWAccel.fromJson(row); + if (value != null) { + result.add(value); + } + } + } + return result.toList(growable: growable); + } +} + +/// Transformation class that can [encode] an instance of [TranscodeHWAccel] to String, +/// and [decode] dynamic data back to [TranscodeHWAccel]. +class TranscodeHWAccelTypeTransformer { + factory TranscodeHWAccelTypeTransformer() => _instance ??= const TranscodeHWAccelTypeTransformer._(); + + const TranscodeHWAccelTypeTransformer._(); + + String encode(TranscodeHWAccel data) => data.value; + + /// Decodes a [dynamic value][data] to a TranscodeHWAccel. + /// + /// If [allowNull] is true and the [dynamic value][data] cannot be decoded successfully, + /// then null is returned. However, if [allowNull] is false and the [dynamic value][data] + /// cannot be decoded successfully, then an [UnimplementedError] is thrown. + /// + /// The [allowNull] is very handy when an API changes and a new enum value is added or removed, + /// and users are still using an old app with the old code. + TranscodeHWAccel? decode(dynamic data, {bool allowNull = true}) { + if (data != null) { + switch (data) { + case r'nvenc': return TranscodeHWAccel.nvenc; + case r'qsv': return TranscodeHWAccel.qsv; + case r'vaapi': return TranscodeHWAccel.vaapi; + case r'disabled': return TranscodeHWAccel.disabled; + default: + if (!allowNull) { + throw ArgumentError('Unknown enum value to decode: $data'); + } + } + } + return null; + } + + /// Singleton [TranscodeHWAccelTypeTransformer] instance. + static TranscodeHWAccelTypeTransformer? _instance; +} + diff --git a/mobile/openapi/test/asset_stats_response_dto_test.dart b/mobile/openapi/test/asset_stats_response_dto_test.dart index eaeace92a7f4f..3e5d8b548f581 100644 --- a/mobile/openapi/test/asset_stats_response_dto_test.dart +++ b/mobile/openapi/test/asset_stats_response_dto_test.dart @@ -21,13 +21,13 @@ void main() { // TODO }); - // int total - test('to test the property `total`', () async { + // int videos + test('to test the property `videos`', () async { // TODO }); - // int videos - test('to test the property `videos`', () async { + // int total + test('to test the property `total`', () async { // TODO }); diff --git a/mobile/openapi/test/system_config_f_fmpeg_dto_test.dart b/mobile/openapi/test/system_config_f_fmpeg_dto_test.dart index f2aac1e60e542..13f085acf74cc 100644 --- a/mobile/openapi/test/system_config_f_fmpeg_dto_test.dart +++ b/mobile/openapi/test/system_config_f_fmpeg_dto_test.dart @@ -16,6 +16,11 @@ void main() { // final instance = SystemConfigFFmpegDto(); group('test SystemConfigFFmpegDto', () { + // TranscodeHWAccel accel + test('to test the property `accel`', () async { + // TODO + }); + // int crf test('to test the property `crf`', () async { // TODO diff --git a/mobile/openapi/test/transcode_hw_accel_test.dart b/mobile/openapi/test/transcode_hw_accel_test.dart new file mode 100644 index 0000000000000..c9887c87d57b5 --- /dev/null +++ b/mobile/openapi/test/transcode_hw_accel_test.dart @@ -0,0 +1,21 @@ +// +// AUTO-GENERATED FILE, DO NOT MODIFY! +// +// @dart=2.12 + +// ignore_for_file: unused_element, unused_import +// ignore_for_file: always_put_required_named_parameters_first +// ignore_for_file: constant_identifier_names +// ignore_for_file: lines_longer_than_80_chars + +import 'package:openapi/api.dart'; +import 'package:test/test.dart'; + +// tests for TranscodeHWAccel +void main() { + + group('test TranscodeHWAccel', () { + + }); + +} diff --git a/server/Dockerfile b/server/Dockerfile index b62c062c5f8c5..4d04b16e98866 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -1,8 +1,19 @@ -FROM node:18.16.0-alpine3.18@sha256:f41850f74ff16a33daff988e2ea06ef8f5daeb6fb84913c7df09552a98caba09 as builder +FROM node:18-bookworm@sha256:c85dc4392f44f5de1d0d72dd20a088a542734445f99bed7aa8ac895c706d370d as builder WORKDIR /usr/src/app -RUN apk add --update-cache build-base imagemagick-dev python3 ffmpeg libraw-dev perl vips-dev vips-heif vips-jxl vips-magick +COPY bin/install-ffmpeg.sh build-lock.json ./ +RUN sed -i -e's/ main/ main contrib non-free non-free-firmware/g' /etc/apt/sources.list.d/debian.sources +RUN apt-get update && apt-get install -yqq build-essential ninja-build meson pkg-config jq \ +libglib2.0-dev libexpat1-dev librsvg2-dev libexif-dev libwebp-dev liborc-0.4-dev libtiff5-dev \ +libjpeg62-turbo-dev libgsf-1-dev libspng-dev libraw-dev libjxl-dev libheif-dev \ +mesa-va-drivers libmimalloc2.0 $(if [ $(arch) = "x86_64" ]; then echo "intel-media-va-driver-non-free"; fi) \ +&& ./install-ffmpeg.sh && apt-get autoremove && apt-get clean && rm -rf /var/lib/apt/lists/* + +# debian build for imagemagick has broken RAW support, so build manually +COPY bin/build-imagemagick.sh bin/build-libvips.sh ./ +RUN ./build-imagemagick.sh +RUN ./build-libvips.sh COPY package.json package-lock.json ./ @@ -15,14 +26,31 @@ FROM builder as prod RUN npm run build RUN npm prune --omit=dev --omit=optional - -FROM node:18.16.0-alpine3.18@sha256:f41850f74ff16a33daff988e2ea06ef8f5daeb6fb84913c7df09552a98caba09 +FROM node:18-bookworm-slim@sha256:a0cca98f2896135d4c0386922211c1f90f98f27a58b8f2c07850d0fbe1c2104e ENV NODE_ENV=production WORKDIR /usr/src/app -RUN apk add --no-cache ffmpeg imagemagick-dev libraw-dev perl tini vips-dev vips-heif vips-jxl vips-magick +COPY bin/install-ffmpeg.sh build-lock.json ./ +RUN sed -i -e's/ main/ main contrib non-free non-free-firmware/g' /etc/apt/sources.list.d/debian.sources +RUN apt-get update && apt-get install -yqq tini libheif1 libwebp7 libwebpdemux2 libwebpmux3 mesa-va-drivers \ +libjpeg62-turbo libexpat1 librsvg2-2 libjxl0.7 libraw20 libtiff6 libspng0 libexif12 libgcc-s1 libglib2.0-0 \ +libgsf-1-114 libopenjp2-7 liblcms2-2 liborc-0.4-0 libopenexr-3-1-30 liblqr-1-0 libltdl7 zlib1g \ +mesa-va-drivers libmimalloc2.0 $(if [ $(arch) = "x86_64" ]; then echo "intel-media-va-driver-non-free"; fi) jq wget \ +&& ./install-ffmpeg.sh && apt-get remove -yqq jq wget && apt-get autoremove -yqq && apt-get clean && rm -rf /var/lib/apt/lists/* \ +&& rm install-ffmpeg.sh && rm build-lock.json +ENV PATH=/usr/lib/jellyfin-ffmpeg:$PATH + +COPY --from=prod /usr/local/bin/magick /usr/local/bin/magick +COPY --from=prod /usr/local/include/ImageMagick-7 /usr/local/include/ImageMagick-7 + +COPY --from=prod /usr/local/bin/vips /usr/local/bin/vips +COPY --from=prod /usr/local/include/vips/ /usr/local/include/vips/ + +COPY --from=prod /usr/local/lib/ /usr/local/lib/ + +RUN ldconfig /usr/local/lib COPY --from=prod /usr/src/app/node_modules ./node_modules COPY --from=prod /usr/src/app/dist ./dist @@ -34,7 +62,6 @@ COPY package.json package-lock.json ./ COPY start*.sh ./ RUN npm link && npm cache clean --force - VOLUME /usr/src/app/upload EXPOSE 3001 diff --git a/server/bin/build-imagemagick.sh b/server/bin/build-imagemagick.sh new file mode 100755 index 0000000000000..01225eb46a0fb --- /dev/null +++ b/server/bin/build-imagemagick.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +set -e + +LOCK=$(jq -c '.packages[] | select(.name == "imagemagick")' build-lock.json) +IMAGEMAGICK_VERSION=${IMAGEMAGICK_VERSION:=$(echo $LOCK | jq -r '.version')} +IMAGEMAGICK_SHA256=${IMAGEMAGICK_SHA256:=$(echo $LOCK | jq -r '.sha256')} + +echo "$IMAGEMAGICK_SHA256 $IMAGEMAGICK_VERSION.tar.gz" > imagemagick.sha256 +mkdir -p ImageMagick +wget -nv https://github.com/ImageMagick/ImageMagick/archive/${IMAGEMAGICK_VERSION}.tar.gz +sha256sum -c imagemagick.sha256 +tar -xvf ${IMAGEMAGICK_VERSION}.tar.gz -C ImageMagick --strip-components=1 +rm ${IMAGEMAGICK_VERSION}.tar.gz +rm imagemagick.sha256 +cd ImageMagick +./configure --with-modules +make -j$(nproc) +make install +cd .. && rm -rf ImageMagick +ldconfig /usr/local/lib diff --git a/server/bin/build-libvips.sh b/server/bin/build-libvips.sh new file mode 100755 index 0000000000000..15ffa819b1538 --- /dev/null +++ b/server/bin/build-libvips.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +set -e + +LOCK=$(jq -c '.packages[] | select(.name == "libvips")' build-lock.json) +LIBVIPS_VERSION=${LIBVIPS_VERSION:=$(echo $LOCK | jq -r '.version')} +LIBVIPS_SHA256=${LIBVIPS_SHA256:=$(echo $LOCK | jq -r '.sha256')} + +echo "$LIBVIPS_SHA256 vips-$LIBVIPS_VERSION.tar.xz" > libvips.sha256 +mkdir -p libvips +wget -nv https://github.com/libvips/libvips/releases/download/v${LIBVIPS_VERSION}/vips-${LIBVIPS_VERSION}.tar.xz +sha256sum -c libvips.sha256 +tar -xvf vips-${LIBVIPS_VERSION}.tar.xz -C libvips --strip-components=1 +rm vips-${LIBVIPS_VERSION}.tar.xz +rm libvips.sha256 +cd libvips +meson setup build --buildtype=release --libdir=lib -Dintrospection=false +cd build +# ninja test # tests set concurrency too high for arm/v7 +ninja install +cd .. && rm -rf libvips +ldconfig /usr/local/lib diff --git a/server/bin/install-ffmpeg.sh b/server/bin/install-ffmpeg.sh new file mode 100755 index 0000000000000..f3547e8d3d961 --- /dev/null +++ b/server/bin/install-ffmpeg.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +set -e + +LOCK=$(jq -c '.packages[] | select(.name == "ffmpeg")' build-lock.json) +export TARGETARCH=${TARGETARCH:=$(dpkg --print-architecture)} +FFMPEG_VERSION=${FFMPEG_VERSION:=$(echo $LOCK | jq -r '.version')} +FFMPEG_SHA256=${FFMPEG_SHA256:=$(echo $LOCK | jq -r '.sha256[$ENV.TARGETARCH]')} + +echo "$FFMPEG_SHA256 jellyfin-ffmpeg6_${FFMPEG_VERSION}-bookworm_${TARGETARCH}.deb" > ffmpeg.sha256 + +wget -nv https://github.com/jellyfin/jellyfin-ffmpeg/releases/download/v${FFMPEG_VERSION}/jellyfin-ffmpeg6_${FFMPEG_VERSION}-bookworm_${TARGETARCH}.deb +sha256sum -c ffmpeg.sha256 +apt-get -yqq -f install ./jellyfin-ffmpeg6_${FFMPEG_VERSION}-bookworm_${TARGETARCH}.deb +rm jellyfin-ffmpeg6_${FFMPEG_VERSION}-bookworm_${TARGETARCH}.deb +rm ffmpeg.sha256 +ldconfig /usr/lib/jellyfin-ffmpeg/lib diff --git a/server/build-lock.json b/server/build-lock.json new file mode 100644 index 0000000000000..0bd768b14fb73 --- /dev/null +++ b/server/build-lock.json @@ -0,0 +1,24 @@ +{ + "packages": [ + { + "name": "imagemagick", + "version": "7.1.1-13", + + "sha256": "8e3ce1aaad19da9f2ca444072bcc631d193a219e3ee11c13ad6d3c895044142c" + }, + { + "name": "libvips", + "version": "8.14.2", + "sha256": "27dad021f0835a5ab14e541d02abd41e4c3bd012d2196438df5a9e754984f7ce" + }, + { + "name": "ffmpeg", + "version": "6.0-4", + "sha256": { + "amd64": "18d98b292b891cde86c2a08e5e989c3430e51a136cdc232bc4162fef3b4f0f44", + "arm64": "67eb1e5a38ac695dd253d9ac290ad0e9fb709e8260449a7445e8460b7db3c516", + "armhf": "a29605ab0eced3511c8a6623504fab5b8bb174a486d87f94bf5522ed9a5970e6" + } + } + ] +} diff --git a/server/immich-openapi-specs.json b/server/immich-openapi-specs.json index 28db8ca291cc2..14c65e2ecd8de 100644 --- a/server/immich-openapi-specs.json +++ b/server/immich-openapi-specs.json @@ -5003,14 +5003,15 @@ "type": "object" }, "AssetStatsResponseDto": { + "type": "object", "properties": { "images": { "type": "integer" }, - "total": { + "videos": { "type": "integer" }, - "videos": { + "total": { "type": "integer" } }, @@ -5018,8 +5019,7 @@ "images", "videos", "total" - ], - "type": "object" + ] }, "AssetTypeEnum": { "enum": [ @@ -6577,6 +6577,9 @@ }, "SystemConfigFFmpegDto": { "properties": { + "accel": { + "$ref": "#/components/schemas/TranscodeHWAccel" + }, "crf": { "type": "integer" }, @@ -6611,6 +6614,7 @@ "targetVideoCodec", "targetAudioCodec", "transcode", + "accel", "preset", "targetResolution", "maxBitrate", @@ -6839,6 +6843,15 @@ ], "type": "string" }, + "TranscodeHWAccel": { + "enum": [ + "nvenc", + "qsv", + "vaapi", + "disabled" + ], + "type": "string" + }, "TranscodePolicy": { "enum": [ "all", diff --git a/server/src/domain/media/media.repository.ts b/server/src/domain/media/media.repository.ts index c6ca835df494e..28e103186ba85 100644 --- a/server/src/domain/media/media.repository.ts +++ b/server/src/domain/media/media.repository.ts @@ -1,3 +1,5 @@ +import { VideoCodec } from '@app/infra/entities'; + export const IMediaRepository = 'IMediaRepository'; export interface ResizeOptions { @@ -55,6 +57,10 @@ export interface VideoCodecSWConfig { getOptions(stream: VideoStreamInfo): TranscodeOptions; } +export interface VideoCodecHWConfig extends VideoCodecSWConfig { + getSupportedCodecs(): Array; +} + export interface IMediaRepository { // image resize(input: string | Buffer, output: string, options: ResizeOptions): Promise; diff --git a/server/src/domain/media/media.service.spec.ts b/server/src/domain/media/media.service.spec.ts index 4ab136a13eeaa..4a48f3587d4d4 100644 --- a/server/src/domain/media/media.service.spec.ts +++ b/server/src/domain/media/media.service.spec.ts @@ -1,4 +1,4 @@ -import { AssetType, SystemConfigKey, TranscodePolicy, VideoCodec } from '@app/infra/entities'; +import { AssetType, SystemConfigKey, TranscodeHWAccel, TranscodePolicy, VideoCodec } from '@app/infra/entities'; import { assetStub, newAssetRepositoryMock, @@ -272,6 +272,7 @@ describe(MediaService.name, () => { '-acodec aac', '-movflags faststart', '-fps_mode passthrough', + '-v verbose', '-preset ultrafast', '-crf 23', ], @@ -309,6 +310,7 @@ describe(MediaService.name, () => { '-acodec aac', '-movflags faststart', '-fps_mode passthrough', + '-v verbose', '-preset ultrafast', '-crf 23', ], @@ -331,6 +333,7 @@ describe(MediaService.name, () => { '-acodec aac', '-movflags faststart', '-fps_mode passthrough', + '-v verbose', '-vf scale=-2:720', '-preset ultrafast', '-crf 23', @@ -357,6 +360,7 @@ describe(MediaService.name, () => { '-acodec aac', '-movflags faststart', '-fps_mode passthrough', + '-v verbose', '-preset ultrafast', '-crf 23', ], @@ -380,6 +384,7 @@ describe(MediaService.name, () => { '-acodec aac', '-movflags faststart', '-fps_mode passthrough', + '-v verbose', '-vf scale=720:-2', '-preset ultrafast', '-crf 23', @@ -404,6 +409,7 @@ describe(MediaService.name, () => { '-acodec aac', '-movflags faststart', '-fps_mode passthrough', + '-v verbose', '-vf scale=-2:720', '-preset ultrafast', '-crf 23', @@ -428,6 +434,7 @@ describe(MediaService.name, () => { '-acodec aac', '-movflags faststart', '-fps_mode passthrough', + '-v verbose', '-vf scale=-2:720', '-preset ultrafast', '-crf 23', @@ -476,6 +483,7 @@ describe(MediaService.name, () => { '-acodec aac', '-movflags faststart', '-fps_mode passthrough', + '-v verbose', '-vf scale=-2:720', '-preset ultrafast', '-crf 23', @@ -505,6 +513,7 @@ describe(MediaService.name, () => { '-acodec aac', '-movflags faststart', '-fps_mode passthrough', + '-v verbose', '-vf scale=-2:720', '-preset ultrafast', '-b:v 3104k', @@ -531,6 +540,7 @@ describe(MediaService.name, () => { '-acodec aac', '-movflags faststart', '-fps_mode passthrough', + '-v verbose', '-vf scale=-2:720', '-preset ultrafast', '-crf 23', @@ -559,6 +569,7 @@ describe(MediaService.name, () => { '-acodec aac', '-movflags faststart', '-fps_mode passthrough', + '-v verbose', '-vf scale=-2:720', '-cpu-used 5', '-row-mt 1', @@ -589,6 +600,7 @@ describe(MediaService.name, () => { '-acodec aac', '-movflags faststart', '-fps_mode passthrough', + '-v verbose', '-vf scale=-2:720', '-cpu-used 2', '-row-mt 1', @@ -618,6 +630,7 @@ describe(MediaService.name, () => { '-acodec aac', '-movflags faststart', '-fps_mode passthrough', + '-v verbose', '-vf scale=-2:720', '-row-mt 1', '-crf 23', @@ -646,6 +659,7 @@ describe(MediaService.name, () => { '-acodec aac', '-movflags faststart', '-fps_mode passthrough', + '-v verbose', '-vf scale=-2:720', '-cpu-used 5', '-row-mt 1', @@ -673,6 +687,7 @@ describe(MediaService.name, () => { '-acodec aac', '-movflags faststart', '-fps_mode passthrough', + '-v verbose', '-vf scale=-2:720', '-preset ultrafast', '-threads 2', @@ -700,6 +715,7 @@ describe(MediaService.name, () => { '-acodec aac', '-movflags faststart', '-fps_mode passthrough', + '-v verbose', '-vf scale=-2:720', '-preset ultrafast', '-crf 23', @@ -727,6 +743,7 @@ describe(MediaService.name, () => { '-acodec aac', '-movflags faststart', '-fps_mode passthrough', + '-v verbose', '-vf scale=-2:720', '-preset ultrafast', '-threads 2', @@ -757,6 +774,7 @@ describe(MediaService.name, () => { '-acodec aac', '-movflags faststart', '-fps_mode passthrough', + '-v verbose', '-vf scale=-2:720', '-preset ultrafast', '-crf 23', @@ -765,5 +783,508 @@ describe(MediaService.name, () => { }, ); }); + + it('should skip transcoding for audioless videos with optimal policy if video codec is correct', async () => { + mediaMock.probe.mockResolvedValue(probeStub.noAudioStreams); + configMock.load.mockResolvedValue([ + { key: SystemConfigKey.FFMPEG_TARGET_VIDEO_CODEC, value: VideoCodec.HEVC }, + { key: SystemConfigKey.FFMPEG_TRANSCODE, value: TranscodePolicy.OPTIMAL }, + { key: SystemConfigKey.FFMPEG_TARGET_RESOLUTION, value: '1080p' }, + ]); + assetMock.getByIds.mockResolvedValue([assetStub.video]); + await sut.handleVideoConversion({ id: assetStub.video.id }); + expect(mediaMock.transcode).not.toHaveBeenCalled(); + }); + + it('should return false if hwaccel is enabled for an unsupported codec', async () => { + mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer); + configMock.load.mockResolvedValue([ + { key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.NVENC }, + { key: SystemConfigKey.FFMPEG_TARGET_VIDEO_CODEC, value: VideoCodec.VP9 }, + ]); + assetMock.getByIds.mockResolvedValue([assetStub.video]); + await expect(sut.handleVideoConversion({ id: assetStub.video.id })).resolves.toEqual(false); + expect(mediaMock.transcode).not.toHaveBeenCalled(); + }); + + it('should return false if hwaccel option is invalid', async () => { + mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer); + configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_ACCEL, value: 'invalid' }]); + assetMock.getByIds.mockResolvedValue([assetStub.video]); + await expect(sut.handleVideoConversion({ id: assetStub.video.id })).resolves.toEqual(false); + expect(mediaMock.transcode).not.toHaveBeenCalled(); + }); + + it('should set two pass options for nvenc when enabled', async () => { + mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer); + configMock.load.mockResolvedValue([ + { key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.NVENC }, + { key: SystemConfigKey.FFMPEG_MAX_BITRATE, value: '10000k' }, + { key: SystemConfigKey.FFMPEG_TWO_PASS, value: true }, + ]); + assetMock.getByIds.mockResolvedValue([assetStub.video]); + await sut.handleVideoConversion({ id: assetStub.video.id }); + expect(mediaMock.transcode).toHaveBeenCalledWith( + '/original/path.ext', + 'upload/encoded-video/user-id/asset-id.mp4', + { + inputOptions: ['-init_hw_device cuda=cuda:0', '-filter_hw_device cuda'], + outputOptions: [ + `-vcodec h264_nvenc`, + '-tune hq', + '-qmin 0', + '-g 250', + '-bf 3', + '-b_ref_mode middle', + '-temporal-aq 1', + '-rc-lookahead 20', + '-i_qfactor 0.75', + '-b_qfactor 1.1', + '-acodec aac', + '-movflags faststart', + '-fps_mode passthrough', + '-v verbose', + '-vf hwupload_cuda,scale_cuda=-2:720', + '-preset p1', + '-b:v 6897k', + '-maxrate 10000k', + '-bufsize 6897k', + '-multipass 2', + ], + twoPass: false, + }, + ); + }); + + it('should set vbr options for nvenc when max bitrate is enabled', async () => { + mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer); + configMock.load.mockResolvedValue([ + { key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.NVENC }, + { key: SystemConfigKey.FFMPEG_MAX_BITRATE, value: '10000k' }, + ]); + assetMock.getByIds.mockResolvedValue([assetStub.video]); + await sut.handleVideoConversion({ id: assetStub.video.id }); + expect(mediaMock.transcode).toHaveBeenCalledWith( + '/original/path.ext', + 'upload/encoded-video/user-id/asset-id.mp4', + { + inputOptions: ['-init_hw_device cuda=cuda:0', '-filter_hw_device cuda'], + outputOptions: [ + `-vcodec h264_nvenc`, + '-tune hq', + '-qmin 0', + '-g 250', + '-bf 3', + '-b_ref_mode middle', + '-temporal-aq 1', + '-rc-lookahead 20', + '-i_qfactor 0.75', + '-b_qfactor 1.1', + '-acodec aac', + '-movflags faststart', + '-fps_mode passthrough', + '-v verbose', + '-vf hwupload_cuda,scale_cuda=-2:720', + '-preset p1', + '-cq:v 23', + '-maxrate 10000k', + '-bufsize 6897k', + ], + twoPass: false, + }, + ); + }); + + it('should set cq options for nvenc when max bitrate is disabled', async () => { + mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer); + configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.NVENC }]); + assetMock.getByIds.mockResolvedValue([assetStub.video]); + await sut.handleVideoConversion({ id: assetStub.video.id }); + expect(mediaMock.transcode).toHaveBeenCalledWith( + '/original/path.ext', + 'upload/encoded-video/user-id/asset-id.mp4', + { + inputOptions: ['-init_hw_device cuda=cuda:0', '-filter_hw_device cuda'], + outputOptions: [ + `-vcodec h264_nvenc`, + '-tune hq', + '-qmin 0', + '-g 250', + '-bf 3', + '-b_ref_mode middle', + '-temporal-aq 1', + '-rc-lookahead 20', + '-i_qfactor 0.75', + '-b_qfactor 1.1', + '-acodec aac', + '-movflags faststart', + '-fps_mode passthrough', + '-v verbose', + '-vf hwupload_cuda,scale_cuda=-2:720', + '-preset p1', + '-cq:v 23', + ], + twoPass: false, + }, + ); + }); + + it('should omit preset for nvenc if invalid', async () => { + mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer); + configMock.load.mockResolvedValue([ + { key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.NVENC }, + { key: SystemConfigKey.FFMPEG_PRESET, value: 'invalid' }, + ]); + assetMock.getByIds.mockResolvedValue([assetStub.video]); + await sut.handleVideoConversion({ id: assetStub.video.id }); + expect(mediaMock.transcode).toHaveBeenCalledWith( + '/original/path.ext', + 'upload/encoded-video/user-id/asset-id.mp4', + { + inputOptions: ['-init_hw_device cuda=cuda:0', '-filter_hw_device cuda'], + outputOptions: [ + `-vcodec h264_nvenc`, + '-tune hq', + '-qmin 0', + '-g 250', + '-bf 3', + '-b_ref_mode middle', + '-temporal-aq 1', + '-rc-lookahead 20', + '-i_qfactor 0.75', + '-b_qfactor 1.1', + '-acodec aac', + '-movflags faststart', + '-fps_mode passthrough', + '-v verbose', + '-vf hwupload_cuda,scale_cuda=-2:720', + '-cq:v 23', + ], + twoPass: false, + }, + ); + }); + + it('should ignore two pass for nvenc if max bitrate is disabled', async () => { + mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer); + configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.NVENC }]); + assetMock.getByIds.mockResolvedValue([assetStub.video]); + await sut.handleVideoConversion({ id: assetStub.video.id }); + expect(mediaMock.transcode).toHaveBeenCalledWith( + '/original/path.ext', + 'upload/encoded-video/user-id/asset-id.mp4', + { + inputOptions: ['-init_hw_device cuda=cuda:0', '-filter_hw_device cuda'], + outputOptions: [ + `-vcodec h264_nvenc`, + '-tune hq', + '-qmin 0', + '-g 250', + '-bf 3', + '-b_ref_mode middle', + '-temporal-aq 1', + '-rc-lookahead 20', + '-i_qfactor 0.75', + '-b_qfactor 1.1', + '-acodec aac', + '-movflags faststart', + '-fps_mode passthrough', + '-v verbose', + '-vf hwupload_cuda,scale_cuda=-2:720', + '-preset p1', + '-cq:v 23', + ], + twoPass: false, + }, + ); + }); + + it('should set options for qsv', async () => { + storageMock.readdir.mockResolvedValue(['renderD128']); + mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer); + configMock.load.mockResolvedValue([ + { key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.QSV }, + { key: SystemConfigKey.FFMPEG_MAX_BITRATE, value: '10000k' }, + ]); + assetMock.getByIds.mockResolvedValue([assetStub.video]); + await sut.handleVideoConversion({ id: assetStub.video.id }); + expect(mediaMock.transcode).toHaveBeenCalledWith( + '/original/path.ext', + 'upload/encoded-video/user-id/asset-id.mp4', + { + inputOptions: ['-init_hw_device qsv=hw', '-filter_hw_device hw'], + outputOptions: [ + `-vcodec h264_qsv`, + '-g 256', + '-extbrc 1', + '-refs 5', + '-bf 7', + '-acodec aac', + '-movflags faststart', + '-fps_mode passthrough', + '-v verbose', + '-vf format=nv12,hwupload=extra_hw_frames=64,scale_qsv=-1:720', + '-preset 7', + '-global_quality 23', + '-maxrate 10000k', + '-bufsize 20000k', + ], + twoPass: false, + }, + ); + }); + + it('should omit preset for qsv if invalid', async () => { + storageMock.readdir.mockResolvedValue(['renderD128']); + mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer); + configMock.load.mockResolvedValue([ + { key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.QSV }, + { key: SystemConfigKey.FFMPEG_PRESET, value: 'invalid' }, + ]); + assetMock.getByIds.mockResolvedValue([assetStub.video]); + await sut.handleVideoConversion({ id: assetStub.video.id }); + expect(mediaMock.transcode).toHaveBeenCalledWith( + '/original/path.ext', + 'upload/encoded-video/user-id/asset-id.mp4', + { + inputOptions: ['-init_hw_device qsv=hw', '-filter_hw_device hw'], + outputOptions: [ + `-vcodec h264_qsv`, + '-g 256', + '-extbrc 1', + '-refs 5', + '-bf 7', + '-acodec aac', + '-movflags faststart', + '-fps_mode passthrough', + '-v verbose', + '-vf format=nv12,hwupload=extra_hw_frames=64,scale_qsv=-1:720', + '-global_quality 23', + ], + twoPass: false, + }, + ); + }); + + it('should set low power mode for qsv if target video codec is vp9', async () => { + storageMock.readdir.mockResolvedValue(['renderD128']); + mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer); + configMock.load.mockResolvedValue([ + { key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.QSV }, + { key: SystemConfigKey.FFMPEG_TARGET_VIDEO_CODEC, value: VideoCodec.VP9 }, + ]); + assetMock.getByIds.mockResolvedValue([assetStub.video]); + await sut.handleVideoConversion({ id: assetStub.video.id }); + expect(mediaMock.transcode).toHaveBeenCalledWith( + '/original/path.ext', + 'upload/encoded-video/user-id/asset-id.mp4', + { + inputOptions: ['-init_hw_device qsv=hw', '-filter_hw_device hw'], + outputOptions: [ + `-vcodec vp9_qsv`, + '-g 256', + '-extbrc 1', + '-refs 5', + '-bf 7', + '-low_power 1', + '-acodec aac', + '-movflags faststart', + '-fps_mode passthrough', + '-v verbose', + '-vf format=nv12,hwupload=extra_hw_frames=64,scale_qsv=-1:720', + '-preset 7', + '-q:v 23', + ], + twoPass: false, + }, + ); + }); + + it('should return false for qsv if no hw devices', async () => { + storageMock.readdir.mockResolvedValue([]); + mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer); + configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.QSV }]); + assetMock.getByIds.mockResolvedValue([assetStub.video]); + await expect(sut.handleVideoConversion({ id: assetStub.video.id })).resolves.toEqual(false); + expect(mediaMock.transcode).not.toHaveBeenCalled(); + }); + + it('should set vbr options for vaapi when max bitrate is enabled', async () => { + storageMock.readdir.mockResolvedValue(['renderD128']); + mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer); + configMock.load.mockResolvedValue([ + { key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.VAAPI }, + { key: SystemConfigKey.FFMPEG_MAX_BITRATE, value: '10000k' }, + ]); + assetMock.getByIds.mockResolvedValue([assetStub.video]); + await sut.handleVideoConversion({ id: assetStub.video.id }); + expect(mediaMock.transcode).toHaveBeenCalledWith( + '/original/path.ext', + 'upload/encoded-video/user-id/asset-id.mp4', + { + inputOptions: ['-init_hw_device vaapi=accel:/dev/dri/renderD128', '-filter_hw_device accel'], + outputOptions: [ + `-vcodec h264_vaapi`, + '-acodec aac', + '-movflags faststart', + '-fps_mode passthrough', + '-v verbose', + '-vf format=nv12,hwupload,scale_vaapi=-2:720', + '-compression_level 7', + '-b:v 6897k', + '-maxrate 10000k', + '-minrate 3448.5k', + '-rc_mode 3', + ], + twoPass: false, + }, + ); + }); + + it('should set cq options for vaapi when max bitrate is disabled', async () => { + storageMock.readdir.mockResolvedValue(['renderD128']); + mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer); + configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.VAAPI }]); + assetMock.getByIds.mockResolvedValue([assetStub.video]); + await sut.handleVideoConversion({ id: assetStub.video.id }); + expect(mediaMock.transcode).toHaveBeenCalledWith( + '/original/path.ext', + 'upload/encoded-video/user-id/asset-id.mp4', + { + inputOptions: ['-init_hw_device vaapi=accel:/dev/dri/renderD128', '-filter_hw_device accel'], + outputOptions: [ + `-vcodec h264_vaapi`, + '-acodec aac', + '-movflags faststart', + '-fps_mode passthrough', + '-v verbose', + '-vf format=nv12,hwupload,scale_vaapi=-2:720', + '-compression_level 7', + '-qp 23', + '-global_quality 23', + '-rc_mode 1', + ], + twoPass: false, + }, + ); + }); + + it('should omit preset for vaapi if invalid', async () => { + storageMock.readdir.mockResolvedValue(['renderD128']); + mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer); + configMock.load.mockResolvedValue([ + { key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.VAAPI }, + { key: SystemConfigKey.FFMPEG_PRESET, value: 'invalid' }, + ]); + assetMock.getByIds.mockResolvedValue([assetStub.video]); + await sut.handleVideoConversion({ id: assetStub.video.id }); + expect(mediaMock.transcode).toHaveBeenCalledWith( + '/original/path.ext', + 'upload/encoded-video/user-id/asset-id.mp4', + { + inputOptions: ['-init_hw_device vaapi=accel:/dev/dri/renderD128', '-filter_hw_device accel'], + outputOptions: [ + `-vcodec h264_vaapi`, + '-acodec aac', + '-movflags faststart', + '-fps_mode passthrough', + '-v verbose', + '-vf format=nv12,hwupload,scale_vaapi=-2:720', + '-qp 23', + '-global_quality 23', + '-rc_mode 1', + ], + twoPass: false, + }, + ); + }); + + it('should prefer gpu for vaapi if available', async () => { + storageMock.readdir.mockResolvedValue(['renderD129', 'card1', 'card0', 'renderD128']); + mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer); + configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.VAAPI }]); + assetMock.getByIds.mockResolvedValue([assetStub.video]); + await sut.handleVideoConversion({ id: assetStub.video.id }); + expect(mediaMock.transcode).toHaveBeenCalledWith( + '/original/path.ext', + 'upload/encoded-video/user-id/asset-id.mp4', + { + inputOptions: ['-init_hw_device vaapi=accel:/dev/dri/card1', '-filter_hw_device accel'], + outputOptions: [ + `-vcodec h264_vaapi`, + '-acodec aac', + '-movflags faststart', + '-fps_mode passthrough', + '-v verbose', + '-vf format=nv12,hwupload,scale_vaapi=-2:720', + '-compression_level 7', + '-qp 23', + '-global_quality 23', + '-rc_mode 1', + ], + twoPass: false, + }, + ); + + storageMock.readdir.mockResolvedValue(['renderD129', 'renderD128']); + await sut.handleVideoConversion({ id: assetStub.video.id }); + expect(mediaMock.transcode).toHaveBeenCalledWith( + '/original/path.ext', + 'upload/encoded-video/user-id/asset-id.mp4', + { + inputOptions: ['-init_hw_device vaapi=accel:/dev/dri/renderD129', '-filter_hw_device accel'], + outputOptions: [ + `-vcodec h264_vaapi`, + '-acodec aac', + '-movflags faststart', + '-fps_mode passthrough', + '-v verbose', + '-vf format=nv12,hwupload,scale_vaapi=-2:720', + '-compression_level 7', + '-qp 23', + '-global_quality 23', + '-rc_mode 1', + ], + twoPass: false, + }, + ); + }); + + it('should fallback to sw transcoding if hw transcoding fails', async () => { + storageMock.readdir.mockResolvedValue(['renderD128']); + mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer); + configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.VAAPI }]); + assetMock.getByIds.mockResolvedValue([assetStub.video]); + mediaMock.transcode.mockRejectedValueOnce(new Error('error')); + await sut.handleVideoConversion({ id: assetStub.video.id }); + expect(mediaMock.transcode).toHaveBeenCalledTimes(2); + expect(mediaMock.transcode).toHaveBeenLastCalledWith( + '/original/path.ext', + 'upload/encoded-video/user-id/asset-id.mp4', + { + inputOptions: [], + outputOptions: [ + '-vcodec h264', + '-acodec aac', + '-movflags faststart', + '-fps_mode passthrough', + '-v verbose', + '-vf scale=-2:720', + '-preset ultrafast', + '-crf 23', + ], + twoPass: false, + }, + ); + }); + + it('should return false for vaapi if no hw devices', async () => { + storageMock.readdir.mockResolvedValue([]); + mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer); + configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.VAAPI }]); + assetMock.getByIds.mockResolvedValue([assetStub.video]); + await expect(sut.handleVideoConversion({ id: assetStub.video.id })).resolves.toEqual(false); + expect(mediaMock.transcode).not.toHaveBeenCalled(); + }); }); }); diff --git a/server/src/domain/media/media.service.ts b/server/src/domain/media/media.service.ts index 98800d5fcf942..54ba4b8b2c2c5 100644 --- a/server/src/domain/media/media.service.ts +++ b/server/src/domain/media/media.service.ts @@ -1,4 +1,4 @@ -import { AssetEntity, AssetType, TranscodePolicy, VideoCodec } from '@app/infra/entities'; +import { AssetEntity, AssetType, TranscodeHWAccel, TranscodePolicy, VideoCodec } from '@app/infra/entities'; import { Inject, Injectable, Logger, UnsupportedMediaTypeException } from '@nestjs/common'; import { join } from 'path'; import { IAssetRepository, WithoutProperty } from '../asset'; @@ -8,8 +8,8 @@ import { IStorageRepository, StorageCore, StorageFolder } from '../storage'; import { ISystemConfigRepository, SystemConfigFFmpegDto } from '../system-config'; import { SystemConfigCore } from '../system-config/system-config.core'; import { JPEG_THUMBNAIL_SIZE, WEBP_THUMBNAIL_SIZE } from './media.constant'; -import { AudioStreamInfo, IMediaRepository, VideoStreamInfo } from './media.repository'; -import { H264Config, HEVCConfig, VP9Config } from './media.util'; +import { AudioStreamInfo, IMediaRepository, VideoCodecHWConfig, VideoStreamInfo } from './media.repository'; +import { H264Config, HEVCConfig, NVENCConfig, QSVConfig, VAAPIConfig, VP9Config } from './media.util'; @Injectable() export class MediaService { @@ -155,14 +155,26 @@ export class MediaService { let transcodeOptions; try { - transcodeOptions = this.getCodecConfig(config).getOptions(mainVideoStream); + transcodeOptions = await this.getCodecConfig(config).then((c) => c.getOptions(mainVideoStream)); } catch (err) { this.logger.error(`An error occurred while configuring transcoding options: ${err}`); return false; } this.logger.log(`Start encoding video ${asset.id} ${JSON.stringify(transcodeOptions)}`); - await this.mediaRepository.transcode(input, output, transcodeOptions); + try { + await this.mediaRepository.transcode(input, output, transcodeOptions); + } catch (err) { + this.logger.error(err); + if (config.accel && config.accel !== TranscodeHWAccel.DISABLED) { + this.logger.error( + `Error occurred during transcoding. Retrying with ${config.accel.toUpperCase()} acceleration disabled.`, + ); + } + config.accel = TranscodeHWAccel.DISABLED; + transcodeOptions = await this.getCodecConfig(config).then((c) => c.getOptions(mainVideoStream)); + await this.mediaRepository.transcode(input, output, transcodeOptions); + } this.logger.log(`Encoding success ${asset.id}`); @@ -195,15 +207,11 @@ export class MediaService { const isTargetContainer = ['mov,mp4,m4a,3gp,3g2,mj2', 'mp4', 'mov'].includes(containerExtension); const isTargetAudioCodec = audioStream == null || audioStream.codecName === ffmpegConfig.targetAudioCodec; - if (audioStream != null) { - this.logger.verbose( - `${asset.id}: AudioCodecName ${audioStream.codecName}, AudioStreamCodecType ${audioStream.codecType}, containerExtension ${containerExtension}`, - ); - } else { - this.logger.verbose( - `${asset.id}: AudioCodecName None, AudioStreamCodecType None, containerExtension ${containerExtension}`, - ); - } + this.logger.verbose( + `${asset.id}: AudioCodecName ${audioStream?.codecName ?? 'None'}, AudioStreamCodecType ${ + audioStream?.codecType ?? 'None' + }, containerExtension ${containerExtension}`, + ); const allTargetsMatching = isTargetVideoCodec && isTargetAudioCodec && isTargetContainer; const scalingEnabled = ffmpegConfig.targetResolution !== 'original'; @@ -228,7 +236,14 @@ export class MediaService { } } - private getCodecConfig(config: SystemConfigFFmpegDto) { + async getCodecConfig(config: SystemConfigFFmpegDto) { + if (config.accel === TranscodeHWAccel.DISABLED) { + return this.getSWCodecConfig(config); + } + return this.getHWCodecConfig(config); + } + + private getSWCodecConfig(config: SystemConfigFFmpegDto) { switch (config.targetVideoCodec) { case VideoCodec.H264: return new H264Config(config); @@ -240,4 +255,31 @@ export class MediaService { throw new UnsupportedMediaTypeException(`Codec '${config.targetVideoCodec}' is unsupported`); } } + + private async getHWCodecConfig(config: SystemConfigFFmpegDto) { + let handler: VideoCodecHWConfig; + let devices: string[]; + switch (config.accel) { + case TranscodeHWAccel.NVENC: + handler = new NVENCConfig(config); + break; + case TranscodeHWAccel.QSV: + devices = await this.storageRepository.readdir('/dev/dri'); + handler = new QSVConfig(config, devices); + break; + case TranscodeHWAccel.VAAPI: + devices = await this.storageRepository.readdir('/dev/dri'); + handler = new VAAPIConfig(config, devices); + break; + default: + throw new UnsupportedMediaTypeException(`${config.accel.toUpperCase()} acceleration is unsupported`); + } + if (!handler.getSupportedCodecs().includes(config.targetVideoCodec)) { + throw new UnsupportedMediaTypeException( + `${config.accel.toUpperCase()} acceleration does not support codec '${config.targetVideoCodec.toUpperCase()}'. Supported codecs: ${handler.getSupportedCodecs()}`, + ); + } + + return handler; + } } diff --git a/server/src/domain/media/media.util.ts b/server/src/domain/media/media.util.ts index bee22e9e65c30..17c88511d4a81 100644 --- a/server/src/domain/media/media.util.ts +++ b/server/src/domain/media/media.util.ts @@ -1,13 +1,26 @@ +import { TranscodeHWAccel, VideoCodec } from '@app/infra/entities'; import { SystemConfigFFmpegDto } from '../system-config/dto'; -import { BitrateDistribution, TranscodeOptions, VideoCodecSWConfig, VideoStreamInfo } from './media.repository'; - +import { + BitrateDistribution, + TranscodeOptions, + VideoCodecHWConfig, + VideoCodecSWConfig, + VideoStreamInfo, +} from './media.repository'; class BaseConfig implements VideoCodecSWConfig { constructor(protected config: SystemConfigFFmpegDto) {} getOptions(stream: VideoStreamInfo) { const options = { inputOptions: this.getBaseInputOptions(), - outputOptions: this.getBaseOutputOptions(), + outputOptions: this.getBaseOutputOptions().concat([ + `-acodec ${this.config.targetAudioCodec}`, + // Makes a second pass moving the moov atom to the + // beginning of the file for improved playback speed. + '-movflags faststart', + '-fps_mode passthrough', + '-v verbose', + ]), twoPass: this.eligibleForTwoPass(), } as TranscodeOptions; const filters = this.getFilterOptions(stream); @@ -26,14 +39,7 @@ class BaseConfig implements VideoCodecSWConfig { } getBaseOutputOptions() { - return [ - `-vcodec ${this.config.targetVideoCodec}`, - `-acodec ${this.config.targetAudioCodec}`, - // Makes a second pass moving the moov atom to the beginning of - // the file for improved playback speed. - '-movflags faststart', - '-fps_mode passthrough', - ]; + return [`-vcodec ${this.config.targetVideoCodec}`]; } getFilterOptions(stream: VideoStreamInfo) { @@ -77,11 +83,11 @@ class BaseConfig implements VideoCodecSWConfig { } eligibleForTwoPass() { - if (!this.config.twoPass) { + if (!this.config.twoPass || this.config.accel !== TranscodeHWAccel.DISABLED) { return false; } - return this.isBitrateConstrained() || this.config.targetVideoCodec === 'vp9'; + return this.isBitrateConstrained() || this.config.targetVideoCodec === VideoCodec.VP9; } getBitrateDistribution() { @@ -107,7 +113,8 @@ class BaseConfig implements VideoCodecSWConfig { getScaling(stream: VideoStreamInfo) { const targetResolution = this.getTargetResolution(stream); - return this.isVideoVertical(stream) ? `${targetResolution}:-2` : `-2:${targetResolution}`; + const mult = this.config.accel === TranscodeHWAccel.QSV ? 1 : 2; // QSV doesn't support scaling numbers below -1 + return this.isVideoVertical(stream) ? `${targetResolution}:-${mult}` : `-${mult}:${targetResolution}`; } isVideoRotated(stream: VideoStreamInfo) { @@ -137,6 +144,34 @@ class BaseConfig implements VideoCodecSWConfig { } } +export class BaseHWConfig extends BaseConfig implements VideoCodecHWConfig { + protected devices: string[]; + + constructor(protected config: SystemConfigFFmpegDto, devices: string[] = []) { + super(config); + this.devices = this.validateDevices(devices); + } + + getSupportedCodecs() { + return [VideoCodec.H264, VideoCodec.HEVC, VideoCodec.VP9]; + } + + validateDevices(devices: string[]) { + return devices + .filter((device) => device.startsWith('renderD') || device.startsWith('card')) + .sort((a, b) => { + // order GPU devices first + if (a.startsWith('card') && b.startsWith('renderD')) { + return -1; + } + if (a.startsWith('renderD') && b.startsWith('card')) { + return 1; + } + return -a.localeCompare(b); + }); + } +} + export class H264Config extends BaseConfig { getThreadOptions() { if (this.config.threads <= 0) { @@ -189,3 +224,168 @@ export class VP9Config extends BaseConfig { return ['-row-mt 1', ...super.getThreadOptions()]; } } + +export class NVENCConfig extends BaseHWConfig { + getSupportedCodecs() { + return [VideoCodec.H264, VideoCodec.HEVC]; + } + + getBaseInputOptions() { + return ['-init_hw_device cuda=cuda:0', '-filter_hw_device cuda']; + } + + getBaseOutputOptions() { + return [ + `-vcodec ${this.config.targetVideoCodec}_nvenc`, + // below settings recommended from https://docs.nvidia.com/video-technologies/video-codec-sdk/12.0/ffmpeg-with-nvidia-gpu/index.html#command-line-for-latency-tolerant-high-quality-transcoding + '-tune hq', + '-qmin 0', + '-g 250', + '-bf 3', + '-b_ref_mode middle', + '-temporal-aq 1', + '-rc-lookahead 20', + '-i_qfactor 0.75', + '-b_qfactor 1.1', + ]; + } + + getFilterOptions(stream: VideoStreamInfo) { + const options = ['hwupload_cuda']; + if (this.shouldScale(stream)) { + options.push(`scale_cuda=${this.getScaling(stream)}`); + } + + return options; + } + + getPresetOptions() { + let presetIndex = this.getPresetIndex(); + if (presetIndex < 0) { + return []; + } + presetIndex = 7 - Math.min(6, presetIndex); // map to p1-p7; p7 is the highest quality, so reverse index + return [`-preset p${presetIndex}`]; + } + + getBitrateOptions() { + const bitrates = this.getBitrateDistribution(); + if (bitrates.max > 0 && this.config.twoPass) { + return [ + `-b:v ${bitrates.target}${bitrates.unit}`, + `-maxrate ${bitrates.max}${bitrates.unit}`, + `-bufsize ${bitrates.target}${bitrates.unit}`, + '-multipass 2', + ]; + } else if (bitrates.max > 0) { + return [ + `-cq:v ${this.config.crf}`, + `-maxrate ${bitrates.max}${bitrates.unit}`, + `-bufsize ${bitrates.target}${bitrates.unit}`, + ]; + } else { + return [`-cq:v ${this.config.crf}`]; + } + } + + getThreadOptions() { + return []; + } +} + +export class QSVConfig extends BaseHWConfig { + getBaseInputOptions() { + if (!this.devices.length) { + throw Error('No QSV device found'); + } + return ['-init_hw_device qsv=hw', '-filter_hw_device hw']; + } + + getBaseOutputOptions() { + // recommended from https://github.com/intel/media-delivery/blob/master/doc/benchmarks/intel-iris-xe-max-graphics/intel-iris-xe-max-graphics.md + const options = [`-vcodec ${this.config.targetVideoCodec}_qsv`, '-g 256', '-extbrc 1', '-refs 5', '-bf 7']; + // VP9 requires enabling low power mode https://git.ffmpeg.org/gitweb/ffmpeg.git/commit/33583803e107b6d532def0f9d949364b01b6ad5a + if (this.config.targetVideoCodec === VideoCodec.VP9) { + options.push('-low_power 1'); + } + return options; + } + + getFilterOptions(stream: VideoStreamInfo) { + const options = ['format=nv12', 'hwupload=extra_hw_frames=64']; + if (this.shouldScale(stream)) { + options.push(`scale_qsv=${this.getScaling(stream)}`); + } + return options; + } + + getPresetOptions() { + let presetIndex = this.getPresetIndex(); + if (presetIndex < 0) { + return []; + } + presetIndex = Math.min(6, presetIndex) + 1; // 1 to 7 + return [`-preset ${presetIndex}`]; + } + + getBitrateOptions() { + const options = []; + if (this.config.targetVideoCodec !== VideoCodec.VP9) { + options.push(`-global_quality ${this.config.crf}`); + } else { + options.push(`-q:v ${this.config.crf}`); + } + const bitrates = this.getBitrateDistribution(); + if (bitrates.max > 0) { + options.push(`-maxrate ${bitrates.max}${bitrates.unit}`); + options.push(`-bufsize ${bitrates.max * 2}${bitrates.unit}`); + } + return options; + } +} + +export class VAAPIConfig extends BaseHWConfig { + getBaseInputOptions() { + if (this.devices.length === 0) { + throw Error('No VAAPI device found'); + } + return [`-init_hw_device vaapi=accel:/dev/dri/${this.devices[0]}`, '-filter_hw_device accel']; + } + + getBaseOutputOptions() { + return [`-vcodec ${this.config.targetVideoCodec}_vaapi`]; + } + + getFilterOptions(stream: VideoStreamInfo) { + const options = ['format=nv12', 'hwupload']; + if (this.shouldScale(stream)) { + options.push(`scale_vaapi=${this.getScaling(stream)}`); + } + + return options; + } + + getPresetOptions() { + let presetIndex = this.getPresetIndex(); + if (presetIndex < 0) { + return []; + } + presetIndex = Math.min(6, presetIndex) + 1; // 1 to 7 + return [`-compression_level ${presetIndex}`]; + } + + getBitrateOptions() { + const bitrates = this.getBitrateDistribution(); + // VAAPI doesn't allow setting both quality and max bitrate + if (bitrates.max > 0) { + return [ + `-b:v ${bitrates.target}${bitrates.unit}`, + `-maxrate ${bitrates.max}${bitrates.unit}`, + `-minrate ${bitrates.min}${bitrates.unit}`, + '-rc_mode 3', + ]; // variable bitrate + } else { + return [`-qp ${this.config.crf}`, `-global_quality ${this.config.crf}`, '-rc_mode 1']; // constant quality + } + } +} diff --git a/server/src/domain/storage/storage.repository.ts b/server/src/domain/storage/storage.repository.ts index 7d312c0752056..62b78094b7943 100644 --- a/server/src/domain/storage/storage.repository.ts +++ b/server/src/domain/storage/storage.repository.ts @@ -29,4 +29,5 @@ export interface IStorageRepository { checkFileExists(filepath: string, mode?: number): Promise; mkdirSync(filepath: string): void; checkDiskUsage(folder: string): Promise; + readdir(folder: string): Promise; } diff --git a/server/src/domain/system-config/dto/system-config-ffmpeg.dto.ts b/server/src/domain/system-config/dto/system-config-ffmpeg.dto.ts index 01f9f9ca7f305..579c72acb1f41 100644 --- a/server/src/domain/system-config/dto/system-config-ffmpeg.dto.ts +++ b/server/src/domain/system-config/dto/system-config-ffmpeg.dto.ts @@ -1,4 +1,4 @@ -import { AudioCodec, TranscodePolicy, VideoCodec } from '@app/infra/entities'; +import { AudioCodec, TranscodeHWAccel, TranscodePolicy, VideoCodec } from '@app/infra/entities'; import { ApiProperty } from '@nestjs/swagger'; import { Type } from 'class-transformer'; import { IsBoolean, IsEnum, IsInt, IsString, Max, Min } from 'class-validator'; @@ -40,4 +40,8 @@ export class SystemConfigFFmpegDto { @IsEnum(TranscodePolicy) @ApiProperty({ enumName: 'TranscodePolicy', enum: TranscodePolicy }) transcode!: TranscodePolicy; + + @IsEnum(TranscodeHWAccel) + @ApiProperty({ enumName: 'TranscodeHWAccel', enum: TranscodeHWAccel }) + accel!: TranscodeHWAccel; } diff --git a/server/src/domain/system-config/system-config.core.ts b/server/src/domain/system-config/system-config.core.ts index 3051b82b3ef6f..bd462deb050cd 100644 --- a/server/src/domain/system-config/system-config.core.ts +++ b/server/src/domain/system-config/system-config.core.ts @@ -4,6 +4,7 @@ import { SystemConfigEntity, SystemConfigKey, SystemConfigValue, + TranscodeHWAccel, TranscodePolicy, VideoCodec, } from '@app/infra/entities'; @@ -27,6 +28,7 @@ export const defaults = Object.freeze({ maxBitrate: '0', twoPass: false, transcode: TranscodePolicy.REQUIRED, + accel: TranscodeHWAccel.DISABLED, }, job: { [QueueName.BACKGROUND_TASK]: { concurrency: 5 }, diff --git a/server/src/domain/system-config/system-config.service.spec.ts b/server/src/domain/system-config/system-config.service.spec.ts index a3296df922825..9e50f416b7abf 100644 --- a/server/src/domain/system-config/system-config.service.spec.ts +++ b/server/src/domain/system-config/system-config.service.spec.ts @@ -3,6 +3,7 @@ import { SystemConfig, SystemConfigEntity, SystemConfigKey, + TranscodeHWAccel, TranscodePolicy, VideoCodec, } from '@app/infra/entities'; @@ -41,6 +42,7 @@ const updatedConfig = Object.freeze({ maxBitrate: '0', twoPass: false, transcode: TranscodePolicy.REQUIRED, + accel: TranscodeHWAccel.DISABLED, }, oauth: { autoLaunch: true, diff --git a/server/src/infra/entities/system-config.entity.ts b/server/src/infra/entities/system-config.entity.ts index af13b47da817a..54a5281ee99fa 100644 --- a/server/src/infra/entities/system-config.entity.ts +++ b/server/src/infra/entities/system-config.entity.ts @@ -23,6 +23,7 @@ export enum SystemConfigKey { FFMPEG_MAX_BITRATE = 'ffmpeg.maxBitrate', FFMPEG_TWO_PASS = 'ffmpeg.twoPass', FFMPEG_TRANSCODE = 'ffmpeg.transcode', + FFMPEG_ACCEL = 'ffmpeg.accel', JOB_THUMBNAIL_GENERATION_CONCURRENCY = 'job.thumbnailGeneration.concurrency', JOB_METADATA_EXTRACTION_CONCURRENCY = 'job.metadataExtraction.concurrency', @@ -71,6 +72,13 @@ export enum AudioCodec { OPUS = 'opus', } +export enum TranscodeHWAccel { + NVENC = 'nvenc', + QSV = 'qsv', + VAAPI = 'vaapi', + DISABLED = 'disabled', +} + export interface SystemConfig { ffmpeg: { crf: number; @@ -82,6 +90,7 @@ export interface SystemConfig { maxBitrate: string; twoPass: boolean; transcode: TranscodePolicy; + accel: TranscodeHWAccel; }; job: Record; oauth: { diff --git a/server/src/infra/repositories/filesystem.provider.ts b/server/src/infra/repositories/filesystem.provider.ts index 62388201bc8ea..8f7ba3438ee44 100644 --- a/server/src/infra/repositories/filesystem.provider.ts +++ b/server/src/infra/repositories/filesystem.provider.ts @@ -1,7 +1,7 @@ import { DiskUsage, ImmichReadStream, ImmichZipStream, IStorageRepository } from '@app/domain'; import archiver from 'archiver'; import { constants, createReadStream, existsSync, mkdirSync } from 'fs'; -import fs from 'fs/promises'; +import fs, { readdir } from 'fs/promises'; import mv from 'mv'; import { promisify } from 'node:util'; import path from 'path'; @@ -92,4 +92,6 @@ export class FilesystemProvider implements IStorageRepository { total: stats.blocks * stats.bsize, }; } + + readdir = readdir; } diff --git a/server/src/infra/repositories/media.repository.ts b/server/src/infra/repositories/media.repository.ts index 4b0345faa4bf3..7ef258366bbd0 100644 --- a/server/src/infra/repositories/media.repository.ts +++ b/server/src/infra/repositories/media.repository.ts @@ -6,6 +6,7 @@ import sharp from 'sharp'; import { promisify } from 'util'; const probe = promisify(ffmpeg.ffprobe); +sharp.concurrency(0); export class MediaRepository implements IMediaRepository { private logger = new Logger(MediaRepository.name); @@ -73,7 +74,7 @@ export class MediaRepository implements IMediaRepository { .map((stream) => ({ height: stream.height || 0, width: stream.width || 0, - codecName: stream.codec_name, + codecName: stream.codec_name === 'h265' ? 'hevc' : stream.codec_name, codecType: stream.codec_type, frameCount: Number.parseInt(stream.nb_frames ?? '0'), rotation: Number.parseInt(`${stream.rotation ?? 0}`), @@ -91,6 +92,7 @@ export class MediaRepository implements IMediaRepository { if (!options.twoPass) { return new Promise((resolve, reject) => { ffmpeg(input, { niceness: 10 }) + .inputOptions(options.inputOptions) .outputOptions(options.outputOptions) .output(output) .on('error', (err, stdout, stderr) => { @@ -106,6 +108,7 @@ export class MediaRepository implements IMediaRepository { // recommended for vp9 for better quality and compression return new Promise((resolve, reject) => { ffmpeg(input, { niceness: 10 }) + .inputOptions(options.inputOptions) .outputOptions(options.outputOptions) .addOptions('-pass', '1') .addOptions('-passlogfile', output) @@ -118,6 +121,7 @@ export class MediaRepository implements IMediaRepository { .on('end', () => { // second pass ffmpeg(input, { niceness: 10 }) + .inputOptions(options.inputOptions) .outputOptions(options.outputOptions) .addOptions('-pass', '2') .addOptions('-passlogfile', output) diff --git a/server/start.sh b/server/start.sh index 617837da70ba2..253dfc56db4d8 100755 --- a/server/start.sh +++ b/server/start.sh @@ -1,5 +1,7 @@ #!/bin/sh +export LD_PRELOAD=/usr/lib/$(arch)-linux-gnu/libmimalloc.so.2 + if [ "$DB_URL_FILE" ]; then export DB_URL=$(cat $DB_URL_FILE) unset DB_URL_FILE diff --git a/server/test/fixtures/media.stub.ts b/server/test/fixtures/media.stub.ts index 8d885a3a441cb..662a0d78d4d16 100644 --- a/server/test/fixtures/media.stub.ts +++ b/server/test/fixtures/media.stub.ts @@ -7,7 +7,7 @@ const probeStubDefaultFormat: VideoFormat = { }; const probeStubDefaultVideoStream: VideoStreamInfo[] = [ - { height: 1080, width: 1920, codecName: 'h265', codecType: 'video', frameCount: 100, rotation: 0 }, + { height: 1080, width: 1920, codecName: 'hevc', codecType: 'video', frameCount: 100, rotation: 0 }, ]; const probeStubDefaultAudioStream: AudioStreamInfo[] = [{ codecName: 'aac', codecType: 'audio' }]; @@ -20,13 +20,14 @@ const probeStubDefault: VideoInfo = { export const probeStub = { noVideoStreams: Object.freeze({ ...probeStubDefault, videoStreams: [] }), + noAudioStreams: Object.freeze({ ...probeStubDefault, audioStreams: [] }), multipleVideoStreams: Object.freeze({ ...probeStubDefault, videoStreams: [ { height: 1080, width: 400, - codecName: 'h265', + codecName: 'hevc', codecType: 'video', frameCount: 100, rotation: 0, @@ -47,7 +48,7 @@ export const probeStub = { { height: 0, width: 400, - codecName: 'h265', + codecName: 'hevc', codecType: 'video', frameCount: 100, rotation: 0, diff --git a/server/test/repositories/storage.repository.mock.ts b/server/test/repositories/storage.repository.mock.ts index 08556a0815b97..94c95228bf80d 100644 --- a/server/test/repositories/storage.repository.mock.ts +++ b/server/test/repositories/storage.repository.mock.ts @@ -11,5 +11,6 @@ export const newStorageRepositoryMock = (): jest.Mocked => { checkFileExists: jest.fn(), mkdirSync: jest.fn(), checkDiskUsage: jest.fn(), + readdir: jest.fn(), }; }; diff --git a/web/src/api/open-api/api.ts b/web/src/api/open-api/api.ts index 453e9ae84e4af..6724ba2bc9c74 100644 --- a/web/src/api/open-api/api.ts +++ b/web/src/api/open-api/api.ts @@ -704,13 +704,13 @@ export interface AssetStatsResponseDto { * @type {number} * @memberof AssetStatsResponseDto */ - 'total': number; + 'videos': number; /** * * @type {number} * @memberof AssetStatsResponseDto */ - 'videos': number; + 'total': number; } /** * @@ -2548,6 +2548,12 @@ export interface SystemConfigDto { * @interface SystemConfigFFmpegDto */ export interface SystemConfigFFmpegDto { + /** + * + * @type {TranscodeHWAccel} + * @memberof SystemConfigFFmpegDto + */ + 'accel': TranscodeHWAccel; /** * * @type {number} @@ -2896,6 +2902,22 @@ export const TimeGroupEnum = { export type TimeGroupEnum = typeof TimeGroupEnum[keyof typeof TimeGroupEnum]; +/** + * + * @export + * @enum {string} + */ + +export const TranscodeHWAccel = { + Nvenc: 'nvenc', + Qsv: 'qsv', + Vaapi: 'vaapi', + Disabled: 'disabled' +} as const; + +export type TranscodeHWAccel = typeof TranscodeHWAccel[keyof typeof TranscodeHWAccel]; + + /** * * @export diff --git a/web/src/lib/components/admin-page/settings/ffmpeg/ffmpeg-settings.svelte b/web/src/lib/components/admin-page/settings/ffmpeg/ffmpeg-settings.svelte index 3d28979bcf54d..5e03c29465aed 100644 --- a/web/src/lib/components/admin-page/settings/ffmpeg/ffmpeg-settings.svelte +++ b/web/src/lib/components/admin-page/settings/ffmpeg/ffmpeg-settings.svelte @@ -3,7 +3,7 @@ notificationController, NotificationType, } from '$lib/components/shared-components/notification/notification'; - import { api, AudioCodec, SystemConfigFFmpegDto, TranscodePolicy, VideoCodec } from '@api'; + import { api, AudioCodec, SystemConfigFFmpegDto, TranscodeHWAccel, TranscodePolicy, VideoCodec } from '@api'; import SettingButtonsRow from '../setting-buttons-row.svelte'; import SettingInputField, { SettingInputFieldType } from '../setting-input-field.svelte'; import SettingSelect from '../setting-select.svelte'; @@ -189,6 +189,29 @@ isEdited={!(ffmpegConfig.transcode == savedConfig.transcode)} /> + +