From fa329d99e0e16a472ac1415e43fc94b4935a8b65 Mon Sep 17 00:00:00 2001 From: jakobkukla Date: Sun, 10 Nov 2024 01:21:19 +0100 Subject: [PATCH] Implement media segment support Most of the proposed changes in the files listed below have been shamelessly copied from the Android TV implementation in https://github.com/jellyfin/jellyfin-androidtv/pull/4052. Authorship of these changes belongs to nielsvanvelzen. app/src/main/java/org/jellyfin/mobile/player/PlayerViewModel.kt app/src/main/java/org/jellyfin/mobile/player/mediasegments/MediaSegmentAction.kt app/src/main/java/org/jellyfin/mobile/player/mediasegments/MediaSegmentRepository.kt app/src/main/java/org/jellyfin/mobile/utils/extensions/MediaSegment.kt --- .../main/assets/native/MediaSegmentsPlugin.js | 32 +++++++ app/src/main/assets/native/nativeshell.js | 3 +- .../java/org/jellyfin/mobile/app/AppModule.kt | 2 + .../org/jellyfin/mobile/app/AppPreferences.kt | 16 ++++ .../jellyfin/mobile/bridge/MediaSegments.kt | 35 +++++++ .../jellyfin/mobile/player/PlayerViewModel.kt | 50 ++++++++++ .../mediasegments/MediaSegmentAction.kt | 20 ++++ .../mediasegments/MediaSegmentRepository.kt | 95 +++++++++++++++++++ .../org/jellyfin/mobile/utils/Constants.kt | 1 + .../mobile/utils/extensions/MediaSegment.kt | 10 ++ .../jellyfin/mobile/webapp/WebViewFragment.kt | 2 + 11 files changed, 265 insertions(+), 1 deletion(-) create mode 100644 app/src/main/assets/native/MediaSegmentsPlugin.js create mode 100644 app/src/main/java/org/jellyfin/mobile/bridge/MediaSegments.kt create mode 100644 app/src/main/java/org/jellyfin/mobile/player/mediasegments/MediaSegmentAction.kt create mode 100644 app/src/main/java/org/jellyfin/mobile/player/mediasegments/MediaSegmentRepository.kt create mode 100644 app/src/main/java/org/jellyfin/mobile/utils/extensions/MediaSegment.kt diff --git a/app/src/main/assets/native/MediaSegmentsPlugin.js b/app/src/main/assets/native/MediaSegmentsPlugin.js new file mode 100644 index 000000000..fe29cb035 --- /dev/null +++ b/app/src/main/assets/native/MediaSegmentsPlugin.js @@ -0,0 +1,32 @@ +export class MediaSegmentsPlugin { + SETTING_PREFIX = 'segmentTypeAction'; + + constructor({ events, appSettings, dashboard }) { + this.appSettings = appSettings; + this.dashboard = dashboard; + + events.on(appSettings, 'change', (_, name) => this.onSettingsChanged(name)); + } + + getSettingId(type) { + return `${this.SETTING_PREFIX}__${type}`; + } + + getSettingValue(id) { + var userId = this.dashboard.getCurrentUserId(); + + return this.appSettings.get(id, userId); + } + + // Update media segment action + onSettingsChanged(name) { + if (name.startsWith(this.SETTING_PREFIX)) { + var type = name.slice(this.SETTING_PREFIX.length + 2); + var action = this.getSettingValue(this.getSettingId(type)); + + if (type != null && action != null) { + MediaSegments.setSegmentTypeAction(type, action); + } + } + } +} diff --git a/app/src/main/assets/native/nativeshell.js b/app/src/main/assets/native/nativeshell.js index 9a9ae006d..bd4121ce9 100644 --- a/app/src/main/assets/native/nativeshell.js +++ b/app/src/main/assets/native/nativeshell.js @@ -18,7 +18,8 @@ const features = [ const plugins = [ 'NavigationPlugin', 'ExoPlayerPlugin', - 'ExternalPlayerPlugin' + 'ExternalPlayerPlugin', + 'MediaSegmentsPlugin' ]; // Add plugin loaders diff --git a/app/src/main/java/org/jellyfin/mobile/app/AppModule.kt b/app/src/main/java/org/jellyfin/mobile/app/AppModule.kt index 476cc8f12..7c9a911f0 100644 --- a/app/src/main/java/org/jellyfin/mobile/app/AppModule.kt +++ b/app/src/main/java/org/jellyfin/mobile/app/AppModule.kt @@ -27,6 +27,7 @@ import org.jellyfin.mobile.events.ActivityEventHandler import org.jellyfin.mobile.player.audio.car.LibraryBrowser import org.jellyfin.mobile.player.deviceprofile.DeviceProfileBuilder import org.jellyfin.mobile.player.interaction.PlayerEvent +import org.jellyfin.mobile.player.mediasegments.MediaSegmentRepository import org.jellyfin.mobile.player.qualityoptions.QualityOptionsProvider import org.jellyfin.mobile.player.source.MediaSourceResolver import org.jellyfin.mobile.player.ui.PlayerFragment @@ -82,6 +83,7 @@ val applicationModule = module { single { MediaSourceResolver(get()) } single { DeviceProfileBuilder(get()) } single { QualityOptionsProvider() } + single { MediaSegmentRepository() } // ExoPlayer factories single { diff --git a/app/src/main/java/org/jellyfin/mobile/app/AppPreferences.kt b/app/src/main/java/org/jellyfin/mobile/app/AppPreferences.kt index e4d02a523..fb41ac640 100644 --- a/app/src/main/java/org/jellyfin/mobile/app/AppPreferences.kt +++ b/app/src/main/java/org/jellyfin/mobile/app/AppPreferences.kt @@ -5,9 +5,12 @@ import android.content.SharedPreferences import android.os.Environment import android.view.WindowManager.LayoutParams.BRIGHTNESS_OVERRIDE_NONE import androidx.core.content.edit +import org.jellyfin.mobile.player.mediasegments.MediaSegmentAction +import org.jellyfin.mobile.player.mediasegments.toMediaSegmentActionsString import org.jellyfin.mobile.settings.ExternalPlayerPackage import org.jellyfin.mobile.settings.VideoPlayerType import org.jellyfin.mobile.utils.Constants +import org.jellyfin.sdk.model.api.MediaSegmentType import java.io.File class AppPreferences(context: Context) { @@ -90,6 +93,19 @@ class AppPreferences(context: Context) { } } + /** + * The actions to take for each media segment type. Managed by the MediaSegmentRepository. + */ + var mediaSegmentActions: String + get() = sharedPreferences.getString( + Constants.PREF_MEDIA_SEGMENT_ACTIONS, + mapOf( + MediaSegmentType.INTRO to MediaSegmentAction.ASK_TO_SKIP, + MediaSegmentType.OUTRO to MediaSegmentAction.ASK_TO_SKIP, + ).toMediaSegmentActionsString(), + )!! + set(value) = sharedPreferences.edit { putString(Constants.PREF_MEDIA_SEGMENT_ACTIONS, value) } + val musicNotificationAlwaysDismissible: Boolean get() = sharedPreferences.getBoolean(Constants.PREF_MUSIC_NOTIFICATION_ALWAYS_DISMISSIBLE, false) diff --git a/app/src/main/java/org/jellyfin/mobile/bridge/MediaSegments.kt b/app/src/main/java/org/jellyfin/mobile/bridge/MediaSegments.kt new file mode 100644 index 000000000..c1fa7ae66 --- /dev/null +++ b/app/src/main/java/org/jellyfin/mobile/bridge/MediaSegments.kt @@ -0,0 +1,35 @@ +package org.jellyfin.mobile.bridge + +import android.content.Context +import android.webkit.JavascriptInterface +import org.jellyfin.mobile.player.mediasegments.MediaSegmentAction +import org.jellyfin.mobile.player.mediasegments.MediaSegmentRepository +import org.jellyfin.sdk.model.api.MediaSegmentType +import org.koin.core.component.KoinComponent +import org.koin.core.component.get + +@Suppress("unused") +class MediaSegments(private val context: Context) : KoinComponent { + private val mediaSegmentRepository: MediaSegmentRepository = get() + + @JavascriptInterface + fun setSegmentTypeAction(typeString: String, actionString: String) { + val type: MediaSegmentType = when(typeString) { + "Intro" -> MediaSegmentType.INTRO + "Outro" -> MediaSegmentType.OUTRO + "Preview" -> MediaSegmentType.PREVIEW + "Recap" -> MediaSegmentType.RECAP + "Commercial" -> MediaSegmentType.COMMERCIAL + else -> return + } + + val action: MediaSegmentAction = when(actionString) { + "None" -> MediaSegmentAction.NOTHING + "Skip" -> MediaSegmentAction.SKIP + "AskToSkip" -> MediaSegmentAction.ASK_TO_SKIP + else -> return + } + + mediaSegmentRepository.setDefaultSegmentTypeAction(type, action) + } +} diff --git a/app/src/main/java/org/jellyfin/mobile/player/PlayerViewModel.kt b/app/src/main/java/org/jellyfin/mobile/player/PlayerViewModel.kt index 961863c57..cf7364e93 100644 --- a/app/src/main/java/org/jellyfin/mobile/player/PlayerViewModel.kt +++ b/app/src/main/java/org/jellyfin/mobile/player/PlayerViewModel.kt @@ -38,6 +38,8 @@ import org.jellyfin.mobile.player.interaction.PlayerEvent import org.jellyfin.mobile.player.interaction.PlayerLifecycleObserver import org.jellyfin.mobile.player.interaction.PlayerMediaSessionCallback import org.jellyfin.mobile.player.interaction.PlayerNotificationHelper +import org.jellyfin.mobile.player.mediasegments.MediaSegmentAction +import org.jellyfin.mobile.player.mediasegments.MediaSegmentRepository import org.jellyfin.mobile.player.queue.QueueManager import org.jellyfin.mobile.player.source.JellyfinMediaSource import org.jellyfin.mobile.player.ui.DecoderType @@ -47,7 +49,9 @@ import org.jellyfin.mobile.utils.Constants import org.jellyfin.mobile.utils.Constants.SUPPORTED_VIDEO_PLAYER_PLAYBACK_ACTIONS import org.jellyfin.mobile.utils.applyDefaultAudioAttributes import org.jellyfin.mobile.utils.applyDefaultLocalAudioAttributes +import org.jellyfin.mobile.utils.extensions.end import org.jellyfin.mobile.utils.extensions.scaleInRange +import org.jellyfin.mobile.utils.extensions.start import org.jellyfin.mobile.utils.extensions.width import org.jellyfin.mobile.utils.getVolumeLevelPercent import org.jellyfin.mobile.utils.getVolumeRange @@ -65,6 +69,7 @@ import org.jellyfin.sdk.api.operations.DisplayPreferencesApi import org.jellyfin.sdk.api.operations.HlsSegmentApi import org.jellyfin.sdk.api.operations.PlayStateApi import org.jellyfin.sdk.api.operations.UserApi +import org.jellyfin.sdk.model.api.MediaSegmentDto import org.jellyfin.sdk.model.api.PlayMethod import org.jellyfin.sdk.model.api.PlaybackOrder import org.jellyfin.sdk.model.api.PlaybackProgressInfo @@ -96,6 +101,7 @@ class PlayerViewModel(application: Application) : AndroidViewModel(application), val queueManager = QueueManager(this) val mediaSourceOrNull: JellyfinMediaSource? get() = queueManager.currentMediaSourceOrNull + private val mediaSegmentRepository = MediaSegmentRepository() // ExoPlayer private val _player = MutableLiveData() @@ -264,6 +270,9 @@ class PlayerViewModel(application: Application) : AndroidViewModel(application), val startTime = jellyfinMediaSource.startTimeMs if (startTime > 0) player.seekTo(startTime) + + applyMediaSegments(jellyfinMediaSource) + player.playWhenReady = playWhenReady mediaSession.setMetadata(jellyfinMediaSource.toMediaMetadata()) @@ -403,6 +412,47 @@ class PlayerViewModel(application: Application) : AndroidViewModel(application), } } + private fun applyMediaSegments(jellyfinMediaSource: JellyfinMediaSource) { + viewModelScope.launch { + if (jellyfinMediaSource.item != null) { + val mediaSegments = runCatching { + mediaSegmentRepository.getSegmentsForItem(jellyfinMediaSource.item) + }.getOrNull().orEmpty() + + for (mediaSegment in mediaSegments) { + val action = mediaSegmentRepository.getMediaSegmentAction(mediaSegment) + + when(action) { + MediaSegmentAction.SKIP -> addSkipAction(mediaSegment) + MediaSegmentAction.NOTHING -> Unit + // Unimplemented + MediaSegmentAction.ASK_TO_SKIP -> Unit + } + } + } + } + + } + + private fun addSkipAction(mediaSegment: MediaSegmentDto) { + val player = playerOrNull ?: return + + player.createMessage { _, _ -> + // We can't seek directly on the ExoPlayer instance as not all media is seekable + // the seek function in the PlaybackController checks this and optionally starts a transcode + // at the requested position + // TODO: The above is probably true for jellyfin-android as well. + // But I believe there is no such logic here. + viewModelScope.launch(Dispatchers.Main) { + player.seekTo(mediaSegment.end.inWholeMilliseconds) + } + } + // Segments at position 0 will never be hit by ExoPlayer so we need to add a minimum value + .setPosition(mediaSegment.start.inWholeMilliseconds.coerceAtLeast(1)) + .setDeleteAfterDelivery(false) + .send() + } + // Player controls fun play() { diff --git a/app/src/main/java/org/jellyfin/mobile/player/mediasegments/MediaSegmentAction.kt b/app/src/main/java/org/jellyfin/mobile/player/mediasegments/MediaSegmentAction.kt new file mode 100644 index 000000000..a2b992900 --- /dev/null +++ b/app/src/main/java/org/jellyfin/mobile/player/mediasegments/MediaSegmentAction.kt @@ -0,0 +1,20 @@ +package org.jellyfin.mobile.player.mediasegments + +enum class MediaSegmentAction { + /** + * Don't take any action for this segment. + */ + NOTHING, + + /** + * Seek to the end of this segment (endTicks). If the duration of this segment is shorter than 1 second it should do nothing to avoid + * lag. The skip action will only execute when playing over the segment start, not when seeking into the segment block. + */ + SKIP, + + /** + * Ask the user if they want to skip this segment. When the user agrees this behaves like [SKIP]. Confirmation should only be asked for + * segments with a duration of at least 3 seconds to avoid UI flickering. + */ + ASK_TO_SKIP, +} diff --git a/app/src/main/java/org/jellyfin/mobile/player/mediasegments/MediaSegmentRepository.kt b/app/src/main/java/org/jellyfin/mobile/player/mediasegments/MediaSegmentRepository.kt new file mode 100644 index 000000000..b9c5cd8e8 --- /dev/null +++ b/app/src/main/java/org/jellyfin/mobile/player/mediasegments/MediaSegmentRepository.kt @@ -0,0 +1,95 @@ +package org.jellyfin.mobile.player.mediasegments + +import org.jellyfin.mobile.app.AppPreferences +import org.jellyfin.mobile.utils.extensions.duration +import org.jellyfin.sdk.api.client.ApiClient +import org.jellyfin.sdk.api.client.extensions.mediaSegmentsApi +import org.jellyfin.sdk.api.operations.MediaSegmentsApi +import org.jellyfin.sdk.model.api.BaseItemDto +import org.jellyfin.sdk.model.api.MediaSegmentDto +import org.jellyfin.sdk.model.api.MediaSegmentType +import org.koin.core.component.KoinComponent +import org.koin.core.component.get +import org.koin.core.component.inject +import kotlin.time.Duration.Companion.seconds + +fun Map.toMediaSegmentActionsString() = + map { "${it.key.serialName}=${it.value.name}" } + .joinToString(",") + +class MediaSegmentRepository : KoinComponent { + companion object { + /** + * All media segments currently supported by the app. + */ + val SupportedTypes = listOf( + MediaSegmentType.INTRO, + MediaSegmentType.OUTRO, + MediaSegmentType.PREVIEW, + MediaSegmentType.RECAP, + MediaSegmentType.COMMERCIAL, + ) + + /** + * The minimum duration for a media segment to allow the [MediaSegmentAction.SKIP] action. + */ + val SkipMinDuration = 1.seconds + } + + private val appPreferences: AppPreferences by inject() + private val apiClient: ApiClient = get() + private val mediaSegmentsApi: MediaSegmentsApi = apiClient.mediaSegmentsApi + + private val mediaTypeActions = mutableMapOf() + + init { + restoreMediaTypeActions() + } + + private fun restoreMediaTypeActions() { + val restoredMediaTypeActions = appPreferences.mediaSegmentActions + .split(",") + .mapNotNull { + runCatching { + val (type, action) = it.split('=', limit = 2) + MediaSegmentType.fromName(type) to MediaSegmentAction.valueOf(action) + }.getOrNull() + } + + mediaTypeActions.clear() + mediaTypeActions.putAll(restoredMediaTypeActions) + } + + private fun saveMediaTypeActions() { + appPreferences.mediaSegmentActions = mediaTypeActions.toMediaSegmentActionsString() + } + + fun getDefaultSegmentTypeAction(type: MediaSegmentType): MediaSegmentAction { + // Always return no action for unsupported types + if (!SupportedTypes.contains(type)) return MediaSegmentAction.NOTHING + + return mediaTypeActions.getOrDefault(type, MediaSegmentAction.NOTHING) + } + + fun setDefaultSegmentTypeAction(type: MediaSegmentType, action: MediaSegmentAction) { + // Don't allow modifying actions for unsupported types + if (!SupportedTypes.contains(type)) return + + mediaTypeActions[type] = action + saveMediaTypeActions() + } + + fun getMediaSegmentAction(segment: MediaSegmentDto): MediaSegmentAction { + val action = getDefaultSegmentTypeAction(segment.type) + // Skip the skip action if timespan is too short + if (action == MediaSegmentAction.SKIP && segment.duration < SkipMinDuration) return MediaSegmentAction.NOTHING + return action + } + + suspend fun getSegmentsForItem(item: BaseItemDto): List = runCatching { + mediaSegmentsApi.getItemSegments( + itemId = item.id, + includeSegmentTypes = SupportedTypes, + ).content.items + }.getOrDefault(emptyList()) +} diff --git a/app/src/main/java/org/jellyfin/mobile/utils/Constants.kt b/app/src/main/java/org/jellyfin/mobile/utils/Constants.kt index 57158181c..e94366a7c 100644 --- a/app/src/main/java/org/jellyfin/mobile/utils/Constants.kt +++ b/app/src/main/java/org/jellyfin/mobile/utils/Constants.kt @@ -42,6 +42,7 @@ object Constants { const val PREF_EXTERNAL_PLAYER_APP = "pref_external_player_app" const val PREF_SUBTITLE_STYLE = "pref_subtitle_style" const val PREF_DOWNLOAD_LOCATION = "pref_download_location" + const val PREF_MEDIA_SEGMENT_ACTIONS = "pref_media_segment_actions" // InputManager commands const val PLAYBACK_MANAGER_COMMAND_PLAY = "unpause" diff --git a/app/src/main/java/org/jellyfin/mobile/utils/extensions/MediaSegment.kt b/app/src/main/java/org/jellyfin/mobile/utils/extensions/MediaSegment.kt new file mode 100644 index 000000000..0c070469d --- /dev/null +++ b/app/src/main/java/org/jellyfin/mobile/utils/extensions/MediaSegment.kt @@ -0,0 +1,10 @@ +package org.jellyfin.mobile.utils.extensions + +import org.jellyfin.sdk.model.api.MediaSegmentDto +import org.jellyfin.sdk.model.extensions.ticks +import kotlin.time.Duration + +val MediaSegmentDto.start get() = startTicks.ticks +val MediaSegmentDto.end get() = endTicks.ticks + +val MediaSegmentDto.duration get() = (endTicks - startTicks).ticks.coerceAtLeast(Duration.ZERO) diff --git a/app/src/main/java/org/jellyfin/mobile/webapp/WebViewFragment.kt b/app/src/main/java/org/jellyfin/mobile/webapp/WebViewFragment.kt index 12ae0af31..abd1865d9 100644 --- a/app/src/main/java/org/jellyfin/mobile/webapp/WebViewFragment.kt +++ b/app/src/main/java/org/jellyfin/mobile/webapp/WebViewFragment.kt @@ -28,6 +28,7 @@ import org.jellyfin.mobile.R import org.jellyfin.mobile.app.ApiClientController import org.jellyfin.mobile.app.AppPreferences import org.jellyfin.mobile.bridge.ExternalPlayer +import org.jellyfin.mobile.bridge.MediaSegments import org.jellyfin.mobile.bridge.NativeInterface import org.jellyfin.mobile.bridge.NativePlayer import org.jellyfin.mobile.data.entity.ServerEntity @@ -192,6 +193,7 @@ class WebViewFragment : Fragment(), BackPressInterceptor, JellyfinWebChromeClien addJavascriptInterface(NativeInterface(requireContext()), "NativeInterface") addJavascriptInterface(nativePlayer, "NativePlayer") addJavascriptInterface(externalPlayer, "ExternalPlayer") + addJavascriptInterface(MediaSegments(requireContext()), "MediaSegments") loadUrl(server.hostname) postDelayed(timeoutRunnable, Constants.INITIAL_CONNECTION_TIMEOUT)