Skip to content

Commit

Permalink
Add track by URL resolving functionality to MultiTrackSearchClient
Browse files Browse the repository at this point in the history
  • Loading branch information
s-frei committed Mar 26, 2024
1 parent 1727aff commit 6c0d6f0
Show file tree
Hide file tree
Showing 11 changed files with 123 additions and 43 deletions.
49 changes: 32 additions & 17 deletions src/main/java/io/sfrei/tracksearch/clients/MultiSearchClient.java
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,6 @@
import io.sfrei.tracksearch.clients.interfaces.Provider;
import io.sfrei.tracksearch.clients.setup.QueryType;
import io.sfrei.tracksearch.clients.setup.TrackSource;
import io.sfrei.tracksearch.clients.soundcloud.SoundCloudClient;
import io.sfrei.tracksearch.clients.youtube.YouTubeClient;
import io.sfrei.tracksearch.config.TrackSearchConfig;
import io.sfrei.tracksearch.exceptions.TrackSearchException;
import io.sfrei.tracksearch.tracks.*;
Expand All @@ -41,23 +39,40 @@ public class MultiSearchClient implements MultiTrackSearchClient, Provider<Track
public static final String POSITION_KEY = "multi" + TrackSearchConfig.POSITION_KEY_SUFFIX;
public static final String OFFSET_KEY = "multi" + TrackSearchConfig.OFFSET_KEY_SUFFIX;

private final TrackSearchClient<YouTubeTrack> youTubeClient;
private final TrackSearchClient<SoundCloudTrack> soundCloudClient;

private final Map<TrackSource, TrackSearchClient<? extends Track>> clientsForSource = new HashMap<>();
private final Map<TrackSource, TrackSearchClient<Track>> clientsBySource;
private final Set<String> validURLPrefixes;

public MultiSearchClient() {
this.youTubeClient = new YouTubeClient();
this.soundCloudClient = new SoundCloudClient();
clientsBySource = Arrays.stream(TrackSource.values())
.collect(Collectors.toMap(source -> source, TrackSource::createClient));

clientsForSource.put(TrackSource.Youtube, youTubeClient);
clientsForSource.put(TrackSource.Soundcloud, soundCloudClient);
validURLPrefixes = clientsBySource.values()
.stream().map(TrackSearchClient::validURLPrefixes)
.flatMap(Set::stream)
.collect(Collectors.toSet());

log.info("TrackSearchClient created with {} clients", allClients().size());
}

private List<TrackSearchClient<? extends Track>> allClients() {
return new ArrayList<>(clientsForSource.values());
return new ArrayList<>(clientsBySource.values());
}

@Override
public Set<String> validURLPrefixes() {
return validURLPrefixes;
}

@Override
public Track getTrack(@NonNull String url) throws TrackSearchException {
final TrackSearchClient<Track> trackSearchClient = clientsBySource.values()
.stream()
.filter(client -> client.isApplicableForURL(url))
.findFirst()
.orElseThrow(() -> new TrackSearchException(String.format("No client found to handle URL: %s", url)));

log().debug("Using {} for URL: {}", trackSearchClient.getClass().getSimpleName(), url);
return trackSearchClient.getTrack(url);
}

@Override
Expand All @@ -79,19 +94,19 @@ public TrackList<Track> getNext(@NonNull final TrackList<? extends Track> trackL
public String getStreamUrl(@NonNull final Track track) throws TrackSearchException {

if (track instanceof YouTubeTrack) {
return youTubeClient.getStreamUrl((YouTubeTrack) track);
return clientsBySource.get(TrackSource.Youtube).getStreamUrl(track);
} else if (track instanceof SoundCloudTrack) {
return soundCloudClient.getStreamUrl((SoundCloudTrack) track);
return clientsBySource.get(TrackSource.Soundcloud).getStreamUrl(track);
}
throw new TrackSearchException("Track type is unknown");
}

@Override
public String getStreamUrl(@NonNull Track track, int retries) throws TrackSearchException {
if (track instanceof YouTubeTrack) {
return youTubeClient.getStreamUrl((YouTubeTrack) track, retries);
return clientsBySource.get(TrackSource.Youtube).getStreamUrl(track, retries);
} else if (track instanceof SoundCloudTrack) {
return soundCloudClient.getStreamUrl((SoundCloudTrack) track, retries);
return clientsBySource.get(TrackSource.Soundcloud).getStreamUrl(track, retries);
}
throw new TrackSearchException("Track type is unknown");
}
Expand All @@ -104,8 +119,8 @@ public TrackList<Track> getTracksForSearch(@NonNull final String search, @NonNul
throw new TrackSearchException("Provide at least one source");

final List<TrackSearchClient<? extends Track>> callClients = sources.stream()
.filter(clientsForSource::containsKey)
.map(clientsForSource::get)
.filter(clientsBySource::containsKey)
.map(clientsBySource::get)
.collect(Collectors.toList());

return getTracksForSearch(search, callClients);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ public interface MultiTrackSearchClient extends TrackSearchClient<Track> {
String getStreamUrl(@NotNull Track track) throws TrackSearchException;

/**
* Search for tracks using a string containing keywords on pre selected track sources.
* Search for tracks using a string containing keywords on pre-selected track sources.
* @param search keywords to search for.
* @param sources available to search on.
* @return a tracklist containing all found tracks for selected clients.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,14 +21,45 @@
import io.sfrei.tracksearch.tracks.TrackList;
import lombok.NonNull;

import java.util.Set;

/**
* Main interface containing all functionality a client offers to the user.
*
* @param <T> the track type the client implementing this is used for.
*/
public interface TrackSearchClient<T extends Track> {

/**
* Retrieve all valid URL prefixes used to check {@link #isApplicableForURL(String)}.
*
* @return the set of valid URL prefixes.
*/
Set<String> validURLPrefixes();

/**
* Test if this client can handle the provided URL to make sure it can
* be used for {@link #getTrack(String)}.
*
* @param url the URL to check.
* @return true if this client is applicable and false if not.
*/
default boolean isApplicableForURL(@NonNull String url) {
return validURLPrefixes().stream().anyMatch(url::startsWith);
}

/**
* Get a track for the given URL.
*
* @param url the URL to create track for.
* @return the track for the provided URL.
* @throws TrackSearchException when the track cannot ba created.
*/
T getTrack(@NonNull String url) throws TrackSearchException;

/**
* Search for tracks using a string containing keywords.
*
* @param search keywords to search for.
* @return a track list containing all found tracks.
* @throws TrackSearchException when the client encountered a problem on searching.
Expand All @@ -37,6 +68,7 @@ public interface TrackSearchClient<T extends Track> {

/**
* Search for the next tracks for last result.
*
* @param trackList a previous search result for that client.
* @return a track list containing the next tracks available.
* @throws TrackSearchException when the client encounters a problem on getting the next tracks.
Expand All @@ -45,6 +77,7 @@ public interface TrackSearchClient<T extends Track> {

/**
* Get the audio stream URL in the highest possible audio resolution.
*
* @param track from this client.
* @return the audio stream URL.
* @throws TrackSearchException when the URL could not be exposed.
Expand All @@ -53,7 +86,8 @@ public interface TrackSearchClient<T extends Track> {

/**
* Get the audio stream URL in the highest possible audio resolution and retry when there was a failure.
* @param track from this client.
*
* @param track from this client.
* @param retries retry when stream URL resolving was not successful. This is determined with another request/s.
* @return the audio stream URL.
* @throws TrackSearchException when the URL could not be exposed.
Expand All @@ -62,6 +96,7 @@ public interface TrackSearchClient<T extends Track> {

/**
* Check the track list for this client if the paging values to get next are present.
*
* @param trackList a previous search result for this client.
* @return either the paging values are present or not.
*/
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
import java.util.function.Function;


public interface ClientHelper extends ClassLogger {
public interface ClientHelper extends ClientLogger {

int INITIAL_TRY = 1;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@

import org.slf4j.Logger;

public interface ClassLogger {
public interface ClientLogger {

/**
* Obtain the logger from the implementing class.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
import io.sfrei.tracksearch.tracks.TrackList;
import org.jetbrains.annotations.Nullable;

public interface Provider<T extends Track> extends TrackSearchClient<T>, ClassLogger {
public interface Provider<T extends Track> extends TrackSearchClient<T>, ClientLogger {

@Nullable
default TrackList<T> provideNext(final TrackList<T> trackList) {
Expand Down
12 changes: 12 additions & 0 deletions src/main/java/io/sfrei/tracksearch/clients/setup/TrackSource.java
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,11 @@

package io.sfrei.tracksearch.clients.setup;

import io.sfrei.tracksearch.clients.TrackSearchClient;
import io.sfrei.tracksearch.clients.soundcloud.SoundCloudClient;
import io.sfrei.tracksearch.clients.youtube.YouTubeClient;
import io.sfrei.tracksearch.tracks.Track;

import java.util.Arrays;
import java.util.Set;
import java.util.stream.Collectors;
Expand All @@ -28,4 +33,11 @@ public static Set<TrackSource> setOf(TrackSource... sources) {
return Arrays.stream(sources).collect(Collectors.toSet());
}

public <T extends Track> TrackSearchClient<T> createClient() {
return (TrackSearchClient<T>) switch (this) {
case Youtube -> new YouTubeClient();
case Soundcloud -> new SoundCloudClient();
};
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -36,22 +36,21 @@
import retrofit2.Retrofit;

import java.net.CookiePolicy;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.*;

@Slf4j
public class SoundCloudClient extends SingleSearchClient<SoundCloudTrack>
implements ClientHelper, Provider<SoundCloudTrack>, UniformClientException {

public static final String HOSTNAME = "https://soundcloud.com";
public static final String URL = "https://soundcloud.com";
private static final String INFORMATION_PREFIX = "sc";
public static final String POSITION_KEY = INFORMATION_PREFIX + TrackSearchConfig.POSITION_KEY_SUFFIX;
public static final String OFFSET_KEY = INFORMATION_PREFIX + TrackSearchConfig.OFFSET_KEY_SUFFIX;
private static final String PAGING_OFFSET = "limit";
private static final String PAGING_POSITION = "position";

private static final Set<String> VALID_URL_PREFIXES = Set.of(URL); // TODO: Extend

private final SoundCloudAPI api;
private final SoundCloudUtility soundCloudUtility;

Expand All @@ -62,7 +61,7 @@ public SoundCloudClient() {
super(CookiePolicy.ACCEPT_ALL, null);

final Retrofit base = new Retrofit.Builder()
.baseUrl(HOSTNAME)
.baseUrl(URL)
.client(okHttpClient)
.addConverterFactory(ResponseProviderFactory.create())
.build();
Expand All @@ -76,7 +75,16 @@ public static Map<String, String> makeQueryInformation(final String query) {
return new HashMap<>(Map.of(TrackList.QUERY_KEY, query));
}

@Override
public Set<String> validURLPrefixes() {
return VALID_URL_PREFIXES;
}

@Override
public SoundCloudTrack getTrack(@NonNull final String url) throws TrackSearchException {
if (!isApplicableForURL(url))
throw new SoundCloudException(String.format("%s not applicable for URL: %s", this.getClass().getSimpleName(), url));

final String trackHTML = requestRefreshingClientId(api.getForUrlWithClientID(url, clientID)).getContentOrThrow();
final String trackURL = soundCloudUtility.extractTrackURL(trackHTML);
final String trackJSON = requestRefreshingClientId(api.getForUrlWithClientID(trackURL, clientID)).getContentOrThrow();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,12 +41,15 @@

import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;

@Slf4j
public class YouTubeClient extends SingleSearchClient<YouTubeTrack>
implements ClientHelper, Provider<YouTubeTrack>, UniformClientException {

public static final String HOSTNAME = "https://www.youtube.com";
public static final String URL = "https://www.youtube.com";
public static final String PAGING_KEY = "ctoken";
private static final String INFORMATION_PREFIX = "yt";
public static final String POSITION_KEY = INFORMATION_PREFIX + TrackSearchConfig.POSITION_KEY_SUFFIX;
Expand All @@ -57,13 +60,11 @@ public class YouTubeClient extends SingleSearchClient<YouTubeTrack>
private static final Map<String, String> VIDEO_SEARCH_PARAMS = Map.of("sp", "EgIQAQ%3D%3D");
public static final Map<String, String> TRACK_PARAMS = Map.of("pbj", "1", "hl", "en", "alt", "json");

private static final Map<String, String> DEFAULT_SEARCH_PARAMS;
private static final Map<String, String> DEFAULT_SEARCH_PARAMS = Stream.of(VIDEO_SEARCH_PARAMS.entrySet(), TRACK_PARAMS.entrySet())
.flatMap(Set::stream)
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));

static {
DEFAULT_SEARCH_PARAMS = new HashMap<>();
DEFAULT_SEARCH_PARAMS.putAll(VIDEO_SEARCH_PARAMS);
DEFAULT_SEARCH_PARAMS.putAll(TRACK_PARAMS);
}
private static final Set<String> VALID_URL_PREFIXES = Set.of(URL); // TODO: Extend

private final YouTubeAPI api;
private final YouTubeUtility youTubeUtility;
Expand All @@ -78,7 +79,7 @@ public YouTubeClient() {
);

final Retrofit base = new Retrofit.Builder()
.baseUrl(HOSTNAME)
.baseUrl(URL)
.client(okHttpClient)
.addConverterFactory(ResponseProviderFactory.create())
.build();
Expand All @@ -92,10 +93,19 @@ public static Map<String, String> makeQueryInformation(final String query, final
return new HashMap<>(Map.of(TrackList.QUERY_KEY, query, PAGING_INFORMATION, pagingToken));
}

public YouTubeTrack getTrack(@NonNull final String trackUrl) throws TrackSearchException {
final String trackJSON = requestTrackJSON(api.getForUrlWithParams(trackUrl, TRACK_PARAMS));
@Override
public Set<String> validURLPrefixes() {
return VALID_URL_PREFIXES;
}

@Override
public YouTubeTrack getTrack(@NonNull final String url) throws TrackSearchException {
if (!isApplicableForURL(url))
throw new YouTubeException(String.format("%s not applicable for URL: %s", this.getClass().getSimpleName(), url));

final String trackJSON = requestTrackJSON(api.getForUrlWithParams(url, TRACK_PARAMS));
final YouTubeTrack youTubeTrack = youTubeUtility.extractYouTubeTrack(trackJSON, this::streamURLProvider);
final YouTubeTrackInfo trackInfo = youTubeUtility.extractTrackInfo(trackJSON, trackUrl, this::requestURL);
final YouTubeTrackInfo trackInfo = youTubeUtility.extractTrackInfo(trackJSON, url, this::requestURL);
youTubeTrack.setTrackInfo(trackInfo);
return youTubeTrack;
}
Expand Down Expand Up @@ -158,7 +168,7 @@ public String getStreamUrl(@NonNull final YouTubeTrack youtubeTrack) throws Trac
log.trace("Use cached script for: {}", scriptUrl);
scriptContent = scriptCache.get(scriptUrl);
} else {
scriptContent = requestURL(HOSTNAME + scriptUrl).getContentOrThrow();
scriptContent = requestURL(URL + scriptUrl).getContentOrThrow();
scriptCache.put(scriptUrl, scriptContent);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ public YouTubeTrack.ListYouTubeTrackBuilder deserialize(final JsonParser p, fina
if (title == null || duration == null || ref == null)
return null;

final String url = YouTubeClient.HOSTNAME.concat("/watch?v=").concat(ref);
final String url = YouTubeClient.URL.concat("/watch?v=").concat(ref);

final YouTubeTrack.ListYouTubeTrackBuilder listYouTubeTrackBuilder = new YouTubeTrack.ListYouTubeTrackBuilder();
final YouTubeTrackBuilder youTubeTrackBuilder = listYouTubeTrackBuilder.getBuilder()
Expand All @@ -66,7 +66,7 @@ public YouTubeTrack.ListYouTubeTrackBuilder deserialize(final JsonParser p, fina

final String channelUrlSuffix = owner.path("navigationEndpoint", "commandMetadata", "webCommandMetadata")
.asString("url");
final String channelUrl = YouTubeClient.HOSTNAME.concat(channelUrlSuffix);
final String channelUrl = YouTubeClient.URL.concat(channelUrlSuffix);

final String streamAmountText = rootElement.path("viewCountText").asString("simpleText");
final String streamAmountDigits = streamAmountText == null || streamAmountText.isEmpty() ?
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ public YouTubeTrack.URLYouTubeTrackBuilder deserialize(final JsonParser p, final
if (title == null || duration == null || ref == null)
return null;

final String url = YouTubeClient.HOSTNAME.concat("/watch?v=").concat(ref);
final String url = YouTubeClient.URL.concat("/watch?v=").concat(ref);

final YouTubeTrack.URLYouTubeTrackBuilder listYouTubeTrackBuilder = new YouTubeTrack.URLYouTubeTrackBuilder();
final YouTubeTrackBuilder youTubeTrackBuilder = listYouTubeTrackBuilder.getBuilder()
Expand Down

0 comments on commit 6c0d6f0

Please sign in to comment.