feat: youtube api (#134)

* feat: Use YouTube's first-party API, if available * feat: Note in `/test` when an alternative platform interface was used * chore: v3.1.0 * chore: Don't need to fetch `liveStreamingDetails` from YouTube * chore: Better organize the various YouTube approaches * fix(tests): Handle another fail case from Invidius * feat(tests): Run tests both with and without a YouTube API key * fix(ci): Pass YouTube API key to src tests * feat(ci): Don't run tests when docs change * fix(ci): Use YouTube API key in prod tests too * chore: Update changelog * feat(ci): Only run integration tests against the minified build
AverageHelper · Aug 23, 2024 · 508920b · 508920b
1 parent 9bfc61d
commit 508920b
Show file tree

Hide file tree

Showing 20 changed files with 499 additions and 204 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -3,6 +3,18 @@ name: Tests
 on:
   pull_request:
     branches: [main]
+    paths:
+      - .github/workflows/**
+      - prisma/**
+      - src/**
+      - tests/**
+      - package-lock.json
+      - package.json
+      - .dockerignore
+      - Dockerfile
+      - rollup.config.ts
+      - vitest.config.e2e.ts
+      - vitest.config.ts
 
   workflow_dispatch:
 
@@ -21,6 +33,9 @@ jobs:
       - run: npm ci
       - run: npm run build
       - run: npm run test:src
+        env:
+          # TODO: Some way for users without a Google account to run other tests:
+          YOUTUBE_API_KEY: ${{ secrets.YOUTUBE_API_KEY }}
 
   integration:
     runs-on: ubuntu-latest
@@ -34,31 +49,14 @@ jobs:
         with:
           node-version: 20.10.x
       - run: npm ci
-      - run: npm run build
-      - run: npm run test:e2e
-        env:
-          CI: true
-          NODE_ENV: "test"
-          DISCORD_TOKEN: ${{ secrets.DISCORD_TOKEN }}
-          SOUNDCLOUD_API_KEY: ${{ secrets.SOUNDCLOUD_API_KEY }}
-          CORDE_TEST_TOKEN: ${{ secrets.CORDE_TEST_TOKEN }}
-          CORDE_BOT_ID: ${{ secrets.CORDE_BOT_ID }}
-          BOT_TEST_ID: ${{ secrets.BOT_TEST_ID }}
-          GUILD_ID: ${{ secrets.GUILD_ID }}
-          CHANNEL_ID: ${{ secrets.CHANNEL_ID }}
-          QUEUE_CHANNEL_ID: ${{ secrets.QUEUE_CHANNEL_ID }}
-          QUEUE_ADMIN_ROLE_ID: ${{ secrets.QUEUE_ADMIN_ROLE_ID }}
-          QUEUE_CREATOR_ROLE_ID: ${{ secrets.QUEUE_CREATOR_ROLE_ID }}
-          BOT_PREFIX: "?"
-
-      # Try again with the minified build
       - run: npm run build --omit=dev
       - run: npm run test:e2e
         env:
           CI: true
           NODE_ENV: "test"
           DISCORD_TOKEN: ${{ secrets.DISCORD_TOKEN }}
           SOUNDCLOUD_API_KEY: ${{ secrets.SOUNDCLOUD_API_KEY }}
+          YOUTUBE_API_KEY: ${{ secrets.YOUTUBE_API_KEY }}
           CORDE_TEST_TOKEN: ${{ secrets.CORDE_TEST_TOKEN }}
           CORDE_BOT_ID: ${{ secrets.CORDE_BOT_ID }}
           BOT_TEST_ID: ${{ secrets.BOT_TEST_ID }}

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,14 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [3.1.0] - 2024-08-22
+### Added
+- Use YouTube's first-party API when an API key is provided.
+- The `/test` command now reports when an alternative source is used for querying platforms. This is especially useful to determine whether Gamgee needed to fall back on an Invidius instance when YTDL failed, or when an API key was not configured.
+
+### Changed
+- More reliable parsing of track duration data from Bandcamp, using [a polyfill](https://github.com/fullcalendar/temporal-polyfill) for the new [Temporal API](https://tc39.es/proposal-temporal/docs/duration.html) instead of RegEx.
+
 ## [3.0.0] - 2024-08-21
 ### Fixed
 - Version bump because of a breaking change in v2.2.1. (Sorry!!) We now require Node 20. Docker users should be unaffected, since the Dockerfile \*should\* be using the latest Node anyway.
@@ -456,6 +464,7 @@ After updating, be sure to run `npm ci && npm run build:clean && npm run migrate
 ### Added
 - Initial commit
 
+[3.1.0]: https://github.com/AverageHelper/Gamgee/compare/v3.0.0...v3.1.0
 [3.0.0]: https://github.com/AverageHelper/Gamgee/compare/v2.2.1...v3.0.0
 [2.2.1]: https://github.com/AverageHelper/Gamgee/compare/v2.2.0...v2.2.1
 [2.2.0]: https://github.com/AverageHelper/Gamgee/compare/v2.1.1...v2.2.0

diff --git a/README.md b/README.md
@@ -134,6 +134,12 @@ LOG_LEVEL={silly | debug | verbose | info | warn | error}
 # optional, the level of logs you should see in the console
 # must be one of [silly, debug, verbose, info, warn, error]
 # defaults to `info` in production mode, `error` in test mode, and `debug` in any other mode
+
+SOUNDCLOUD_API_KEY=YOUR_SOUNDCLOUD_KEY_HERE
+# optional, used for communicating with SoundCloud more reliably
+
+YOUTUBE_API_KEY=YOUR_YOUTUBE_KEY_HERE
+# optional, used for communicating with YouTube more reliably
 ```
 
 ### Install dependencies

diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "gamgee",
-	"version": "3.0.0",
+	"version": "3.1.0",
 	"description": "A Discord bot for managing a song request queue.",
 	"private": true,
 	"scripts": {
@@ -61,6 +61,7 @@
 		"soundcloud-scraper": "5.0.3",
 		"source-map-support": "0.5.21",
 		"superstruct": "1.0.3",
+		"temporal-polyfill": "0.2.5",
 		"winston": "3.8.1",
 		"winston-daily-rotate-file": "4.7.1",
 		"ytdl-core": "4.11.5"

diff --git a/src/actions/getVideoDetails.ts b/src/actions/getVideoDetails.ts
@@ -7,12 +7,28 @@ import { richErrorMessage } from "../helpers/richErrorMessage.js";
 import { MILLISECONDS_IN_SECOND } from "../constants/time.js";
 import { useLogger } from "../logger.js";
 
+export interface VideoMetaSource {
+	/** The name of the source platform. */
+	platformName: "youtube" | "soundcloud" | "bandcamp" | "pony.fm";
+
+	/** The name of the alternative interface used to source the data. */
+	alternative: string | null;
+}
+
 export interface VideoDetails {
+	/** The canonical URL of the track. */
 	url: string;
+
+	/** The title of the track. */
 	title: string;
+
+	/** The duration of the track. */
 	duration: {
 		seconds: number;
 	};
+
+	/** The source of the track metadata. */
+	metaSource: VideoMetaSource;
 }
 
 /**

diff --git a/src/actions/network/getBandcampTrack.test.ts b/src/actions/network/getBandcampTrack.test.ts
@@ -34,21 +34,17 @@ describe("Bandcamp track details", () => {
 		await expect(() => getBandcampTrack(new URL(url))).rejects.toThrow(VideoError);
 	});
 
+	// ISO strings given here are the ones found on Bandcamp as of 22 Aug 2024:
 	test.each`
-		url                                                                             | duration
-		${"https://poniesatdawn.bandcamp.com/track/let-the-magic-fill-your-soul"}       | ${233}
-		${"https://forestrainmedia.com/track/bad-wolf"}                                 | ${277}
-		${"https://lehtmojoe.bandcamp.com/track/were-not-going-home-dallas-stars-2020"} | ${170}
+		url                                                                             | duration | iso
+		${"https://poniesatdawn.bandcamp.com/track/let-the-magic-fill-your-soul"}       | ${233}   | ${"P00H03M53S"}
+		${"https://forestrainmedia.com/track/bad-wolf"}                                 | ${277}   | ${"P00H04M37S"}
+		${"https://lehtmojoe.bandcamp.com/track/were-not-going-home-dallas-stars-2020"} | ${170}   | ${"P00H02M50S"}
 	`(
 		"returns info for Bandcamp track $url, $duration seconds long",
-		async ({ url, duration }: { url: string; duration: number }) => {
+		async ({ url, duration, iso }: { url: string; duration: number; iso: string }) => {
 			mockFetchMetadata.mockResolvedValue({
-				jsonld: [
-					{
-						name: "sample",
-						duration: `0H${Math.floor(duration / 60)}M${duration % 60}S`,
-					},
-				],
+				jsonld: [{ name: "sample", duration: iso }],
 			} as unknown as Result);
 
 			const details = await getBandcampTrack(new URL(url));

diff --git a/src/actions/network/getBandcampTrack.ts b/src/actions/network/getBandcampTrack.ts
@@ -1,8 +1,27 @@
-import type { VideoDetails } from "../getVideoDetails.js";
+import type { VideoDetails, VideoMetaSource } from "../getVideoDetails.js";
+import { array, is, string, type } from "superstruct";
 import { fetchMetadata } from "../../helpers/fetchMetadata.js";
-import { isString } from "../../helpers/guards.js";
+import { richErrorMessage } from "../../helpers/richErrorMessage.js";
+import { secondsFromIso8601Duration } from "../../helpers/secondsFromIso8601Duration.js";
+import { useLogger } from "../../logger.js";
 import { VideoError } from "../../errors/index.js";
 
+const logger = useLogger();
+
+const bandcampJsonld = array(
+	type({
+		/** The title of the track. */
+		name: string(),
+		/** An ISO 8601 duration string. */
+		duration: string(),
+	}),
+);
+
+const metaSource: Readonly<VideoMetaSource> = {
+	platformName: "bandcamp",
+	alternative: null,
+};
+
 /**
  * Gets information about a Bandcamp track.
  *
@@ -17,35 +36,36 @@ import { VideoError } from "../../errors/index.js";
 export async function getBandcampTrack(url: URL, signal?: AbortSignal): Promise<VideoDetails> {
 	const metadata = await fetchMetadata(url, signal);
 
-	type Digit = 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9;
 	const jsonld = metadata.jsonld ?? [];
-	const json = jsonld[0] as
-		| { name?: string; duration: `${string}H${Digit}${Digit}M${Digit}${Digit}S` }
-		| undefined;
-	if (!json) throw new VideoError("Duration and title not found");
-	if (!json.duration || !isString(json.duration)) throw new VideoError("Duration data not found");
+	if (!is(jsonld, bandcampJsonld)) throw new VideoError("Duration or title not found");
 
-	const durationPropertiesMatch = json.duration.matchAll(/H([0-9]+)M([0-9]+)S/gu);
-	const durationProperties = Array.from(durationPropertiesMatch)[0] as
-		| [string, string, string, ...Array<string>] // at least 3 strings
-		| undefined;
-
-	// Sanity checks (I have a college education and I don't understand regex)
-	if (!durationProperties || durationProperties.length < 3 || !durationProperties.every(isString))
-		throw new VideoError("Duration not found");
-
-	const minutes = Number.parseInt(durationProperties[1], 10);
-	const seconds = Number.parseInt(durationProperties[2], 10);
-	if (Number.isNaN(minutes) || Number.isNaN(seconds)) throw new VideoError("Duration not found");
+	const json = jsonld[0];
+	if (!json) throw new VideoError("Duration and title not found");
 
-	const totalSeconds = minutes * 60 + seconds;
+	const durationString = json["duration"];
+	let durationSeconds: number;
+	try {
+		let durationStringToParse = durationString;
+		// Bandcamp's duration strings often don't parse correctly...
+		if (durationString.startsWith("P00H")) {
+			logger.debug(`Got nonstandard duration string '${durationString}'. Attempting workaround...`);
+			durationStringToParse = `PT${durationString.slice(1)}`;
+		}
+		durationSeconds = secondsFromIso8601Duration(durationStringToParse);
+	} catch (error) {
+		logger.error(
+			richErrorMessage(`Failed to parse ISO 8601 duration string '${durationString}'.`, error),
+		);
+		throw new VideoError("Duration could not be parsed");
+	}
 
-	const title: string | null = json.name ?? null;
-	if (title === null || !title) throw new VideoError("Title not found");
+	const title: string = json["name"];
+	if (!title) throw new VideoError("Title not found");
 
 	return {
 		url: url.href,
 		title,
-		duration: { seconds: Math.floor(totalSeconds) },
+		duration: { seconds: durationSeconds },
+		metaSource,
 	};
 }
diff --git a/src/actions/network/getPonyFmTrack.ts b/src/actions/network/getPonyFmTrack.ts
@@ -1,5 +1,5 @@
 import type { Infer } from "superstruct";
-import type { VideoDetails } from "../getVideoDetails.js";
+import type { VideoDetails, VideoMetaSource } from "../getVideoDetails.js";
 import { InvalidPonyFmUrlError, VideoError } from "../../errors/index.js";
 import { is, string, type } from "superstruct";
 
@@ -26,6 +26,11 @@ function isPonyFmTrackAPIError(tbd: unknown): tbd is PonyFmTrackAPIError {
 	return is(tbd, ponyFmTrackAPIError);
 }
 
+const metaSource: Readonly<VideoMetaSource> = {
+	platformName: "pony.fm",
+	alternative: null,
+};
+
 /**
  * Gets information about a Pony.fm track.
  *
@@ -101,5 +106,6 @@ export async function getPonyFmTrack(url: URL, signal?: AbortSignal): Promise<Vi
 		url: trackData.url,
 		title: trackData.title,
 		duration: { seconds: Math.floor(Number.parseFloat(trackData.duration)) },
+		metaSource,
 	};
 }
diff --git a/src/actions/network/getSoundCloudTrack.ts b/src/actions/network/getSoundCloudTrack.ts
@@ -1,4 +1,4 @@
-import type { VideoDetails } from "../getVideoDetails.js";
+import type { VideoDetails, VideoMetaSource } from "../getVideoDetails.js";
 import type { Song } from "soundcloud-scraper";
 import { Client as SoundCloudClient } from "soundcloud-scraper";
 import { richErrorMessage } from "../../helpers/richErrorMessage.js";
@@ -7,6 +7,11 @@ import { VideoError } from "../../errors/VideoError.js";
 
 const logger = useLogger();
 
+const metaSource: Readonly<VideoMetaSource> = {
+	platformName: "soundcloud",
+	alternative: null,
+};
+
 /**
  * Gets information about a SoundCloud track.
  *
@@ -49,5 +54,6 @@ export async function getSoundCloudTrack(url: URL, signal?: AbortSignal): Promis
 		url: song.url,
 		title: song.title,
 		duration: { seconds: Math.floor(song.duration / 1000) },
+		metaSource,
 	};
 }