From 0906021dd5794c55f8eb2b5bcfa113a656eedf04 Mon Sep 17 00:00:00 2001 From: Andrea Barzaghi <54246837+dbrn@users.noreply.github.com> Date: Mon, 16 Nov 2020 22:48:18 +0100 Subject: [PATCH] Fixed a couple of bugs Fixed a bug that prevented the correct parsing of some JSON objects. Added a try-catch in the removeHtml function. --- lib/util.js | 114 ++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 80 insertions(+), 34 deletions(-) diff --git a/lib/util.js b/lib/util.js index 2cd5641..2fdd519 100644 --- a/lib/util.js +++ b/lib/util.js @@ -1,17 +1,18 @@ -const ENTITIES = require('html-entities').AllHtmlEntities; +const ENTITIES = require("html-entities").AllHtmlEntities; -const PLAYLIST_URL = 'https://www.youtube.com/playlist?list='; +const PLAYLIST_URL = "https://www.youtube.com/playlist?list="; // eslint-disable-next-line max-len -const DEFAULT_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36'; -const DEFAULT_HEADERS = { 'user-agent': DEFAULT_USER_AGENT }; +const DEFAULT_USER_AGENT = + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36"; +const DEFAULT_HEADERS = { "user-agent": DEFAULT_USER_AGENT }; const DEFAULT_OPTIONS = { limit: 100, headers: DEFAULT_HEADERS }; // Guarantee that all arguments are valid exports.checkArgs = (linkOrId, options) => { // Validation if (!linkOrId) { - throw new Error('linkOrId is mandatory'); + throw new Error("linkOrId is mandatory"); } // Normalisation @@ -20,16 +21,23 @@ exports.checkArgs = (linkOrId, options) => { return obj; }; -exports.URLquery = '&hl=en&disable_polymer=true'; +exports.URLquery = "&hl=en&disable_polymer=true"; exports.getGeneralInfo = (body, plistID) => { const dataObject = getDataObject(body); const description = dataObject.metadata.playlistMetadataRenderer.description; - const authorChannel = dataObject.sidebar.playlistSidebarRenderer.items[1].playlistSidebarSecondaryInfoRenderer.videoOwner.videoOwnerRenderer.title.runs[0].navigationEndpoint.browseEndpoint.canonicalBaseUrl; + const authorChannel = + dataObject.sidebar.playlistSidebarRenderer.items[1] + .playlistSidebarSecondaryInfoRenderer.videoOwner.videoOwnerRenderer.title + .runs[0].navigationEndpoint.browseEndpoint.canonicalBaseUrl; let lastUpdated = null; try { - lastUpdated = dataObject.sidebar.playlistSidebarRenderer.items[0].playlistSidebarPrimaryInfoRenderer.stats[2].runs[1].text || dataObject.sidebar.playlistSidebarRenderer.items[0].playlistSidebarPrimaryInfoRenderer.stats[2].runs[1].text; + lastUpdated = + dataObject.sidebar.playlistSidebarRenderer.items[0] + .playlistSidebarPrimaryInfoRenderer.stats[2].runs[1].text || + dataObject.sidebar.playlistSidebarRenderer.items[0] + .playlistSidebarPrimaryInfoRenderer.stats[2].runs[1].text; } catch { lastUpdated = "Today"; } @@ -40,14 +48,29 @@ exports.getGeneralInfo = (body, plistID) => { title: removeHtml(dataObject.metadata.playlistMetadataRenderer.title), // visibility: importantTxt.includes('data-tooltip-text="') ? 'link only' : 'everyone', description: removeHtml(description), - total_items: Number(dataObject.sidebar.playlistSidebarRenderer.items[0].playlistSidebarPrimaryInfoRenderer.stats[0].runs[0].text), - views: Number(dataObject.sidebar.playlistSidebarRenderer.items[0].playlistSidebarPrimaryInfoRenderer.stats[1].simpleText.replace("views", "").trim().replace(/,/ig, "")), + total_items: Number( + dataObject.sidebar.playlistSidebarRenderer.items[0] + .playlistSidebarPrimaryInfoRenderer.stats[0].runs[0].text + ), + views: Number( + dataObject.sidebar.playlistSidebarRenderer.items[0].playlistSidebarPrimaryInfoRenderer.stats[1].simpleText + .replace("views", "") + .trim() + .replace(/,/gi, "") + ), last_updated: lastUpdated, author: { - id: authorChannel.replace(/\/c\//ig, ""), - name: removeHtml(dataObject.sidebar.playlistSidebarRenderer.items[1].playlistSidebarSecondaryInfoRenderer.videoOwner.videoOwnerRenderer.title.runs[0].text), - avatar: dataObject.sidebar.playlistSidebarRenderer.items[1].playlistSidebarSecondaryInfoRenderer.videoOwner.videoOwnerRenderer.thumbnail.thumbnails[2].url, - user: authorChannel.replace(/\/c\//ig, ""), + id: authorChannel.replace(/\/c\//gi, ""), + name: removeHtml( + dataObject.sidebar.playlistSidebarRenderer.items[1] + .playlistSidebarSecondaryInfoRenderer.videoOwner.videoOwnerRenderer + .title.runs[0].text + ), + avatar: + dataObject.sidebar.playlistSidebarRenderer.items[1] + .playlistSidebarSecondaryInfoRenderer.videoOwner.videoOwnerRenderer + .thumbnail.thumbnails[2].url, + user: authorChannel.replace(/\/c\//gi, ""), channel_url: `https://www.youtube.com${authorChannel}`, user_url: `https://www.youtube.com${authorChannel}`, }, @@ -58,13 +81,16 @@ exports.getGeneralInfo = (body, plistID) => { }; // Splits out the video container -exports.getVideoContainers = body => { +exports.getVideoContainers = (body) => { const dataObject = getDataObject(body); - const videos = dataObject.contents.twoColumnBrowseResultsRenderer.tabs[0].tabRenderer.content.sectionListRenderer.contents[0].itemSectionRenderer.contents[0].playlistVideoListRenderer.contents; + const videos = + dataObject.contents.twoColumnBrowseResultsRenderer.tabs[0].tabRenderer + .content.sectionListRenderer.contents[0].itemSectionRenderer.contents[0] + .playlistVideoListRenderer.contents; return [...videos]; -} +}; -exports.buildVideoObject = videoObject => { +exports.buildVideoObject = (videoObject) => { return { id: videoObject.playlistVideoRenderer.videoId, url: `https://www.youtube.com/watch?v=${videoObject.playlistVideoRenderer.videoId}`, @@ -73,34 +99,54 @@ exports.buildVideoObject = videoObject => { duration: videoObject.playlistVideoRenderer.lengthText.simpleText, author: { name: videoObject.playlistVideoRenderer.shortBylineText.runs[0].text, - ref: `https://www.youtube.com/${videoObject.playlistVideoRenderer.shortBylineText.runs[0].navigationEndpoint.commandMetadata.url}` + ref: `https://www.youtube.com/${videoObject.playlistVideoRenderer.shortBylineText.runs[0].navigationEndpoint.commandMetadata.url}`, }, }; }; // Taken from https://github.com/fent/node-ytdl-core/ -const between = exports.between = (haystack, left, right) => { +const between = (exports.between = (haystack, left, right) => { let pos; pos = haystack.indexOf(left); - if (pos === -1) { return ''; } + if (pos === -1) { + return ""; + } haystack = haystack.slice(pos + left.length); - if (!right) { return haystack; } + if (!right) { + return haystack; + } pos = haystack.indexOf(right); - if (pos === -1) { return ''; } + if (pos === -1) { + return ""; + } haystack = haystack.slice(0, pos); return haystack; -}; +}); // Cleans up html text -const removeHtml = exports.removeHtml = string => new ENTITIES().decode( - string.replace(/\n\r?/g, ' ') - .replace(/\s*<\s*br\s*\/?\s*>\s*/gi, '\n') - .replace(/<\s*\/\s*p\s*>\s*<\s*p[^>]*>/gi, '\n') - .replace(/<.*?>/gi, '')).trim(); +const removeHtml = (exports.removeHtml = (string) => { + try { + return new ENTITIES() + .decode( + string + .replace(/\n\r?/g, " ") + .replace(/\s*<\s*br\s*\/?\s*>\s*/gi, "\n") + .replace(/<\s*\/\s*p\s*>\s*<\s*p[^>]*>/gi, "\n") + .replace(/<.*?>/gi, "") + ) + .trim(); + } catch { + return string; + } +}); -const getDataObject = body => { - let scriptBody = between(body, 'window["ytInitialData"] = ', 'window["ytInitialPlayerResponse"] = null;'); - scriptBody = scriptBody.trim().replace(";", ""); - const scriptBodyObject = JSON.parse(scriptBody); +const getDataObject = (body) => { + let scriptBody = between( + body, + 'window["ytInitialData"] = ', + 'window["ytInitialPlayerResponse"] = null;' + ); + scriptBody = scriptBody.trim().replace(/;/gi, ""); + let scriptBodyObject = JSON.parse(scriptBody); return scriptBodyObject; -} +};