feat(youtube): Use full-fat list diffing to watch for new plays #156

* Use superdiff to diff PlayObject lists and detect changes as well as append/prepend scenarios
* Replace YTM recently played logic with list diffing, only accept prepend-validated lists
  * On non-prepend scenarios replace existing recently played and log human readable diff

package.json

@@ -47,6 +47,7 @@
     "@astronautlabs/mdns": "^1.0.7",
     "@awaitjs/express": "^0.6.3",
     "@curvenote/ansi-to-react": "^7.0.0",
+    "@donedeal0/superdiff": "^1.1.1",
     "@fortawesome/fontawesome-svg-core": "^6.4.2",
     "@fortawesome/free-solid-svg-icons": "^6.4.2",
     "@fortawesome/react-fontawesome": "^0.2.0",

src/backend/sources/YTMusicSource.ts

@@ -6,7 +6,13 @@ import { IPlaylistDetail, ITrackDetail } from "youtube-music-ts-api/interfaces-s
 import { PlayObject } from "../../core/Atomic.js";
 import { FormatPlayObjectOptions, InternalConfig } from "../common/infrastructure/Atomic.js";
 import { YTMusicCredentials, YTMusicSourceConfig } from "../common/infrastructure/config/source/ytmusic.js";
-import { parseDurationFromTimestamp, playObjDataMatch, readJson, writeFile } from "../utils.js";
+import { parseDurationFromTimestamp, readJson, writeFile } from "../utils.js";
+import {
+    getPlaysDiff,
+    humanReadableDiff,
+    playsAreAddedOnly,
+    playsAreSortConsistent
+} from "../utils/PlayComparisonUtils.js";
 import AbstractSource, { RecentlyPlayedOptions } from "./AbstractSource.js";
 
 export default class YTMusicSource extends AbstractSource {
@@ -230,28 +236,21 @@ export default class YTMusicSource extends AbstractSource {
             newPlays = plays;
         } else {
 
-            // iterate through each play until we find one that matched the "newest" from the recently played
-            for (const [i, value] of plays.entries()) {
-                if (this.recentlyPlayed.length === 0) {
-                    // playlist was empty when we started, nothing to compare to so all tracks are new
-                    newPlays.push(value);
-                } else if (this.recentlyPlayed.length !== plays.length) { // if there is a difference in list length we need to check for consecutive repeat tracks as well
-                    const match = playObjDataMatch(value, this.recentlyPlayed[0]);
-                    if (!match) {
-                        newPlays.push(value)
-                    } else if (match && plays.length > i + 1 && playObjDataMatch(plays[i + 1], this.recentlyPlayed[0])) { // if it matches but next ALSO matches the current it's a repeat "new"
-                        // check if repeated track
-                        newPlays.push(value)
-                    } else {
-                        break;
-                    }
-                } else if (!playObjDataMatch(value, this.recentlyPlayed[0])) {
-                    // if the newest doesn't match a play then the play is new
-                    newPlays.push(value);
-                } else {
-                    // otherwise we're back known plays
-                    break;
-                }
+            if(playsAreSortConsistent(this.recentlyPlayed, plays)) {
+                return newPlays;
+            }
+            const [ok, diff, addType] = playsAreAddedOnly(this.recentlyPlayed, plays);
+            if(!ok || addType === 'insert' || addType === 'append') {
+                const playsDiff = getPlaysDiff(this.recentlyPlayed, plays)
+                const humanDiff = humanReadableDiff(this.recentlyPlayed, plays, playsDiff);
+                this.logger.warn('YTM History returned temporally inconsistent order, resetting watched history to new list.');
+                this.logger.warn(`Changes from last seen list:
+${humanDiff}`);
+                this.recentlyPlayed = plays;
+                return newPlays;
+            } else {
+                // new plays
+                newPlays = [...diff].reverse();
             }
 
             if(newPlays.length > 0) {

src/backend/tests/utils/PlayTestUtils.ts

@@ -93,3 +93,7 @@ export const generatePlay = (data: ObjectPlayData = {}, meta: PlayMeta = {}): Pl
         }
     }
 }
+
+export const generatePlays = (numberOfPlays: number, data: ObjectPlayData = {}, meta: PlayMeta = {}): PlayObject[] => {
+    return Array.from(Array(numberOfPlays), () => generatePlay(data, meta));
+}

src/backend/tests/utils/playComparisons.test.ts

@@ -0,0 +1,101 @@
+import { loggerTest } from "@foxxmd/logging";
+import { assert } from 'chai';
+import clone from "clone";
+import { describe, it } from 'mocha';
+import { playsAreAddedOnly, playsAreSortConsistent } from "../../utils/PlayComparisonUtils.js";
+import { generatePlay, generatePlays } from "./PlayTestUtils.js";
+
+const logger = loggerTest;
+
+const newPlay = generatePlay();
+
+const existingList = generatePlays(10);
+
+describe('Compare lists by order', function () {
+
+    describe('Identity', function () {
+        it('Identical lists are equal', function () {
+            const identicalList = [...existingList.map(x => clone(x))];
+            assert.isTrue(playsAreSortConsistent(existingList, identicalList));
+        });
+
+        it('Non-identical lists are not equal', function () {
+            assert.isFalse(playsAreSortConsistent(existingList, generatePlays(11)));
+        });
+
+        it('Non-identical lists with modifications are not equal', function () {
+            const modified = [...existingList.map(x => clone(x))];
+            modified.splice(2, 1, generatePlay());
+            modified[6].data.track = 'A CHANGE';
+            modified.splice(8, 0, generatePlay());
+            const modded = [...modified, generatePlay()];
+            assert.isFalse(playsAreSortConsistent(existingList, modded));
+        });
+    });
+
+    describe('Added Only', function () {
+
+        it('Non-identical lists are not add only', function () {
+            const [ok, diff, addType] = playsAreAddedOnly(existingList, generatePlays(10))
+            assert.isFalse(ok);
+        });
+
+        it('Lists with only prepended additions are detected', function () {
+            const [ok, diff, addType] = playsAreAddedOnly(existingList, [generatePlay(), generatePlay(), ...existingList])
+            assert.isTrue(ok);
+            assert.equal(addType, 'prepend');
+        });
+
+        it('Lists with only appended additions are detected', function () {
+            const [ok, diff, addType] = playsAreAddedOnly(existingList, [...existingList, generatePlay(), generatePlay()])
+            assert.isTrue(ok);
+            assert.equal(addType, 'append');
+        });
+
+        it('Lists of fixed length with prepends are correctly detected', function () {
+            const [ok, diff, addType] = playsAreAddedOnly(existingList, [generatePlay(), generatePlay(), ...existingList].slice(0, 9))
+            assert.isTrue(ok);
+            assert.equal(addType, 'prepend');
+        });
+
+        it('Lists with inserts are detected', function () {
+            const splicedList1 = [...existingList.map(x => clone(x))];
+            splicedList1.splice(4, 0, generatePlay())
+            const [ok, diff, addType] = playsAreAddedOnly(existingList, splicedList1)
+            assert.isFalse(ok)
+            //assert.equal(addType, 'insert');
+
+            const splicedList2 = [...existingList.map(x => clone(x))];
+            splicedList2.splice(2, 0, generatePlay())
+            splicedList2.splice(6, 0, generatePlay())
+            const [ok2, diff2, addType2] = playsAreAddedOnly(existingList, splicedList2)
+            assert.isFalse(ok2)
+            //assert.equal(addType2, 'insert');
+        });
+
+        it('Lists with inserts and prepends are detected as inserts', function () {
+            const splicedList = [...existingList.map(x => clone(x))];
+            splicedList.splice(2, 0, generatePlay())
+            splicedList.splice(6, 0, generatePlay())
+            const [ok, diff3, addType] = playsAreAddedOnly(existingList, [generatePlay(), generatePlay(), ...splicedList])
+            assert.isFalse(ok);
+            //assert.equal(addType, 'insert');
+        });
+
+        it('Lists with inserts and appends are detected as inserts', function () {
+            const splicedList = [...existingList.map(x => clone(x))];
+            splicedList.splice(2, 0, generatePlay())
+            splicedList.splice(6, 0, generatePlay())
+            const [ok, diff4, addType] = playsAreAddedOnly(existingList, [...splicedList, generatePlay(), generatePlay()])
+            assert.isFalse(ok);
+            //assert.equal(addType, 'insert');
+        });
+
+        it('Lists with inserts and appends and prepends are detected as inserts', function () {
+            const splicedList = [...existingList.map(x => clone(x))];
+            const [ok, diff, addType] = playsAreAddedOnly(existingList, [generatePlay(), generatePlay(), ...splicedList, generatePlay(), generatePlay()])
+            assert.isFalse(ok);
+            //assert.equal(addType, 'insert');
+        });
+    })
+});

src/backend/utils/PlayComparisonUtils.ts

@@ -0,0 +1,160 @@
+import { getListDiff } from "@donedeal0/superdiff";
+import { PlayObject } from "../../core/Atomic.js";
+import { buildTrackString } from "../../core/StringUtils.js";
+
+
+export const metaInvariantTransform = (play: PlayObject): PlayObject => {
+    const {
+        meta: {
+            trackId
+        } = {},
+    } = play;
+    return {
+        ...play,
+        meta: {
+            trackId
+        }
+    }
+}
+
+export const playDateInvariantTransform = (play: PlayObject): PlayObject => {
+    const {
+        meta: {
+            trackId
+        } = {},
+    } = play;
+    return {
+        ...play,
+        data: {
+            ...play.data,
+            playDate: undefined
+        }
+    }
+}
+
+
+export type PlayTransformer = (play: PlayObject) => PlayObject;
+export type ListTransformers = PlayTransformer[];
+
+export const defaultListTransformers: ListTransformers = [metaInvariantTransform, playDateInvariantTransform];
+
+export const getPlaysDiff = (aPlays: PlayObject[], bPlays: PlayObject[], transformers: ListTransformers = defaultListTransformers) => {
+    const cleanAPlays = transformers === undefined ? aPlays : transformers.reduce((acc: PlayObject[], curr) => acc.map(curr), aPlays);
+    const cleanBPlays = transformers === undefined ? bPlays : transformers.reduce((acc: PlayObject[], curr) => acc.map(curr), bPlays);
+
+    return getListDiff(cleanAPlays, cleanBPlays);
+}
+
+export const playsAreSortConsistent = (aPlays: PlayObject[], bPlays: PlayObject[], transformers: ListTransformers = defaultListTransformers) => {
+    const diff = getPlaysDiff(aPlays, bPlays, transformers);
+    return diff.status === 'equal';
+}
+
+export const getDiffIndexState = (results: any, index: number) => {
+    const replaced = results.diff.filter(x => (x.status === 'deleted' && x.prevIndex === index) || (x.status === 'added' && x.newIndex === index));
+    if(replaced.length === 2) {
+        return 'replaced';
+    }
+    let diff = results.diff.find(x => x.newIndex === index);
+    if(diff !== undefined) {
+        return diff.status;
+    }
+    diff = results.diff.find(x => x.prevIndex === index);
+    if(diff !== undefined) {
+        return diff.status;
+    }
+    return undefined;
+}
+
+export const playsAreAddedOnly = (aPlays: PlayObject[], bPlays: PlayObject[], transformers: ListTransformers = defaultListTransformers): [boolean, PlayObject[]?, ('append' | 'prepend' | 'insert')?] => {
+    const results = getPlaysDiff(aPlays, bPlays, transformers);
+     if(results.status === 'equal' || results.status === 'deleted') {
+        return [false];
+    }
+
+    let addType: 'insert' | 'append' | 'prepend';
+     for(const [index, play] of bPlays.entries()) {
+         const isEqual = results.diff.some(x => x.status === 'equal' && x.prevIndex === index && x.newIndex === index);
+
+         if(isEqual) {
+             continue;
+         }
+
+         const replaced = results.diff.filter(x => (x.status === 'deleted' && x.prevIndex === index) || (x.status === 'added' && x.newIndex === index));
+         if(replaced.length === 2) {
+             addType = 'insert';
+             return [false];
+         }
+
+         const moved = results.diff.some(x => x.status === 'moved' && x.newIndex === index);
+         if(moved) {
+             continue;
+         }
+
+         const added = results.diff.find(x => x.status === 'added' && x.newIndex === index);
+         if(added !== undefined) {
+
+             if(added.newIndex === 0) {
+                 addType = 'prepend';
+             } else if(added.newIndex === bPlays.length - 1) {
+                 addType = 'append';
+             } else {
+                 const prevDiff = getDiffIndexState(results, index - 1);
+                 const nextDiff = getDiffIndexState(results, index + 1);
+                 if(prevDiff !== 'added' && nextDiff !== 'added') {
+                     addType = 'insert';
+                     return [false];
+                 } else if(addType !== 'prepend' && nextDiff !== 'added') {
+                     addType = 'insert';
+                     return [false];
+                 } else if(addType === 'prepend' && prevDiff !== 'added') {
+                     return [false];
+                 }
+             }
+         }
+     }
+    const added = results.diff.filter(x => x.status === 'added');
+    return [addType !== 'insert', added.map(x => bPlays[x.newIndex]), addType];
+}
+
+export const humanReadableDiff = (aPlay: PlayObject[], bPlay: PlayObject[], result: any): string => {
+    const changes: [string, string?][] = [];
+    for(const [index, play] of bPlay.entries()) {
+        const ab: [string, string?] = [`${index + 1}. ${buildTrackString(play)}`];
+
+        const isEqual = result.diff.some(x => x.status === 'equal' && x.prevIndex === index && x.newIndex === index);
+        if(!isEqual) {
+            const moved = result.diff.filter(x => x.status === 'moved' && x.newIndex === index);
+            if(moved.length > 0) {
+                ab.push(`Moved -  Originally at ${moved[0].prevIndex + 1}`);
+            } else {
+                // look for replaced first
+                const replaced = result.diff.filter(x => (x.status === 'deleted' && x.prevIndex === index) || (x.status === 'added' && x.newIndex === index));
+                if(replaced.length === 2) {
+                    const newPlay = replaced.filter(x => x.status === 'deleted');
+                    ab.push(`Replaced - Original => ${buildTrackString( newPlay[0].value)}`);
+                } else {
+                    const added = result.diff.some(x => x.status === 'added' && x.newIndex === index);
+                    if(added) {
+                        ab.push('Added');
+                    } else {
+                        // was updated, probably??
+                        const updated = result.diff.filter(x => x.status === 'deleted' && x.prevIndex === index);
+                        if(updated.length > 0) {
+                            ab.push(`Updated - Original => ${buildTrackString( aPlay[updated[0].preIndex])}`);
+                        } else {
+                            ab.push('Should not have gotten this far!');
+                        }
+                    }
+                }
+            }
+        }
+        changes.push(ab);
+    }
+    return changes.map(([a,b]) => {
+       if(b === undefined) {
+           return a;
+       }
+       return `${a} => ${b}`;
+    }).join('\n');
+}