Merge pull request #389 from mchangrh/replication-mirror

changes for mirror mode

databases/_sponsorTimes.db.sql

@@ -38,5 +38,6 @@ CREATE TABLE IF NOT EXISTS "config" (
 );
 
 CREATE EXTENSION IF NOT EXISTS pgcrypto; --!sqlite-ignore
+CREATE EXTENSION IF NOT EXISTS pg_trgm; --!sqlite-ignore
 
 COMMIT;

src/databases/databases.ts

@@ -4,7 +4,6 @@ import { Mysql } from "./Mysql";
 import { Postgres } from "./Postgres";
 import { IDatabase } from "./IDatabase";
 
-
 let db: IDatabase;
 let privateDB: IDatabase;
 if (config.mysql) {
@@ -68,6 +67,15 @@ async function initDb(): Promise<void> {
         // Attach private db to main db
         (db as Sqlite).attachDatabase(config.privateDB, "privateDB");
     }
+
+    if (config.mode === "mirror" && db instanceof Postgres) {
+        const tables = config?.dumpDatabase?.tables ?? [];
+        const tableNames = tables.map(table => table.name);
+        for (const table of tableNames) {
+            const filePath = `${config?.dumpDatabase?.postgresExportPath}/${table}.csv`;
+            await db.prepare("run", `COPY "${table}" FROM '${filePath}' WITH (FORMAT CSV, HEADER true);`);
+        }
+    }
 }
 
 export {

src/index.ts

@@ -12,6 +12,9 @@ async function init() {
     });
 
     await initDb();
+    // edge case clause for creating compatible .db files, do not enable
+    if (config.mode === "init-db-and-exit") process.exit(0);
+    // do not enable init-db-only mode for usage.
     (global as any).HEADCOMMIT = config.mode === "development" ? "development"
         : config.mode === "test" ? "test"
             : getCommit() as string;

src/routes/dumpDatabase.ts

@@ -184,7 +184,7 @@ export async function redirectLink(req: Request, res: Response): Promise<void> {
         res.sendStatus(404);
     }
 
-    await queueDump();
+    if (req.query.generate !== "false") await queueDump();
 }
 
 function updateQueueTime(): void {

test/cases/getLockCategories.ts

@@ -2,6 +2,7 @@ import { getHash } from "../../src/utils/getHash";
 import { db } from "../../src/databases/databases";
 import assert from "assert";
 import { client } from "../utils/httpClient";
+import { mixedDeepEquals } from "../utils/partialDeepEquals";
 const endpoint = "/api/lockCategories";
 const getLockCategories = (videoID: string) => client.get(endpoint, { params: { videoID } });
 const getLockCategoriesWithService = (videoID: string, service: string) => client.get(endpoint, { params: { videoID, service } });
@@ -12,13 +13,13 @@ describe("getLockCategories", () => {
         await db.prepare("run", insertVipUserQuery, [getHash("getLockCategoriesVIP")]);
 
         const insertLockCategoryQuery = 'INSERT INTO "lockCategories" ("userID", "videoID", "category", "reason", "service") VALUES (?, ?, ?, ?, ?)';
-        await db.prepare("run", insertLockCategoryQuery, [getHash("getLockCategoriesVIP"), "getLock1", "sponsor", "1-short", "YouTube"]);
-        await db.prepare("run", insertLockCategoryQuery, [getHash("getLockCategoriesVIP"), "getLock1", "interaction", "1-longer-reason", "YouTube"]);
+        await db.prepare("run", insertLockCategoryQuery, [getHash("getLockCategoriesVIP"), "getLockCategory1", "sponsor", "1-short", "YouTube"]);
+        await db.prepare("run", insertLockCategoryQuery, [getHash("getLockCategoriesVIP"), "getLockCategory1", "interaction", "1-longer-reason", "YouTube"]);
 
-        await db.prepare("run", insertLockCategoryQuery, [getHash("getLockCategoriesVIP"), "getLock2", "preview", "2-reason", "YouTube"]);
+        await db.prepare("run", insertLockCategoryQuery, [getHash("getLockCategoriesVIP"), "getLockCategory2", "preview", "2-reason", "YouTube"]);
 
-        await db.prepare("run", insertLockCategoryQuery, [getHash("getLockCategoriesVIP"), "getLock3", "nonmusic", "3-reason", "PeerTube"]);
-        await db.prepare("run", insertLockCategoryQuery, [getHash("getLockCategoriesVIP"), "getLock3", "sponsor", "3-reason", "YouTube"]);
+        await db.prepare("run", insertLockCategoryQuery, [getHash("getLockCategoriesVIP"), "getLockCategory3", "nonmusic", "3-reason", "PeerTube"]);
+        await db.prepare("run", insertLockCategoryQuery, [getHash("getLockCategoriesVIP"), "getLockCategory3", "sponsor", "3-reason", "YouTube"]);
     });
 
     it("Should update the database version when starting the application", async () => {
@@ -27,7 +28,7 @@ describe("getLockCategories", () => {
     });
 
     it("Should be able to get multiple locks", (done) => {
-        getLockCategories("getLock1")
+        getLockCategories("getLockCategory1")
             .then(res => {
                 assert.strictEqual(res.status, 200);
                 const expected = {
@@ -37,14 +38,14 @@ describe("getLockCategories", () => {
                     ],
                     reason: "1-longer-reason"
                 };
-                assert.deepStrictEqual(res.data, expected);
+                assert.ok(mixedDeepEquals(res.data, expected));
                 done();
             })
             .catch(err => done(err));
     });
 
     it("Should be able to get single locks", (done) => {
-        getLockCategories("getLock2")
+        getLockCategories("getLockCategory2")
             .then(res => {
                 assert.strictEqual(res.status, 200);
                 const expected = {
@@ -60,7 +61,7 @@ describe("getLockCategories", () => {
     });
 
     it("should return 404 if no lock exists", (done) => {
-        getLockCategories("getLockNull")
+        getLockCategories("getLockCategoryNull")
             .then(res => {
                 assert.strictEqual(res.status, 404);
                 done();
@@ -78,7 +79,7 @@ describe("getLockCategories", () => {
     });
 
     it("Should be able to get multiple locks with service", (done) => {
-        getLockCategoriesWithService("getLock1", "YouTube")
+        getLockCategoriesWithService("getLockCategory1", "YouTube")
             .then(res => {
                 assert.strictEqual(res.status, 200);
                 const expected = {
@@ -88,14 +89,14 @@ describe("getLockCategories", () => {
                     ],
                     reason: "1-longer-reason"
                 };
-                assert.deepStrictEqual(res.data, expected);
+                assert.ok(mixedDeepEquals(res.data, expected));
                 done();
             })
             .catch(err => done(err));
     });
 
     it("Should be able to get single locks with service", (done) => {
-        getLockCategoriesWithService("getLock3", "PeerTube")
+        getLockCategoriesWithService("getLockCategory3", "PeerTube")
             .then(res => {
                 assert.strictEqual(res.status, 200);
                 const expected = {
@@ -111,7 +112,7 @@ describe("getLockCategories", () => {
     });
 
     it("Should be able to get single locks with service", (done) => {
-        getLockCategoriesWithService("getLock3", "Youtube")
+        getLockCategoriesWithService("getLockCategory3", "Youtube")
             .then(res => {
                 assert.strictEqual(res.status, 200);
                 const expected = {
@@ -127,7 +128,7 @@ describe("getLockCategories", () => {
     });
 
     it("should return result from Youtube service if service not match", (done) => {
-        getLockCategoriesWithService("getLock3", "Dailymotion")
+        getLockCategoriesWithService("getLockCategory3", "Dailymotion")
             .then(res => {
                 assert.strictEqual(res.status, 200);
                 const expected = {

test/cases/postSkipSegments.ts

@@ -1,6 +1,6 @@
 import { config } from "../../src/config";
 import { getHash } from "../../src/utils/getHash";
-import { partialDeepEquals } from "../utils/partialDeepEquals";
+import { partialDeepEquals, arrayDeepEquals } from "../utils/partialDeepEquals";
 import { db } from "../../src/databases/databases";
 import { ImportMock } from "ts-mock-imports";
 import * as YouTubeAPIModule from "../../src/utils/youtubeApi";
@@ -29,7 +29,7 @@ describe("postSkipSegments", () => {
 
     const submitUserOneHash = getHash(submitUserOne);
     const submitVIPuser = `VIPPostSkipUser${".".repeat(16)}`;
-    const warnVideoID = "dQw4w9WgXcF";
+    const warnVideoID = "postSkip2";
     const badInputVideoID = "dQw4w9WgXcQ";
 
     const queryDatabase = (videoID: string) => db.prepare("get", `SELECT "startTime", "endTime", "locked", "category" FROM "sponsorTimes" WHERE "videoID" = ?`, [videoID]);
@@ -91,7 +91,7 @@ describe("postSkipSegments", () => {
     });
 
     it("Should be able to submit a single time (Params method)", (done) => {
-        const videoID = "dQw4w9WgXcR";
+        const videoID = "postSkip1";
         postSkipSegmentParam({
             videoID,
             startTime: 2,
@@ -125,7 +125,7 @@ describe("postSkipSegments", () => {
     });
 
     it("Should be able to submit a single time (JSON method)", (done) => {
-        const videoID = "dQw4w9WgXcF";
+        const videoID = "postSkip2";
         postSkipSegmentJSON({
             userID: submitUserOne,
             videoID,
@@ -150,7 +150,7 @@ describe("postSkipSegments", () => {
     });
 
     it("Should be able to submit a single time with an action type (JSON method)", (done) => {
-        const videoID = "dQw4w9WgXcV";
+        const videoID = "postSkip3";
         postSkipSegmentJSON({
             userID: submitUserOne,
             videoID,
@@ -176,7 +176,7 @@ describe("postSkipSegments", () => {
     });
 
     it("Should not be able to submit an intro with mute action type (JSON method)", (done) => {
-        const videoID = "dQw4w9WgXpQ";
+        const videoID = "postSkip4";
         postSkipSegmentJSON({
             userID: submitUserOne,
             videoID,
@@ -196,7 +196,7 @@ describe("postSkipSegments", () => {
     });
 
     it("Should be able to submit a single time with a duration from the YouTube API (JSON method)", (done) => {
-        const videoID = "dQw4w9WgXZX";
+        const videoID = "postSkip5";
         postSkipSegmentJSON({
             userID: submitUserOne,
             videoID,
@@ -222,7 +222,7 @@ describe("postSkipSegments", () => {
     });
 
     it("Should be able to submit a single time with a precise duration close to the one from the YouTube API (JSON method)", (done) => {
-        const videoID = "dQw4w9WgXZH";
+        const videoID = "postSkip6";
         postSkipSegmentJSON({
             userID: submitUserOne,
             videoID,
@@ -331,7 +331,7 @@ describe("postSkipSegments", () => {
     });
 
     it("Should be able to submit a single time under a different service (JSON method)", (done) => {
-        const videoID = "dQw4w9WgXcG";
+        const videoID = "postSkip7";
         postSkipSegmentJSON({
             userID: submitUserOne,
             videoID,
@@ -383,7 +383,7 @@ describe("postSkipSegments", () => {
     });
 
     it("Should be able to submit multiple times (JSON method)", (done) => {
-        const videoID = "dQw4w9WgXcT";
+        const videoID = "postSkip11";
         postSkipSegmentJSON({
             userID: submitUserOne,
             videoID,
@@ -407,14 +407,14 @@ describe("postSkipSegments", () => {
                     endTime: 60,
                     category: "intro"
                 }];
-                assert.deepStrictEqual(rows, expected);
+                assert.ok(arrayDeepEquals(rows, expected));
                 done();
             })
             .catch(err => done(err));
     }).timeout(5000);
 
     it("Should allow multiple times if total is under 80% of video(JSON method)", (done) => {
-        const videoID = "L_jWHffIx5E";
+        const videoID = "postSkip9";
         postSkipSegmentJSON({
             userID: submitUserOne,
             videoID,
@@ -452,7 +452,7 @@ describe("postSkipSegments", () => {
                     endTime: 170,
                     category: "sponsor"
                 }];
-                assert.deepStrictEqual(rows, expected);
+                assert.ok(arrayDeepEquals(rows, expected));
                 done();
             })
             .catch(err => done(err));
@@ -505,20 +505,20 @@ describe("postSkipSegments", () => {
             .then(async res => {
                 assert.strictEqual(res.status, 403);
                 const expected = [{
-                    category: "sponsor",
-                    startTime: 2000,
-                    endTime: 4000
+                    category: "interaction",
+                    startTime: 0,
+                    endTime: 1000
                 }, {
-                    category: "sponsor",
-                    startTime: 1500,
-                    endTime: 2750
+                    category: "interaction",
+                    startTime: 1001,
+                    endTime: 1005
                 }, {
-                    category: "sponsor",
-                    startTime: 4050,
-                    endTime: 4750
+                    category: "interaction",
+                    startTime: 0,
+                    endTime: 5000
                 }];
-                const rows = await queryDatabase(videoID);
-                assert.notDeepStrictEqual(rows, expected);
+                const rows = await db.prepare("all", `SELECT "category", "startTime", "endTime" FROM "sponsorTimes" WHERE "videoID" = ?`, [videoID]);
+                assert.ok(arrayDeepEquals(rows, expected));
                 done();
             })
             .catch(err => done(err));

test/utils/partialDeepEquals.ts

@@ -21,4 +21,35 @@ export const partialDeepEquals = (actual: Record<string, any>, expected: Record<
         }
     }
     return true;
-};
+};
+
+export const arrayDeepEquals = (actual: Record<string, any>, expected: Record<string, any>, print = true): boolean => {
+    if (actual.length !== expected.length) return false;
+    let flag = true;
+    const actualString = JSON.stringify(actual);
+    const expectedString = JSON.stringify(expected);
+    // check every value in arr1 for match in arr2
+    actual.every((value: any) => { if (flag && !expectedString.includes(JSON.stringify(value))) flag = false; });
+    // check arr2 for match in arr1
+    expected.every((value: any) => { if (flag && !actualString.includes(JSON.stringify(value))) flag = false; });
+
+    if (!flag && print) printActualExpected(actual, expected);
+    return flag;
+};
+
+export const mixedDeepEquals = (actual: Record<string, any>, expected: Record<string, any>, print = true): boolean => {
+    for (const [ key, value ] of Object.entries(expected)) {
+        // if value is object or array, recurse
+        if (Array.isArray(value)) {
+            if (!arrayDeepEquals(actual?.[key], value, false)) {
+                if (print) printActualExpected(actual, expected);
+                return false;
+            }
+        }
+        else if (actual?.[key] !== value) {
+            if (print) printActualExpected(actual, expected);
+            return false;
+        }
+    }
+    return true;
+};