diff --git a/databases/_sponsorTimes.db.sql b/databases/_sponsorTimes.db.sql index 38ed203f..1ab52105 100644 --- a/databases/_sponsorTimes.db.sql +++ b/databases/_sponsorTimes.db.sql @@ -38,5 +38,6 @@ CREATE TABLE IF NOT EXISTS "config" ( ); CREATE EXTENSION IF NOT EXISTS pgcrypto; --!sqlite-ignore +CREATE EXTENSION IF NOT EXISTS pg_trgm; --!sqlite-ignore COMMIT; \ No newline at end of file diff --git a/src/databases/databases.ts b/src/databases/databases.ts index 96a5b5f8..5e2ae0e4 100644 --- a/src/databases/databases.ts +++ b/src/databases/databases.ts @@ -4,7 +4,6 @@ import { Mysql } from "./Mysql"; import { Postgres } from "./Postgres"; import { IDatabase } from "./IDatabase"; - let db: IDatabase; let privateDB: IDatabase; if (config.mysql) { @@ -68,6 +67,15 @@ async function initDb(): Promise { // Attach private db to main db (db as Sqlite).attachDatabase(config.privateDB, "privateDB"); } + + if (config.mode === "mirror" && db instanceof Postgres) { + const tables = config?.dumpDatabase?.tables ?? []; + const tableNames = tables.map(table => table.name); + for (const table of tableNames) { + const filePath = `${config?.dumpDatabase?.postgresExportPath}/${table}.csv`; + await db.prepare("run", `COPY "${table}" FROM '${filePath}' WITH (FORMAT CSV, HEADER true);`); + } + } } export { diff --git a/src/index.ts b/src/index.ts index 8c572175..d9451789 100644 --- a/src/index.ts +++ b/src/index.ts @@ -12,6 +12,9 @@ async function init() { }); await initDb(); + // edge case clause for creating compatible .db files, do not enable + if (config.mode === "init-db-and-exit") process.exit(0); + // do not enable init-db-only mode for usage. (global as any).HEADCOMMIT = config.mode === "development" ? "development" : config.mode === "test" ? "test" : getCommit() as string; diff --git a/src/routes/dumpDatabase.ts b/src/routes/dumpDatabase.ts index e876449e..ba85f00f 100644 --- a/src/routes/dumpDatabase.ts +++ b/src/routes/dumpDatabase.ts @@ -184,7 +184,7 @@ export async function redirectLink(req: Request, res: Response): Promise { res.sendStatus(404); } - await queueDump(); + if (req.query.generate !== "false") await queueDump(); } function updateQueueTime(): void { diff --git a/test/cases/getLockCategories.ts b/test/cases/getLockCategories.ts index b8285bf8..9666bdaa 100644 --- a/test/cases/getLockCategories.ts +++ b/test/cases/getLockCategories.ts @@ -2,6 +2,7 @@ import { getHash } from "../../src/utils/getHash"; import { db } from "../../src/databases/databases"; import assert from "assert"; import { client } from "../utils/httpClient"; +import { mixedDeepEquals } from "../utils/partialDeepEquals"; const endpoint = "/api/lockCategories"; const getLockCategories = (videoID: string) => client.get(endpoint, { params: { videoID } }); const getLockCategoriesWithService = (videoID: string, service: string) => client.get(endpoint, { params: { videoID, service } }); @@ -12,13 +13,13 @@ describe("getLockCategories", () => { await db.prepare("run", insertVipUserQuery, [getHash("getLockCategoriesVIP")]); const insertLockCategoryQuery = 'INSERT INTO "lockCategories" ("userID", "videoID", "category", "reason", "service") VALUES (?, ?, ?, ?, ?)'; - await db.prepare("run", insertLockCategoryQuery, [getHash("getLockCategoriesVIP"), "getLock1", "sponsor", "1-short", "YouTube"]); - await db.prepare("run", insertLockCategoryQuery, [getHash("getLockCategoriesVIP"), "getLock1", "interaction", "1-longer-reason", "YouTube"]); + await db.prepare("run", insertLockCategoryQuery, [getHash("getLockCategoriesVIP"), "getLockCategory1", "sponsor", "1-short", "YouTube"]); + await db.prepare("run", insertLockCategoryQuery, [getHash("getLockCategoriesVIP"), "getLockCategory1", "interaction", "1-longer-reason", "YouTube"]); - await db.prepare("run", insertLockCategoryQuery, [getHash("getLockCategoriesVIP"), "getLock2", "preview", "2-reason", "YouTube"]); + await db.prepare("run", insertLockCategoryQuery, [getHash("getLockCategoriesVIP"), "getLockCategory2", "preview", "2-reason", "YouTube"]); - await db.prepare("run", insertLockCategoryQuery, [getHash("getLockCategoriesVIP"), "getLock3", "nonmusic", "3-reason", "PeerTube"]); - await db.prepare("run", insertLockCategoryQuery, [getHash("getLockCategoriesVIP"), "getLock3", "sponsor", "3-reason", "YouTube"]); + await db.prepare("run", insertLockCategoryQuery, [getHash("getLockCategoriesVIP"), "getLockCategory3", "nonmusic", "3-reason", "PeerTube"]); + await db.prepare("run", insertLockCategoryQuery, [getHash("getLockCategoriesVIP"), "getLockCategory3", "sponsor", "3-reason", "YouTube"]); }); it("Should update the database version when starting the application", async () => { @@ -27,7 +28,7 @@ describe("getLockCategories", () => { }); it("Should be able to get multiple locks", (done) => { - getLockCategories("getLock1") + getLockCategories("getLockCategory1") .then(res => { assert.strictEqual(res.status, 200); const expected = { @@ -37,14 +38,14 @@ describe("getLockCategories", () => { ], reason: "1-longer-reason" }; - assert.deepStrictEqual(res.data, expected); + assert.ok(mixedDeepEquals(res.data, expected)); done(); }) .catch(err => done(err)); }); it("Should be able to get single locks", (done) => { - getLockCategories("getLock2") + getLockCategories("getLockCategory2") .then(res => { assert.strictEqual(res.status, 200); const expected = { @@ -60,7 +61,7 @@ describe("getLockCategories", () => { }); it("should return 404 if no lock exists", (done) => { - getLockCategories("getLockNull") + getLockCategories("getLockCategoryNull") .then(res => { assert.strictEqual(res.status, 404); done(); @@ -78,7 +79,7 @@ describe("getLockCategories", () => { }); it("Should be able to get multiple locks with service", (done) => { - getLockCategoriesWithService("getLock1", "YouTube") + getLockCategoriesWithService("getLockCategory1", "YouTube") .then(res => { assert.strictEqual(res.status, 200); const expected = { @@ -88,14 +89,14 @@ describe("getLockCategories", () => { ], reason: "1-longer-reason" }; - assert.deepStrictEqual(res.data, expected); + assert.ok(mixedDeepEquals(res.data, expected)); done(); }) .catch(err => done(err)); }); it("Should be able to get single locks with service", (done) => { - getLockCategoriesWithService("getLock3", "PeerTube") + getLockCategoriesWithService("getLockCategory3", "PeerTube") .then(res => { assert.strictEqual(res.status, 200); const expected = { @@ -111,7 +112,7 @@ describe("getLockCategories", () => { }); it("Should be able to get single locks with service", (done) => { - getLockCategoriesWithService("getLock3", "Youtube") + getLockCategoriesWithService("getLockCategory3", "Youtube") .then(res => { assert.strictEqual(res.status, 200); const expected = { @@ -127,7 +128,7 @@ describe("getLockCategories", () => { }); it("should return result from Youtube service if service not match", (done) => { - getLockCategoriesWithService("getLock3", "Dailymotion") + getLockCategoriesWithService("getLockCategory3", "Dailymotion") .then(res => { assert.strictEqual(res.status, 200); const expected = { diff --git a/test/cases/postSkipSegments.ts b/test/cases/postSkipSegments.ts index abc7ce73..27c6d3c7 100644 --- a/test/cases/postSkipSegments.ts +++ b/test/cases/postSkipSegments.ts @@ -1,6 +1,6 @@ import { config } from "../../src/config"; import { getHash } from "../../src/utils/getHash"; -import { partialDeepEquals } from "../utils/partialDeepEquals"; +import { partialDeepEquals, arrayDeepEquals } from "../utils/partialDeepEquals"; import { db } from "../../src/databases/databases"; import { ImportMock } from "ts-mock-imports"; import * as YouTubeAPIModule from "../../src/utils/youtubeApi"; @@ -29,7 +29,7 @@ describe("postSkipSegments", () => { const submitUserOneHash = getHash(submitUserOne); const submitVIPuser = `VIPPostSkipUser${".".repeat(16)}`; - const warnVideoID = "dQw4w9WgXcF"; + const warnVideoID = "postSkip2"; const badInputVideoID = "dQw4w9WgXcQ"; const queryDatabase = (videoID: string) => db.prepare("get", `SELECT "startTime", "endTime", "locked", "category" FROM "sponsorTimes" WHERE "videoID" = ?`, [videoID]); @@ -91,7 +91,7 @@ describe("postSkipSegments", () => { }); it("Should be able to submit a single time (Params method)", (done) => { - const videoID = "dQw4w9WgXcR"; + const videoID = "postSkip1"; postSkipSegmentParam({ videoID, startTime: 2, @@ -125,7 +125,7 @@ describe("postSkipSegments", () => { }); it("Should be able to submit a single time (JSON method)", (done) => { - const videoID = "dQw4w9WgXcF"; + const videoID = "postSkip2"; postSkipSegmentJSON({ userID: submitUserOne, videoID, @@ -150,7 +150,7 @@ describe("postSkipSegments", () => { }); it("Should be able to submit a single time with an action type (JSON method)", (done) => { - const videoID = "dQw4w9WgXcV"; + const videoID = "postSkip3"; postSkipSegmentJSON({ userID: submitUserOne, videoID, @@ -176,7 +176,7 @@ describe("postSkipSegments", () => { }); it("Should not be able to submit an intro with mute action type (JSON method)", (done) => { - const videoID = "dQw4w9WgXpQ"; + const videoID = "postSkip4"; postSkipSegmentJSON({ userID: submitUserOne, videoID, @@ -196,7 +196,7 @@ describe("postSkipSegments", () => { }); it("Should be able to submit a single time with a duration from the YouTube API (JSON method)", (done) => { - const videoID = "dQw4w9WgXZX"; + const videoID = "postSkip5"; postSkipSegmentJSON({ userID: submitUserOne, videoID, @@ -222,7 +222,7 @@ describe("postSkipSegments", () => { }); it("Should be able to submit a single time with a precise duration close to the one from the YouTube API (JSON method)", (done) => { - const videoID = "dQw4w9WgXZH"; + const videoID = "postSkip6"; postSkipSegmentJSON({ userID: submitUserOne, videoID, @@ -331,7 +331,7 @@ describe("postSkipSegments", () => { }); it("Should be able to submit a single time under a different service (JSON method)", (done) => { - const videoID = "dQw4w9WgXcG"; + const videoID = "postSkip7"; postSkipSegmentJSON({ userID: submitUserOne, videoID, @@ -383,7 +383,7 @@ describe("postSkipSegments", () => { }); it("Should be able to submit multiple times (JSON method)", (done) => { - const videoID = "dQw4w9WgXcT"; + const videoID = "postSkip11"; postSkipSegmentJSON({ userID: submitUserOne, videoID, @@ -407,14 +407,14 @@ describe("postSkipSegments", () => { endTime: 60, category: "intro" }]; - assert.deepStrictEqual(rows, expected); + assert.ok(arrayDeepEquals(rows, expected)); done(); }) .catch(err => done(err)); }).timeout(5000); it("Should allow multiple times if total is under 80% of video(JSON method)", (done) => { - const videoID = "L_jWHffIx5E"; + const videoID = "postSkip9"; postSkipSegmentJSON({ userID: submitUserOne, videoID, @@ -452,7 +452,7 @@ describe("postSkipSegments", () => { endTime: 170, category: "sponsor" }]; - assert.deepStrictEqual(rows, expected); + assert.ok(arrayDeepEquals(rows, expected)); done(); }) .catch(err => done(err)); @@ -505,20 +505,20 @@ describe("postSkipSegments", () => { .then(async res => { assert.strictEqual(res.status, 403); const expected = [{ - category: "sponsor", - startTime: 2000, - endTime: 4000 + category: "interaction", + startTime: 0, + endTime: 1000 }, { - category: "sponsor", - startTime: 1500, - endTime: 2750 + category: "interaction", + startTime: 1001, + endTime: 1005 }, { - category: "sponsor", - startTime: 4050, - endTime: 4750 + category: "interaction", + startTime: 0, + endTime: 5000 }]; - const rows = await queryDatabase(videoID); - assert.notDeepStrictEqual(rows, expected); + const rows = await db.prepare("all", `SELECT "category", "startTime", "endTime" FROM "sponsorTimes" WHERE "videoID" = ?`, [videoID]); + assert.ok(arrayDeepEquals(rows, expected)); done(); }) .catch(err => done(err)); diff --git a/test/utils/partialDeepEquals.ts b/test/utils/partialDeepEquals.ts index 6571794c..200252a2 100644 --- a/test/utils/partialDeepEquals.ts +++ b/test/utils/partialDeepEquals.ts @@ -21,4 +21,35 @@ export const partialDeepEquals = (actual: Record, expected: Record< } } return true; -}; \ No newline at end of file +}; + +export const arrayDeepEquals = (actual: Record, expected: Record, print = true): boolean => { + if (actual.length !== expected.length) return false; + let flag = true; + const actualString = JSON.stringify(actual); + const expectedString = JSON.stringify(expected); + // check every value in arr1 for match in arr2 + actual.every((value: any) => { if (flag && !expectedString.includes(JSON.stringify(value))) flag = false; }); + // check arr2 for match in arr1 + expected.every((value: any) => { if (flag && !actualString.includes(JSON.stringify(value))) flag = false; }); + + if (!flag && print) printActualExpected(actual, expected); + return flag; +}; + +export const mixedDeepEquals = (actual: Record, expected: Record, print = true): boolean => { + for (const [ key, value ] of Object.entries(expected)) { + // if value is object or array, recurse + if (Array.isArray(value)) { + if (!arrayDeepEquals(actual?.[key], value, false)) { + if (print) printActualExpected(actual, expected); + return false; + } + } + else if (actual?.[key] !== value) { + if (print) printActualExpected(actual, expected); + return false; + } + } + return true; +};