* Fixed duplicate creator posts

* Streamlined array.ts functions
* Updated dependencies
* Removed sleep.ts (use setTimeout from node:times/promises)
* Reduced size of YouTube payloads

.eslintrc.js

@@ -19,6 +19,7 @@ module.exports = {
   rules: {
     "@typescript-eslint/consistent-type-definitions": ["error", "type"],
     "@typescript-eslint/consistent-type-imports": "error",
+    "@typescript-eslint/no-unsafe-enum-comparison": "off",
     "no-console": "error",
     "unicorn/no-array-callback-reference": "off",
     "unicorn/no-null": "off",

package.json

@@ -4,38 +4,38 @@
     "@googleapis/youtube": "13.0.0",
     "canvas": "2.11.2",
     "compress-tag": "3.0.0",
-    "cron": "3.1.5",
+    "cron": "3.1.6",
     "csv": "6.3.5",
-    "discord.js": "14.13.0",
+    "discord.js": "14.14.1",
     "express": "4.18.2",
     "express-prom-bundle": "6.6.0",
     "glob": "10.3.10",
     "ioredis": "5.3.2",
     "lunr": "2.3.9",
     "pg": "8.11.3",
-    "pino": "8.16.1",
-    "pino-http": "8.5.0",
+    "pino": "8.16.2",
+    "pino-http": "8.5.1",
     "prom-client": "15.0.0",
-    "puppeteer": "21.4.1",
+    "puppeteer": "21.6.0",
     "puppeteer-extra": "3.3.6",
     "puppeteer-extra-plugin-stealth": "2.11.2",
     "rss-parser": "3.13.0",
-    "vega": "5.25.0"
+    "vega": "5.26.1"
   },
   "devDependencies": {
-    "@types/express": "4.17.20",
-    "@types/lunr": "2.3.6",
-    "@types/pg": "8.10.7",
-    "@typescript-eslint/eslint-plugin": "6.9.0",
-    "@typescript-eslint/parser": "6.9.0",
-    "eslint": "8.52.0",
-    "eslint-plugin-perfectionist": "2.2.0",
-    "eslint-plugin-sonarjs": "0.21.0",
-    "eslint-plugin-unicorn": "48.0.1",
+    "@types/express": "4.17.21",
+    "@types/lunr": "2.3.7",
+    "@types/pg": "8.10.9",
+    "@typescript-eslint/eslint-plugin": "6.13.2",
+    "@typescript-eslint/parser": "6.13.2",
+    "eslint": "8.55.0",
+    "eslint-plugin-perfectionist": "2.5.0",
+    "eslint-plugin-sonarjs": "0.23.0",
+    "eslint-plugin-unicorn": "49.0.0",
     "pino-pretty": "10.2.3",
-    "prettier": "3.0.3",
+    "prettier": "3.1.0",
     "shx": "0.3.4",
-    "typescript": "5.2.2"
+    "typescript": "5.3.3"
   },
   "license": "MIT",
   "name": "pedestrian",

src/commands/youtube/youtube.manager.ts

@@ -23,7 +23,7 @@ export const getChannels = (query: string) => {
     const { data } = await search.list({
       fields:
         "items(snippet(channelId,channelTitle,description,publishedAt,thumbnails,title))",
-      maxResults: 50,
+      maxResults: 25,
       part: ["snippet"],
       q: query,
       type: ["channel"],
@@ -66,7 +66,7 @@ export const getVideos = (playlistId: string) => {
   const callback = async () => {
     const { data } = await playlistItems.list({
       fields: "items(snippet(publishedAt,resourceId,title))",
-      maxResults: 50,
+      maxResults: 25,
       part: ["snippet"],
       playlistId,
     });

src/creators/post/index.ts

@@ -14,7 +14,7 @@ import type Nullable from "../../shared/nullable";
 import type { CreatorType } from "../constants";
 import type { CreatorSubscription } from "./database";
 
-import { byDate, isUnique } from "../../shared/array";
+import { by, unique } from "../../shared/array";
 import discord from "../../shared/discord";
 import loggerFactory from "../../shared/logger";
 import { isNullable } from "../../shared/nullable";
@@ -65,10 +65,10 @@ const getOptions = async (creatorSubscription: CreatorSubscription) => {
     const options = await poster(creatorDomainId);
     // Iterate options from latest to oldest
     const orderedOptions = options.sort(
-      byDate(({ timestamp }) => timestamp, "desc"),
+      by(({ timestamp }) => timestamp, "desc"),
     );
 
-    const optionsToPost = [];
+    let optionsToPost = [];
     for (const [index, option] of orderedOptions.entries()) {
       const { contentId, timestamp } = option;
 
@@ -90,6 +90,9 @@ const getOptions = async (creatorSubscription: CreatorSubscription) => {
     // If no options will be posted skip checking for posted content
     if (optionsToPost.length === 0) return optionsToPost;
 
+    // Do not post duplicate options (handles some edge cases)
+    optionsToPost = optionsToPost.filter(unique(({ contentId }) => contentId));
+
     const contentIds = optionsToPost.map(({ contentId }) => contentId);
     const postedContentIds =
       // prettier-ignore
@@ -236,7 +239,7 @@ const postAll = async (creatorSubscriptions: CreatorSubscription[]) => {
   }
 
   if (obsoleteCreatorChannelIds.length > 0) {
-    obsoleteCreatorChannelIds = obsoleteCreatorChannelIds.filter(isUnique);
+    obsoleteCreatorChannelIds = obsoleteCreatorChannelIds.filter(unique());
     const childLogger = logger.child({ obsoleteCreatorChannelIds });
 
     try {

src/creators/subscribe/index.ts

@@ -8,7 +8,7 @@ import assert from "node:assert";
 
 import type { CreatorType } from "../constants";
 
-import { isUnique } from "../../shared/array";
+import { unique } from "../../shared/array";
 import Environment from "../../shared/environment";
 import * as creatorsDatabase from "../database";
 import * as subscribeDatabase from "./database";
@@ -40,7 +40,7 @@ export const checkSubscribeRequirements = async (
   const channels = guildChannelManager.valueOf();
   const obsoleteCreatorChannelIds = creatorSubscriptions
     .map(({ creatorChannelId }) => creatorChannelId)
-    .filter(isUnique)
+    .filter(unique())
     .filter((creatorChannelId) => !channels.has(creatorChannelId));
 
   if (obsoleteCreatorChannelIds.length > 0)

src/creators/unsubscribe/index.ts

@@ -4,7 +4,7 @@ import assert from "node:assert";
 
 import type { CreatorType } from "../constants";
 
-import { isUnique } from "../../shared/array";
+import { unique } from "../../shared/array";
 import session from "./context";
 import * as database from "./database";
 import UI from "./ui";
@@ -27,7 +27,7 @@ export const unsubscribe = async (
 
   const namePromises = creatorSubscriptions
     .map(({ creatorDomainId }) => creatorDomainId)
-    .filter(isUnique)
+    .filter(unique())
     .map(async (creatorDomainId) => {
       const domainName = await name(creatorDomainId);
       return { [creatorDomainId]: domainName };

src/shared/array.ts

@@ -1,32 +1,41 @@
-import type Nullable from "./nullable";
+// region Types
+type Order = "asc" | "desc";
 
-import { isNullable } from "./nullable";
+type Sort<T> = (a: T, b: T) => number;
 
-export const byDate =
-  <T, U extends Date | Nullable | number | string>(
-    property: (element: T) => U,
-    order: "asc" | "desc" = "asc",
-  ) =>
+type By = {
+  <T, U>(property: (element: T) => U, order?: Order): Sort<T>;
+  <T>(order?: Order): Sort<T>;
+};
+// endregion
+
+export const by: By =
+  <T, U>(property?: ((element: T) => U) | Order, order?: Order) =>
   (a: T, b: T) => {
-    const aValue = property(a);
-    if (isNullable(aValue)) return 0;
-    const aDate = new Date(aValue);
-    const aTime = aDate.getTime();
-
-    const bValue = property(b);
-    if (isNullable(bValue)) return 0;
-    const bDate = new Date(bValue);
-    const bTime = bDate.getTime();
-
-    switch (order) {
-      case "asc": {
-        return aTime - bTime;
-      }
-      case "desc": {
-        return bTime - aTime;
-      }
+    if (typeof property === "string") {
+      order = property;
+      property = undefined;
     }
+
+    property ??= (element: unknown) => element as U;
+    const aProperty = property(a);
+    const bProperty = property(b);
+
+    if (aProperty === bProperty) return 0;
+    const result = aProperty > bProperty ? 1 : -1;
+
+    order ??= "asc";
+    return order === "asc" ? result : result * -1;
   };
 
-export const isUnique = <T>(value: T, index: number, array: T[]) =>
-  array.indexOf(value) === index;
+export const unique = <T, U>(property?: (element: T) => U) => {
+  const set = new Set<U>();
+  return (value: T) => {
+    property ??= (element: unknown) => element as U;
+    const valueProperty = property(value);
+
+    if (set.has(valueProperty)) return false;
+    set.add(valueProperty);
+    return true;
+  };
+};

src/shared/redis.ts

@@ -2,11 +2,11 @@ import type { Callback, Result } from "ioredis";
 
 import { Redis } from "ioredis";
 import * as crypto from "node:crypto";
+import { setTimeout } from "node:timers/promises";
 
 import Environment from "./environment";
 import loggerFactory from "./logger";
 import { isNullable } from "./nullable";
-import sleep from "./sleep";
 
 const logger = loggerFactory(module);
 
@@ -131,7 +131,7 @@ const lock = async (key: string, expireInMilliseconds: number) => {
       logger.error(error, "LOCK_ERROR");
     }
 
-    await sleep(untilLockExpires);
+    await setTimeout(untilLockExpires);
     lockObject = await tryLock(lockKey, expireInMilliseconds);
   }
 
@@ -168,7 +168,7 @@ const useLock = async <T>(
     let callbackResult = callback(lockObject);
 
     while (callbackResult instanceof Promise) {
-      const sleepPromise = sleep(expireInMilliseconds / 2);
+      const sleepPromise = setTimeout(expireInMilliseconds / 2);
       const result = await Promise.race([callbackResult, sleepPromise]);
 
       if (result === undefined) {

tsconfig.json

@@ -5,7 +5,7 @@
     "forceConsistentCasingInFileNames": true,
     "inlineSourceMap": false,
     "inlineSources": true,
-    "lib": ["es2021"],
+    "lib": ["dom", "es2021"],
     "module": "node16",
     "moduleResolution": "node16",
     "noEmit": false,