Merge pull request #8 from pkreissel/removerecommend

Removerecommend
pkreissel · Aug 3, 2024 · 894826f · 894826f
2 parents c0bec0f + d26ed08
commit 894826f
Show file tree

Hide file tree

Showing 16 changed files with 82 additions and 51 deletions.
diff --git a/.gitignore b/.gitignore
@@ -2,4 +2,5 @@ node_modules
 coverage
 .env
 
-.DS_Store
+.DS_Store
+dist
diff --git a/dist/features/FeatureStore.js b/dist/features/FeatureStore.js
@@ -51,7 +51,8 @@ class FeatureStorage extends Storage_1.default {
             return topReblogs;
         }
         else {
-            const reblogs = await (0, reblogsFeature_1.default)(api);
+            const user = await this.getIdentity();
+            const reblogs = await (0, reblogsFeature_1.default)(api, user);
             await this.set(Storage_1.Key.TOP_REBLOGS, reblogs);
             return reblogs;
         }

diff --git a/dist/features/coreServerFeature.js b/dist/features/coreServerFeature.js
@@ -39,14 +39,14 @@ async function coreServerFeature(api, user) {
         return accumulator;
     }, {});
     console.log(serverFrequ);
-    // for top 30 servers
-    const top30 = Object.keys(serverFrequ).sort((a, b) => serverFrequ[b] - serverFrequ[a]).slice(0, 30);
-    console.log("Top 30 servers: ", top30);
-    const monthlyUsers = await Promise.all(top30.map(server => getMonthlyUsers(server)));
+    // for top 20 servers
+    const top20 = Object.keys(serverFrequ).sort((a, b) => serverFrequ[b] - serverFrequ[a]).slice(0, 30);
+    console.log("Top 30 servers: ", top20);
+    const monthlyUsers = await Promise.all(top20.map(server => getMonthlyUsers(server)));
     console.log("Monthly Users: ", monthlyUsers);
-    const overrepresentedServerFrequ = top30.reduce((acc, server, index) => {
+    const overrepresentedServerFrequ = top20.reduce((acc, server, index) => {
         const activeUsers = monthlyUsers[index];
-        if (activeUsers < 1)
+        if (activeUsers < 10)
             return acc;
         const ratio = serverFrequ[server] / activeUsers;
         return { ...acc, [server]: ratio };

diff --git a/dist/features/reblogsFeature.d.ts b/dist/features/reblogsFeature.d.ts
@@ -1,2 +1,2 @@
 import { mastodon } from "masto";
-export default function getReblogsFeature(api: mastodon.rest.Client): Promise<Record<string, number>>;
+export default function getReblogsFeature(api: mastodon.rest.Client, user: mastodon.v1.Account): Promise<Record<string, number>>;
diff --git a/dist/features/reblogsFeature.js b/dist/features/reblogsFeature.js
@@ -1,10 +1,10 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-async function getReblogsFeature(api) {
+async function getReblogsFeature(api, user) {
     let results = [];
     let pages = 3;
     try {
-        for await (const page of api.v1.timelines.home.list({ limit: 80 })) {
+        for await (const page of api.v1.accounts.$select(user.id).statuses.list({ limit: 80 })) {
             results = results.concat(page);
             pages--;
             if (pages === 0 || results.length < 80) {
@@ -16,6 +16,7 @@ async function getReblogsFeature(api) {
         console.error(e);
         return {};
     }
+    console.log(results);
     const reblogFrequ = results.reduce((accumulator, status) => {
         if (status.reblog) {
             if (status.reblog.account.acct in accumulator) {
@@ -27,6 +28,7 @@ async function getReblogsFeature(api) {
         }
         return accumulator;
     }, {});
+    console.log(reblogFrequ);
     return reblogFrequ;
 }
 exports.default = getReblogsFeature;
diff --git a/dist/feeds/topPostsFeed.js b/dist/feeds/topPostsFeed.js
@@ -19,14 +19,20 @@ async function getTopPostFeed(api) {
     results = await Promise.all(servers.map(async (server) => {
         if (server === "undefined" || typeof server == "undefined" || server === "")
             return [];
-        const data = await (0, helpers_1.mastodonFetch)(server, "api/v1/timelines/public");
-        return data?.map((status) => {
+        const data = await (0, helpers_1.mastodonFetch)(server, "api/v1/trends/statuses");
+        return data?.filter(status => status?.favouritesCount > 0 || status?.reblogsCount > 0).map((status) => {
             status.topPost = true;
             return status;
         }).slice(0, 10) ?? [];
     }));
     console.log(results);
     const lastOpened = new Date((await Storage_1.default.getLastOpened() ?? 0) - 28800000);
-    return results.flat().filter((status) => new Date(status.createdAt) > lastOpened);
+    return results.flat().filter((status) => new Date(status.createdAt) > lastOpened).map((status) => {
+        const acct = status.account.acct;
+        if (acct && !acct.includes("@")) {
+            status.account.acct = `${acct}@${status.account.url.split("/")[2]}`;
+        }
+        return status;
+    });
 }
 exports.default = getTopPostFeed;
diff --git a/dist/helpers.js b/dist/helpers.js
@@ -23,12 +23,18 @@ const _transformKeys = (data, transform) => {
 };
 exports._transformKeys = _transformKeys;
 const mastodonFetch = async (server, endpoint) => {
-    const json = await axios_1.default.get(`https://${server}${endpoint}`);
-    if (!(json.status === 200) || !json.data) {
-        console.error(`Error fetching data for server ${server}:`, json);
+    try {
+        const json = await axios_1.default.get(`https://${server}${endpoint}`);
+        if (!(json.status === 200) || !json.data) {
+            console.error(`Error fetching data for server ${server}:`, json);
+            return;
+        }
+        const data = (0, exports._transformKeys)(json.data, change_case_1.camelCase);
+        return data;
+    }
+    catch (error) {
+        console.error(`Error fetching data for server ${server}:`, error);
         return;
     }
-    const data = (0, exports._transformKeys)(json.data, change_case_1.camelCase);
-    return data;
 };
 exports.mastodonFetch = mastodonFetch;
diff --git a/dist/index.js b/dist/index.js
@@ -76,7 +76,7 @@ class TheAlgorithm {
         // Add Time Penalty
         scoredFeed = scoredFeed.map((item) => {
             const seconds = Math.floor((new Date().getTime() - new Date(item.createdAt).getTime()) / 1000);
-            const timediscount = Math.pow((1 + 0.7 * 0.2), -Math.pow((seconds / 3600), 2));
+            const timediscount = Math.pow((1 + 0.05), -Math.pow((seconds / 3600), 2));
             item.value = (item.value ?? 0) * timediscount;
             return item;
         });

diff --git a/dist/scorer/feed/reblogsFeedScorer.js b/dist/scorer/feed/reblogsFeedScorer.js
@@ -9,23 +9,23 @@ class reblogsFeedScorer extends FeedScorer_1.default {
         super("reblogsFeed", "More Weight to posts that are reblogged a lot", 6);
     }
     feedExtractor(feed) {
-        return feed.reduce((obj, status) => {
+        // for each uri in the feed, count the number of times it appears
+        const feedScorer = feed.reduce((obj, status) => {
+            obj[status.uri] = (obj[status.uri] || 0) + 1;
             if (status.reblog) {
-                obj[status.reblog.uri] = (obj[status.reblog.uri] || -1) + 1;
-            }
-            else {
-                obj[status.uri] = (obj[status.uri] || -1) + 1;
+                obj[status.reblog.uri] = (obj[status.reblog.uri] || 0) + 1;
             }
             return obj;
         }, {});
+        return feedScorer;
     }
     async score(status) {
         super.score(status);
         const features = this.features;
         if (status.reblog) {
             return features[status.reblog.uri] || 0;
         }
-        return 0;
+        return features[status.uri] || 0;
     }
 }
 exports.default = reblogsFeedScorer;
diff --git a/src/features/FeatureStore.ts b/src/features/FeatureStore.ts
@@ -26,7 +26,8 @@ export default class FeatureStorage extends Storage {
         if (topReblogs != null && await this.getOpenings() % 10 < 9) {
             return topReblogs;
         } else {
-            const reblogs = await reblogsFeature(api);
+            const user = await this.getIdentity()
+            const reblogs = await reblogsFeature(api, user);
             await this.set(Key.TOP_REBLOGS, reblogs);
             return reblogs;
         }

diff --git a/src/features/coreServerFeature.ts b/src/features/coreServerFeature.ts
@@ -42,17 +42,17 @@ export default async function coreServerFeature(api: mastodon.rest.Client, user:
 
     console.log(serverFrequ)
 
-    // for top 30 servers
-    const top30 = Object.keys(serverFrequ).sort((a, b) => serverFrequ[b] - serverFrequ[a]).slice(0, 30)
+    // for top 20 servers
+    const top20 = Object.keys(serverFrequ).sort((a, b) => serverFrequ[b] - serverFrequ[a]).slice(0, 30)
 
-    console.log("Top 30 servers: ", top30)
-    const monthlyUsers = await Promise.all(top30.map(server => getMonthlyUsers(server)))
+    console.log("Top 30 servers: ", top20)
+    const monthlyUsers = await Promise.all(top20.map(server => getMonthlyUsers(server)))
 
     console.log("Monthly Users: ", monthlyUsers)
 
-    const overrepresentedServerFrequ = top30.reduce((acc, server, index) => {
+    const overrepresentedServerFrequ = top20.reduce((acc, server, index) => {
         const activeUsers = monthlyUsers[index];
-        if (activeUsers < 1) return acc;
+        if (activeUsers < 10) return acc;
         const ratio = serverFrequ[server] / activeUsers;
         return { ...acc, [server]: ratio }
     }, {})

diff --git a/src/features/reblogsFeature.ts b/src/features/reblogsFeature.ts
@@ -1,10 +1,10 @@
 import { mastodon } from "masto";
 
-export default async function getReblogsFeature(api: mastodon.rest.Client) {
+export default async function getReblogsFeature(api: mastodon.rest.Client, user: mastodon.v1.Account): Promise<Record<string, number>> {
     let results: mastodon.v1.Status[] = [];
     let pages = 3;
     try {
-        for await (const page of api.v1.timelines.home.list({ limit: 80 })) {
+        for await (const page of api.v1.accounts.$select(user.id).statuses.list({ limit: 80 })) {
             results = results.concat(page)
             pages--;
             if (pages === 0 || results.length < 80) {
@@ -15,6 +15,7 @@ export default async function getReblogsFeature(api: mastodon.rest.Client) {
         console.error(e)
         return {};
     }
+    console.log(results)
 
     const reblogFrequ = results.reduce((accumulator: Record<string, number>, status: mastodon.v1.Status) => {
         if (status.reblog) {
@@ -26,6 +27,6 @@ export default async function getReblogsFeature(api: mastodon.rest.Client) {
         }
         return accumulator
     }, {})
-
+    console.log(reblogFrequ)
     return reblogFrequ;
 }
diff --git a/src/feeds/topPostsFeed.ts b/src/feeds/topPostsFeed.ts
@@ -21,14 +21,22 @@ export default async function getTopPostFeed(api: mastodon.rest.Client): Promise
     results = await Promise.all(servers.map(async (server: string): Promise<StatusType[]> => {
         if (server === "undefined" || typeof server == "undefined" || server === "") return [];
 
-        const data = await mastodonFetch<StatusType[]>(server, "api/v1/timelines/public")
-        return data?.map((status: StatusType) => {
+        const data = await mastodonFetch<StatusType[]>(server, "api/v1/trends/statuses")
+        return data?.filter(
+            status => status?.favouritesCount > 0 || status?.reblogsCount > 0
+        ).map((status: StatusType) => {
             status.topPost = true;
             return status;
         }).slice(0, 10) ?? []
     }))
     console.log(results)
 
     const lastOpened = new Date((await Storage.getLastOpened() ?? 0) - 28800000)
-    return results.flat().filter((status: StatusType) => new Date(status.createdAt) > lastOpened)
+    return results.flat().filter((status: StatusType) => new Date(status.createdAt) > lastOpened).map((status: StatusType) => {
+        const acct = status.account.acct
+        if (acct && !acct.includes("@")) {
+            status.account.acct = `${acct}@${status.account.url.split("/")[2]}`
+        }
+        return status
+    })
 }
diff --git a/src/helpers.ts b/src/helpers.ts
@@ -26,11 +26,16 @@ export const _transformKeys = <T>(
 };
 
 export const mastodonFetch = async <T>(server: string, endpoint: string): Promise<T | undefined> => {
-    const json = await axios.get<T>(`https://${server}${endpoint}`)
-    if (!(json.status === 200) || !json.data) {
-        console.error(`Error fetching data for server ${server}:`, json);
+    try {
+        const json = await axios.get<T>(`https://${server}${endpoint}`)
+        if (!(json.status === 200) || !json.data) {
+            console.error(`Error fetching data for server ${server}:`, json);
+            return
+        }
+        const data: T = _transformKeys(json.data, camelCase)
+        return data;
+    } catch (error) {
+        console.error(`Error fetching data for server ${server}:`, error);
         return
     }
-    const data: T = _transformKeys(json.data, camelCase)
-    return data;
 }
diff --git a/src/index.ts b/src/index.ts
@@ -72,7 +72,6 @@ export default class TheAlgorithm {
             // Load Scores for each status
             const featureScore = await Promise.all(featureScorer.map(scorer => scorer.score(this.api, status)));
             const feedScore = await Promise.all(feedScorer.map(scorer => scorer.score(status)));
-
             // Turn Scores into Weight Objects
             const featureScoreObj = this._getScoreObj(scoreNames, featureScore);
             const feedScoreObj = this._getScoreObj(feedScoreNames, feedScore);
@@ -97,7 +96,7 @@ export default class TheAlgorithm {
         // Add Time Penalty
         scoredFeed = scoredFeed.map((item: StatusType) => {
             const seconds = Math.floor((new Date().getTime() - new Date(item.createdAt).getTime()) / 1000);
-            const timediscount = Math.pow((1 + 0.7 * 0.2), - Math.pow((seconds / 3600), 2));
+            const timediscount = Math.pow((1 + 0.05), - Math.pow((seconds / 3600), 2));
             item.value = (item.value ?? 0) * timediscount
             return item;
         })

diff --git a/src/scorer/feed/reblogsFeedScorer.ts b/src/scorer/feed/reblogsFeedScorer.ts
@@ -7,14 +7,15 @@ export default class reblogsFeedScorer extends FeedScorer {
     }
 
     feedExtractor(feed: StatusType[]) {
-        return feed.reduce((obj: Record<string, number>, status) => {
+        // for each uri in the feed, count the number of times it appears
+        const feedScorer = feed.reduce((obj: Record<string, number>, status: StatusType) => {
+            obj[status.uri] = (obj[status.uri] || 0) + 1;
             if (status.reblog) {
-                obj[status.reblog.uri] = (obj[status.reblog.uri] || -1) + 1;
-            } else {
-                obj[status.uri] = (obj[status.uri] || -1) + 1;
+                obj[status.reblog.uri] = (obj[status.reblog.uri] || 0) + 1;
             }
             return obj;
         }, {});
+        return feedScorer;
     }
 
     async score(status: StatusType) {
@@ -23,6 +24,6 @@ export default class reblogsFeedScorer extends FeedScorer {
         if (status.reblog) {
             return features[status.reblog.uri] || 0;
         }
-        return 0;
+        return features[status.uri] || 0;
     }
 }