diff --git a/.gitignore b/.gitignore index c240a8b..3c6a32b 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ node_modules coverage .env -.DS_Store \ No newline at end of file +.DS_Store +dist \ No newline at end of file diff --git a/dist/features/FeatureStore.js b/dist/features/FeatureStore.js index 6e1aa67..a0c3dec 100644 --- a/dist/features/FeatureStore.js +++ b/dist/features/FeatureStore.js @@ -51,7 +51,8 @@ class FeatureStorage extends Storage_1.default { return topReblogs; } else { - const reblogs = await (0, reblogsFeature_1.default)(api); + const user = await this.getIdentity(); + const reblogs = await (0, reblogsFeature_1.default)(api, user); await this.set(Storage_1.Key.TOP_REBLOGS, reblogs); return reblogs; } diff --git a/dist/features/coreServerFeature.js b/dist/features/coreServerFeature.js index c448c3d..844b6f0 100644 --- a/dist/features/coreServerFeature.js +++ b/dist/features/coreServerFeature.js @@ -39,14 +39,14 @@ async function coreServerFeature(api, user) { return accumulator; }, {}); console.log(serverFrequ); - // for top 30 servers - const top30 = Object.keys(serverFrequ).sort((a, b) => serverFrequ[b] - serverFrequ[a]).slice(0, 30); - console.log("Top 30 servers: ", top30); - const monthlyUsers = await Promise.all(top30.map(server => getMonthlyUsers(server))); + // for top 20 servers + const top20 = Object.keys(serverFrequ).sort((a, b) => serverFrequ[b] - serverFrequ[a]).slice(0, 30); + console.log("Top 30 servers: ", top20); + const monthlyUsers = await Promise.all(top20.map(server => getMonthlyUsers(server))); console.log("Monthly Users: ", monthlyUsers); - const overrepresentedServerFrequ = top30.reduce((acc, server, index) => { + const overrepresentedServerFrequ = top20.reduce((acc, server, index) => { const activeUsers = monthlyUsers[index]; - if (activeUsers < 1) + if (activeUsers < 10) return acc; const ratio = serverFrequ[server] / activeUsers; return { ...acc, [server]: ratio }; diff --git a/dist/features/reblogsFeature.d.ts b/dist/features/reblogsFeature.d.ts index efee422..ae8c039 100644 --- a/dist/features/reblogsFeature.d.ts +++ b/dist/features/reblogsFeature.d.ts @@ -1,2 +1,2 @@ import { mastodon } from "masto"; -export default function getReblogsFeature(api: mastodon.rest.Client): Promise>; +export default function getReblogsFeature(api: mastodon.rest.Client, user: mastodon.v1.Account): Promise>; diff --git a/dist/features/reblogsFeature.js b/dist/features/reblogsFeature.js index 0d32df9..72a578e 100644 --- a/dist/features/reblogsFeature.js +++ b/dist/features/reblogsFeature.js @@ -1,10 +1,10 @@ "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); -async function getReblogsFeature(api) { +async function getReblogsFeature(api, user) { let results = []; let pages = 3; try { - for await (const page of api.v1.timelines.home.list({ limit: 80 })) { + for await (const page of api.v1.accounts.$select(user.id).statuses.list({ limit: 80 })) { results = results.concat(page); pages--; if (pages === 0 || results.length < 80) { @@ -16,6 +16,7 @@ async function getReblogsFeature(api) { console.error(e); return {}; } + console.log(results); const reblogFrequ = results.reduce((accumulator, status) => { if (status.reblog) { if (status.reblog.account.acct in accumulator) { @@ -27,6 +28,7 @@ async function getReblogsFeature(api) { } return accumulator; }, {}); + console.log(reblogFrequ); return reblogFrequ; } exports.default = getReblogsFeature; diff --git a/dist/feeds/topPostsFeed.js b/dist/feeds/topPostsFeed.js index 2c645bb..f483fa1 100644 --- a/dist/feeds/topPostsFeed.js +++ b/dist/feeds/topPostsFeed.js @@ -19,14 +19,20 @@ async function getTopPostFeed(api) { results = await Promise.all(servers.map(async (server) => { if (server === "undefined" || typeof server == "undefined" || server === "") return []; - const data = await (0, helpers_1.mastodonFetch)(server, "api/v1/timelines/public"); - return data?.map((status) => { + const data = await (0, helpers_1.mastodonFetch)(server, "api/v1/trends/statuses"); + return data?.filter(status => status?.favouritesCount > 0 || status?.reblogsCount > 0).map((status) => { status.topPost = true; return status; }).slice(0, 10) ?? []; })); console.log(results); const lastOpened = new Date((await Storage_1.default.getLastOpened() ?? 0) - 28800000); - return results.flat().filter((status) => new Date(status.createdAt) > lastOpened); + return results.flat().filter((status) => new Date(status.createdAt) > lastOpened).map((status) => { + const acct = status.account.acct; + if (acct && !acct.includes("@")) { + status.account.acct = `${acct}@${status.account.url.split("/")[2]}`; + } + return status; + }); } exports.default = getTopPostFeed; diff --git a/dist/helpers.js b/dist/helpers.js index 8e7567f..8ae2891 100644 --- a/dist/helpers.js +++ b/dist/helpers.js @@ -23,12 +23,18 @@ const _transformKeys = (data, transform) => { }; exports._transformKeys = _transformKeys; const mastodonFetch = async (server, endpoint) => { - const json = await axios_1.default.get(`https://${server}${endpoint}`); - if (!(json.status === 200) || !json.data) { - console.error(`Error fetching data for server ${server}:`, json); + try { + const json = await axios_1.default.get(`https://${server}${endpoint}`); + if (!(json.status === 200) || !json.data) { + console.error(`Error fetching data for server ${server}:`, json); + return; + } + const data = (0, exports._transformKeys)(json.data, change_case_1.camelCase); + return data; + } + catch (error) { + console.error(`Error fetching data for server ${server}:`, error); return; } - const data = (0, exports._transformKeys)(json.data, change_case_1.camelCase); - return data; }; exports.mastodonFetch = mastodonFetch; diff --git a/dist/index.js b/dist/index.js index b61cf4c..b9bbe60 100644 --- a/dist/index.js +++ b/dist/index.js @@ -76,7 +76,7 @@ class TheAlgorithm { // Add Time Penalty scoredFeed = scoredFeed.map((item) => { const seconds = Math.floor((new Date().getTime() - new Date(item.createdAt).getTime()) / 1000); - const timediscount = Math.pow((1 + 0.7 * 0.2), -Math.pow((seconds / 3600), 2)); + const timediscount = Math.pow((1 + 0.05), -Math.pow((seconds / 3600), 2)); item.value = (item.value ?? 0) * timediscount; return item; }); diff --git a/dist/scorer/feed/reblogsFeedScorer.js b/dist/scorer/feed/reblogsFeedScorer.js index 3cdae11..cc2ec33 100644 --- a/dist/scorer/feed/reblogsFeedScorer.js +++ b/dist/scorer/feed/reblogsFeedScorer.js @@ -9,15 +9,15 @@ class reblogsFeedScorer extends FeedScorer_1.default { super("reblogsFeed", "More Weight to posts that are reblogged a lot", 6); } feedExtractor(feed) { - return feed.reduce((obj, status) => { + // for each uri in the feed, count the number of times it appears + const feedScorer = feed.reduce((obj, status) => { + obj[status.uri] = (obj[status.uri] || 0) + 1; if (status.reblog) { - obj[status.reblog.uri] = (obj[status.reblog.uri] || -1) + 1; - } - else { - obj[status.uri] = (obj[status.uri] || -1) + 1; + obj[status.reblog.uri] = (obj[status.reblog.uri] || 0) + 1; } return obj; }, {}); + return feedScorer; } async score(status) { super.score(status); @@ -25,7 +25,7 @@ class reblogsFeedScorer extends FeedScorer_1.default { if (status.reblog) { return features[status.reblog.uri] || 0; } - return 0; + return features[status.uri] || 0; } } exports.default = reblogsFeedScorer; diff --git a/src/features/FeatureStore.ts b/src/features/FeatureStore.ts index 2354e8b..da86646 100644 --- a/src/features/FeatureStore.ts +++ b/src/features/FeatureStore.ts @@ -26,7 +26,8 @@ export default class FeatureStorage extends Storage { if (topReblogs != null && await this.getOpenings() % 10 < 9) { return topReblogs; } else { - const reblogs = await reblogsFeature(api); + const user = await this.getIdentity() + const reblogs = await reblogsFeature(api, user); await this.set(Key.TOP_REBLOGS, reblogs); return reblogs; } diff --git a/src/features/coreServerFeature.ts b/src/features/coreServerFeature.ts index 92b4d5d..3e52211 100644 --- a/src/features/coreServerFeature.ts +++ b/src/features/coreServerFeature.ts @@ -42,17 +42,17 @@ export default async function coreServerFeature(api: mastodon.rest.Client, user: console.log(serverFrequ) - // for top 30 servers - const top30 = Object.keys(serverFrequ).sort((a, b) => serverFrequ[b] - serverFrequ[a]).slice(0, 30) + // for top 20 servers + const top20 = Object.keys(serverFrequ).sort((a, b) => serverFrequ[b] - serverFrequ[a]).slice(0, 30) - console.log("Top 30 servers: ", top30) - const monthlyUsers = await Promise.all(top30.map(server => getMonthlyUsers(server))) + console.log("Top 30 servers: ", top20) + const monthlyUsers = await Promise.all(top20.map(server => getMonthlyUsers(server))) console.log("Monthly Users: ", monthlyUsers) - const overrepresentedServerFrequ = top30.reduce((acc, server, index) => { + const overrepresentedServerFrequ = top20.reduce((acc, server, index) => { const activeUsers = monthlyUsers[index]; - if (activeUsers < 1) return acc; + if (activeUsers < 10) return acc; const ratio = serverFrequ[server] / activeUsers; return { ...acc, [server]: ratio } }, {}) diff --git a/src/features/reblogsFeature.ts b/src/features/reblogsFeature.ts index 200bedb..bd60e00 100644 --- a/src/features/reblogsFeature.ts +++ b/src/features/reblogsFeature.ts @@ -1,10 +1,10 @@ import { mastodon } from "masto"; -export default async function getReblogsFeature(api: mastodon.rest.Client) { +export default async function getReblogsFeature(api: mastodon.rest.Client, user: mastodon.v1.Account): Promise> { let results: mastodon.v1.Status[] = []; let pages = 3; try { - for await (const page of api.v1.timelines.home.list({ limit: 80 })) { + for await (const page of api.v1.accounts.$select(user.id).statuses.list({ limit: 80 })) { results = results.concat(page) pages--; if (pages === 0 || results.length < 80) { @@ -15,6 +15,7 @@ export default async function getReblogsFeature(api: mastodon.rest.Client) { console.error(e) return {}; } + console.log(results) const reblogFrequ = results.reduce((accumulator: Record, status: mastodon.v1.Status) => { if (status.reblog) { @@ -26,6 +27,6 @@ export default async function getReblogsFeature(api: mastodon.rest.Client) { } return accumulator }, {}) - + console.log(reblogFrequ) return reblogFrequ; } \ No newline at end of file diff --git a/src/feeds/topPostsFeed.ts b/src/feeds/topPostsFeed.ts index 66001fb..dd56013 100644 --- a/src/feeds/topPostsFeed.ts +++ b/src/feeds/topPostsFeed.ts @@ -21,8 +21,10 @@ export default async function getTopPostFeed(api: mastodon.rest.Client): Promise results = await Promise.all(servers.map(async (server: string): Promise => { if (server === "undefined" || typeof server == "undefined" || server === "") return []; - const data = await mastodonFetch(server, "api/v1/timelines/public") - return data?.map((status: StatusType) => { + const data = await mastodonFetch(server, "api/v1/trends/statuses") + return data?.filter( + status => status?.favouritesCount > 0 || status?.reblogsCount > 0 + ).map((status: StatusType) => { status.topPost = true; return status; }).slice(0, 10) ?? [] @@ -30,5 +32,11 @@ export default async function getTopPostFeed(api: mastodon.rest.Client): Promise console.log(results) const lastOpened = new Date((await Storage.getLastOpened() ?? 0) - 28800000) - return results.flat().filter((status: StatusType) => new Date(status.createdAt) > lastOpened) + return results.flat().filter((status: StatusType) => new Date(status.createdAt) > lastOpened).map((status: StatusType) => { + const acct = status.account.acct + if (acct && !acct.includes("@")) { + status.account.acct = `${acct}@${status.account.url.split("/")[2]}` + } + return status + }) } \ No newline at end of file diff --git a/src/helpers.ts b/src/helpers.ts index a9f8de9..1c2e614 100644 --- a/src/helpers.ts +++ b/src/helpers.ts @@ -26,11 +26,16 @@ export const _transformKeys = ( }; export const mastodonFetch = async (server: string, endpoint: string): Promise => { - const json = await axios.get(`https://${server}${endpoint}`) - if (!(json.status === 200) || !json.data) { - console.error(`Error fetching data for server ${server}:`, json); + try { + const json = await axios.get(`https://${server}${endpoint}`) + if (!(json.status === 200) || !json.data) { + console.error(`Error fetching data for server ${server}:`, json); + return + } + const data: T = _transformKeys(json.data, camelCase) + return data; + } catch (error) { + console.error(`Error fetching data for server ${server}:`, error); return } - const data: T = _transformKeys(json.data, camelCase) - return data; } diff --git a/src/index.ts b/src/index.ts index 8498236..690f73b 100644 --- a/src/index.ts +++ b/src/index.ts @@ -72,7 +72,6 @@ export default class TheAlgorithm { // Load Scores for each status const featureScore = await Promise.all(featureScorer.map(scorer => scorer.score(this.api, status))); const feedScore = await Promise.all(feedScorer.map(scorer => scorer.score(status))); - // Turn Scores into Weight Objects const featureScoreObj = this._getScoreObj(scoreNames, featureScore); const feedScoreObj = this._getScoreObj(feedScoreNames, feedScore); @@ -97,7 +96,7 @@ export default class TheAlgorithm { // Add Time Penalty scoredFeed = scoredFeed.map((item: StatusType) => { const seconds = Math.floor((new Date().getTime() - new Date(item.createdAt).getTime()) / 1000); - const timediscount = Math.pow((1 + 0.7 * 0.2), - Math.pow((seconds / 3600), 2)); + const timediscount = Math.pow((1 + 0.05), - Math.pow((seconds / 3600), 2)); item.value = (item.value ?? 0) * timediscount return item; }) diff --git a/src/scorer/feed/reblogsFeedScorer.ts b/src/scorer/feed/reblogsFeedScorer.ts index 9e42e79..8f0af36 100644 --- a/src/scorer/feed/reblogsFeedScorer.ts +++ b/src/scorer/feed/reblogsFeedScorer.ts @@ -7,14 +7,15 @@ export default class reblogsFeedScorer extends FeedScorer { } feedExtractor(feed: StatusType[]) { - return feed.reduce((obj: Record, status) => { + // for each uri in the feed, count the number of times it appears + const feedScorer = feed.reduce((obj: Record, status: StatusType) => { + obj[status.uri] = (obj[status.uri] || 0) + 1; if (status.reblog) { - obj[status.reblog.uri] = (obj[status.reblog.uri] || -1) + 1; - } else { - obj[status.uri] = (obj[status.uri] || -1) + 1; + obj[status.reblog.uri] = (obj[status.reblog.uri] || 0) + 1; } return obj; }, {}); + return feedScorer; } async score(status: StatusType) { @@ -23,6 +24,6 @@ export default class reblogsFeedScorer extends FeedScorer { if (status.reblog) { return features[status.reblog.uri] || 0; } - return 0; + return features[status.uri] || 0; } } \ No newline at end of file