From 982765562261f06d843f4d69b5373e9a46700cd8 Mon Sep 17 00:00:00 2001 From: Devin Ivy Date: Fri, 1 Nov 2024 16:17:58 -0400 Subject: [PATCH 1/4] appview: start setting up index-at epoch, applying to posts --- packages/bsky/src/config.ts | 13 +++++++++++++ packages/bsky/src/hydration/actor.ts | 2 ++ packages/bsky/src/hydration/hydrator.ts | 1 + packages/bsky/src/hydration/util.ts | 3 +++ packages/bsky/src/index.ts | 6 +++++- packages/bsky/src/views/index.ts | 13 ++++++++++--- 6 files changed, 34 insertions(+), 4 deletions(-) diff --git a/packages/bsky/src/config.ts b/packages/bsky/src/config.ts index 4e9e1b27d24..629ab485a48 100644 --- a/packages/bsky/src/config.ts +++ b/packages/bsky/src/config.ts @@ -36,6 +36,7 @@ export interface ServerConfigValues { modServiceDid: string adminPasswords: string[] labelsFromIssuerDids?: string[] + indexedAtEpoch?: Date // misc/dev blobCacheLocation?: string statsigKey?: string @@ -125,6 +126,13 @@ export class ServerConfig { : process.env.BSKY_STATSIG_ENV || 'development' const clientCheckEmailConfirmed = process.env.BSKY_CLIENT_CHECK_EMAIL_CONFIRMED === 'true' + const indexedAtEpoch = process.env.BSKY_INDEXED_AT_EPOCH + ? new Date(process.env.BSKY_INDEXED_AT_EPOCH) + : undefined + assert( + !indexedAtEpoch || !isNaN(indexedAtEpoch.getTime()), + 'invalid BSKY_INDEXED_AT_EPOCH', + ) return new ServerConfig({ version, debugMode, @@ -161,6 +169,7 @@ export class ServerConfig { statsigKey, statsigEnv, clientCheckEmailConfirmed, + indexedAtEpoch, ...stripUndefineds(overrides ?? {}), }) } @@ -317,6 +326,10 @@ export class ServerConfig { get clientCheckEmailConfirmed() { return this.cfg.clientCheckEmailConfirmed } + + get indexedAtEpoch() { + return this.cfg.indexedAtEpoch + } } function stripUndefineds( diff --git a/packages/bsky/src/hydration/actor.ts b/packages/bsky/src/hydration/actor.ts index 9898dc7eb79..0b3e4ede0f0 100644 --- a/packages/bsky/src/hydration/actor.ts +++ b/packages/bsky/src/hydration/actor.ts @@ -17,6 +17,7 @@ export type Actor = { profileCid?: string profileTakedownRef?: string sortedAt?: Date + indexedAt?: Date takedownRef?: string isLabeler: boolean allowIncomingChatsFrom?: string @@ -126,6 +127,7 @@ export class ActorHydrator { profileCid: profile?.cid, profileTakedownRef: safeTakedownRef(profile), sortedAt: profile?.sortedAt?.toDate(), + indexedAt: profile?.indexedAt?.toDate(), takedownRef: safeTakedownRef(actor), isLabeler: actor.labeler ?? false, allowIncomingChatsFrom: actor.allowIncomingChatsFrom || undefined, diff --git a/packages/bsky/src/hydration/hydrator.ts b/packages/bsky/src/hydration/hydrator.ts index 9c4919d5c95..c19adb3b3d8 100644 --- a/packages/bsky/src/hydration/hydrator.ts +++ b/packages/bsky/src/hydration/hydrator.ts @@ -976,6 +976,7 @@ export class Hydrator { record: actor.profile, cid: actor.profileCid, sortedAt: actor.sortedAt ?? new Date(0), // @NOTE will be present since profile record is present + indexedAt: actor.indexedAt ?? new Date(0), // @NOTE will be present since profile record is present takedownRef: actor.profileTakedownRef, } } diff --git a/packages/bsky/src/hydration/util.ts b/packages/bsky/src/hydration/util.ts index e07a8af02ed..7fe3b0269d7 100644 --- a/packages/bsky/src/hydration/util.ts +++ b/packages/bsky/src/hydration/util.ts @@ -22,6 +22,7 @@ export type RecordInfo = { record: T cid: string sortedAt: Date + indexedAt: Date takedownRef: string | undefined } @@ -65,6 +66,7 @@ export const parseRecord = ( const record = parseRecordBytes(entry.record) const cid = entry.cid const sortedAt = entry.sortedAt?.toDate() ?? new Date(0) + const indexedAt = entry.indexedAt?.toDate() ?? new Date(0) if (!record || !cid) return if (!isValidRecord(record)) { return @@ -73,6 +75,7 @@ export const parseRecord = ( record, cid, sortedAt, + indexedAt, takedownRef: safeTakedownRef(entry), } } diff --git a/packages/bsky/src/index.ts b/packages/bsky/src/index.ts index 9c883d57f22..85002e4c7b2 100644 --- a/packages/bsky/src/index.ts +++ b/packages/bsky/src/index.ts @@ -101,7 +101,11 @@ export class BskyAppView { rejectUnauthorized: !config.dataplaneIgnoreBadTls, }) const hydrator = new Hydrator(dataplane, config.labelsFromIssuerDids) - const views = new Views(imgUriBuilder, videoUriBuilder) + const views = new Views( + imgUriBuilder, + videoUriBuilder, + config.indexedAtEpoch, + ) const bsyncClient = createBsyncClient({ baseUrl: config.bsyncUrl, diff --git a/packages/bsky/src/views/index.ts b/packages/bsky/src/views/index.ts index 69983870756..26199ab3b56 100644 --- a/packages/bsky/src/views/index.ts +++ b/packages/bsky/src/views/index.ts @@ -78,6 +78,7 @@ export class Views { constructor( public imgUriBuilder: ImageUriBuilder, public videoUriBuilder: VideoUriBuilder, + public indexedAtEpoch: Date | undefined, ) {} // Actor @@ -600,7 +601,7 @@ export class Views { repostCount: aggs?.reposts ?? 0, likeCount: aggs?.likes ?? 0, quoteCount: aggs?.quotes ?? 0, - indexedAt: post.sortedAt.toISOString(), + indexedAt: this.indexedAt(post).toISOString(), viewer: viewer ? { repost: viewer.repost, @@ -1170,11 +1171,12 @@ export class Views { } else if (uri.collection === ids.AppBskyActorProfile) { const actor = state.actors?.get(authorDid) recordInfo = - actor && actor.profile && actor.profileCid && actor.sortedAt + actor && actor.profile && actor.profileCid ? { record: actor.profile, cid: actor.profileCid, - sortedAt: actor.sortedAt, + sortedAt: actor.sortedAt ?? new Date(0), // @NOTE will be present since profile record is present + indexedAt: actor.indexedAt ?? new Date(0), // @NOTE will be present since profile record is present takedownRef: actor.profileTakedownRef, } : null @@ -1202,6 +1204,11 @@ export class Views { labels: [...labels, ...selfLabels], } } + + indexedAt({ sortedAt, indexedAt }: { sortedAt: Date; indexedAt: Date }) { + if (!this.indexedAtEpoch) return sortedAt + return indexedAt > this.indexedAtEpoch ? indexedAt : sortedAt + } } const getRootUri = (uri: string, post: Post): string => { From 70ab5f07be6da413d83f9addc34ae2fef199423e Mon Sep 17 00:00:00 2001 From: Devin Ivy Date: Mon, 4 Nov 2024 11:28:07 -0500 Subject: [PATCH 2/4] appview: present all indexed-at times w/ epoch --- packages/bsky/src/views/index.ts | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/packages/bsky/src/views/index.ts b/packages/bsky/src/views/index.ts index 26199ab3b56..80bb3055146 100644 --- a/packages/bsky/src/views/index.ts +++ b/packages/bsky/src/views/index.ts @@ -183,7 +183,13 @@ export class Views { return { ...basicView, description: actor.profile?.description || undefined, - indexedAt: actor.sortedAt?.toISOString(), + indexedAt: + actor.indexedAt && actor.sortedAt + ? this.indexedAt({ + sortedAt: actor.sortedAt, + indexedAt: actor.indexedAt, + }).toISOString() + : undefined, } } @@ -331,7 +337,7 @@ export class Views { creator, description: list.record.description, descriptionFacets: list.record.descriptionFacets, - indexedAt: list.sortedAt.toISOString(), + indexedAt: this.indexedAt(list).toISOString(), } } @@ -357,7 +363,7 @@ export class Views { ) : undefined, listItemCount: listAgg?.listItems ?? 0, - indexedAt: list.sortedAt.toISOString(), + indexedAt: this.indexedAt(list).toISOString(), labels, viewer: listViewer ? { @@ -387,7 +393,7 @@ export class Views { joinedAllTimeCount: agg?.joinedAllTime ?? 0, joinedWeekCount: agg?.joinedWeek ?? 0, labels, - indexedAt: sp.sortedAt.toISOString(), + indexedAt: this.indexedAt(sp).toISOString(), } } @@ -464,7 +470,7 @@ export class Views { like: viewer.like, } : undefined, - indexedAt: labeler.sortedAt.toISOString(), + indexedAt: this.indexedAt(labeler).toISOString(), labels, } } @@ -552,7 +558,7 @@ export class Views { like: viewer.like, } : undefined, - indexedAt: feedgen.sortedAt.toISOString(), + indexedAt: this.indexedAt(feedgen).toISOString(), } } @@ -725,7 +731,7 @@ export class Views { return { $type: 'app.bsky.feed.defs#reasonRepost', by: creator, - indexedAt: repost.sortedAt.toISOString(), + indexedAt: this.indexedAt(repost).toISOString(), } } @@ -1207,7 +1213,7 @@ export class Views { indexedAt({ sortedAt, indexedAt }: { sortedAt: Date; indexedAt: Date }) { if (!this.indexedAtEpoch) return sortedAt - return indexedAt > this.indexedAtEpoch ? indexedAt : sortedAt + return indexedAt && indexedAt > this.indexedAtEpoch ? indexedAt : sortedAt } } From 94812e9882f040225bc1cc02fbe043bfa2614589 Mon Sep 17 00:00:00 2001 From: Devin Ivy Date: Mon, 4 Nov 2024 11:35:33 -0500 Subject: [PATCH 3/4] appview: update Views constructor to take args as object --- packages/bsky/src/index.ts | 10 +++++----- packages/bsky/src/views/index.ts | 13 +++++++++---- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/packages/bsky/src/index.ts b/packages/bsky/src/index.ts index 85002e4c7b2..212cd22e92f 100644 --- a/packages/bsky/src/index.ts +++ b/packages/bsky/src/index.ts @@ -101,11 +101,11 @@ export class BskyAppView { rejectUnauthorized: !config.dataplaneIgnoreBadTls, }) const hydrator = new Hydrator(dataplane, config.labelsFromIssuerDids) - const views = new Views( - imgUriBuilder, - videoUriBuilder, - config.indexedAtEpoch, - ) + const views = new Views({ + imgUriBuilder: imgUriBuilder, + videoUriBuilder: videoUriBuilder, + indexedAtEpoch: config.indexedAtEpoch, + }) const bsyncClient = createBsyncClient({ baseUrl: config.bsyncUrl, diff --git a/packages/bsky/src/views/index.ts b/packages/bsky/src/views/index.ts index 80bb3055146..58dcc2c28e0 100644 --- a/packages/bsky/src/views/index.ts +++ b/packages/bsky/src/views/index.ts @@ -75,10 +75,15 @@ import { Notification } from '../proto/bsky_pb' import { postUriToThreadgateUri, postUriToPostgateUri } from '../util/uris' export class Views { + public imgUriBuilder: ImageUriBuilder = this.opts.imgUriBuilder + public videoUriBuilder: VideoUriBuilder = this.opts.videoUriBuilder + public indexedAtEpoch: Date | undefined = this.opts.indexedAtEpoch constructor( - public imgUriBuilder: ImageUriBuilder, - public videoUriBuilder: VideoUriBuilder, - public indexedAtEpoch: Date | undefined, + private opts: { + imgUriBuilder: ImageUriBuilder + videoUriBuilder: VideoUriBuilder + indexedAtEpoch: Date | undefined + }, ) {} // Actor @@ -149,7 +154,7 @@ export class Views { } : undefined, banner: actor.profile?.banner - ? this.imgUriBuilder.getPresetUri( + ? this.opts.imgUriBuilder.getPresetUri( 'banner', did, cidFromBlobJson(actor.profile.banner), From 140a9373dd552ea263e1078d37a8334c6cc7914c Mon Sep 17 00:00:00 2001 From: devin ivy Date: Mon, 4 Nov 2024 16:34:27 -0500 Subject: [PATCH 4/4] tidy --- packages/bsky/src/views/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/bsky/src/views/index.ts b/packages/bsky/src/views/index.ts index 58dcc2c28e0..ba01ab264aa 100644 --- a/packages/bsky/src/views/index.ts +++ b/packages/bsky/src/views/index.ts @@ -154,7 +154,7 @@ export class Views { } : undefined, banner: actor.profile?.banner - ? this.opts.imgUriBuilder.getPresetUri( + ? this.imgUriBuilder.getPresetUri( 'banner', did, cidFromBlobJson(actor.profile.banner),