fix(media): filter first before getting git blob hash

src/constants/constants.ts

@@ -2,24 +2,24 @@
 
 import { config } from "@config/config"
 
 export enum JobStatus {
   Ready = "READY", // Ready to run jobs
   Running = "RUNNING", // A job is running
   Failed = "FAILED", // A job has failed and recovery is needed
 }
 
 export enum SiteStatus {
   Empty = "EMPTY", // A site record site is being initialized
   Initialized = "INITIALIZED",
   Launched = "LAUNCHED",
 }
 
 export enum RedirectionTypes {
   CNAME = "CNAME",
   A = "A",
 }
 
 export enum CollaboratorRoles {
   Admin = "ADMIN",
   Contributor = "CONTRIBUTOR",
   IsomerAdmin = "ISOMERADMIN",
@@ -30,7 +30,7 @@
   CollaboratorRoles.IsomerAdmin
 >
 
 export enum ReviewRequestStatus {
   Approved = "APPROVED",
   Open = "OPEN",
   Merged = "MERGED",
@@ -88,3 +88,5 @@
 )
 export const STAGING_BRANCH = "staging"
 export const STAGING_LITE_BRANCH = "staging-lite"
+export const PLACEHOLDER_FILE_NAME = ".keep"
+export const GIT_SYSTEM_DIRECTORY = ".git"

src/services/db/GitFileSystemService.ts

@@ -32,12 +32,14 @@ import { MediaTypeError } from "@root/errors/MediaTypeError"
 import { MediaFileOutput } from "@root/types"
 import { GitHubCommitData } from "@root/types/commitData"
 import type {
+  DirectoryContents,
   GitCommitResult,
   GitDirectoryItem,
   GitFile,
 } from "@root/types/gitfilesystem"
 import type { IsomerCommitMessage } from "@root/types/github"
 import { ALLOWED_FILE_EXTENSIONS } from "@root/utils/file-upload-utils"
+import { getPaginatedDirectoryContents } from "@root/utils/files"
 
 export default class GitFileSystemService {
   private readonly git: SimpleGit
@@ -900,7 +902,7 @@ export default class GitFileSystemService {
           mediaUrl: `${dataUrlPrefix},${file.content}`,
           mediaPath: `${directoryName}/${fileName}`,
           type: fileType,
-          addedTime: stats.birthtimeMs,
+          addedTime: stats.ctimeMs,
           size: stats.size,
         })
       })
@@ -914,6 +916,7 @@ export default class GitFileSystemService {
   ): ResultAsync<GitDirectoryItem[], GitFileSystemError | NotFoundError> {
     const efsVolPath = this.getEfsVolPathFromBranch(branchName)
     const isStaging = this.isStagingFromBranchName(branchName)
+
     return this.getFilePathStats(
       repoName,
       directoryPath,
@@ -952,38 +955,86 @@ export default class GitFileSystemService {
           const path = directoryPath === "" ? name : `${directoryPath}/${name}`
           const type = isDirectory ? "dir" : "file"
 
-          return this.getGitBlobHash(repoName, path, isStaging)
-            .orElse(() => okAsync(""))
-            .andThen((sha) =>
-              ResultAsync.combine([
-                okAsync(sha),
-                this.getFilePathStats(
-                  repoName,
-                  path,
-                  branchName !== STAGING_LITE_BRANCH
-                ),
-              ])
-            )
-            .andThen((shaAndStats) => {
-              const [sha, stats] = shaAndStats
+          return this.getFilePathStats(repoName, path, isStaging).andThen(
+            (stats) => {
               const result: GitDirectoryItem = {
                 name,
                 type,
-                sha,
                 path,
                 size: type === "dir" ? 0 : stats.size,
-                addedTime: stats.birthtimeMs,
+                addedTime: stats.ctimeMs,
               }
 
               return okAsync(result)
-            })
+            }
+          )
         })
 
         return ResultAsync.combine(resultAsyncs)
       })
-      .andThen((directoryItems) =>
+  }
+
+  listPaginatedDirectoryContents(
+    repoName: string,
+    directoryPath: string,
+    branchName: string,
+    page = 0,
+    limit = 0,
+    search = ""
+  ): ResultAsync<DirectoryContents, GitFileSystemError | NotFoundError> {
+    const isStaging = this.isStagingFromBranchName(branchName)
+
+    return this.listDirectoryContents(repoName, directoryPath, branchName)
+      .andThen((directoryContents) =>
+        okAsync(
+          getPaginatedDirectoryContents(directoryContents, page, limit, search)
+        )
+      )
+      .andThen((paginatedDirectoryContents) => {
+        const directories = paginatedDirectoryContents.directories.map(
+          (directory) =>
+            this.getGitBlobHash(repoName, directory.path, isStaging)
+              .orElse(() => okAsync(""))
+              .andThen((sha) => {
+                const result: GitDirectoryItem = {
+                  ...directory,
+                  sha,
+                }
+
+                return okAsync(result)
+              })
+        )
+
+        const files = paginatedDirectoryContents.files.map((file) =>
+          this.getGitBlobHash(repoName, file.path, isStaging)
+            .orElse(() => okAsync(""))
+            .andThen((sha) => {
+              const result: GitDirectoryItem = {
+                ...file,
+                sha,
+              }
+
+              return okAsync(result)
+            })
+        )
+
+        return ResultAsync.combine([
+          ResultAsync.combine(directories),
+          ResultAsync.combine(files),
+          okAsync(paginatedDirectoryContents.total),
+        ])
+      })
+      .andThen(([directories, files, total]) =>
         // Note: The sha is empty if the file is not tracked by Git
-        okAsync(directoryItems.filter((item) => item.sha !== ""))
+        // This may result in the number of files being less than the requested
+        // limit (if limit is greater than 0), but the trade-off is acceptable
+        // here because the user can use pagination to get the next set of files,
+        // which is guaranteed to be a fresh set of files
+        okAsync({
+          directories: directories.filter((directory) => directory.sha !== ""),
+          files: files.filter((file) => file.sha !== ""),
+          total,
+        })
       )
   }
 

src/services/db/RepoService.ts

@@ -10,67 +10,23 @@ import UserWithSiteSessionData from "@root/classes/UserWithSiteSessionData"
 import { FEATURE_FLAGS, STAGING_BRANCH } from "@root/constants"
 import { GitHubCommitData } from "@root/types/commitData"
 import type {
+  DirectoryContents,
   GitCommitResult,
   GitDirectoryItem,
   GitFile,
 } from "@root/types/gitfilesystem"
 import { RawGitTreeEntry } from "@root/types/github"
 import { MediaDirOutput, MediaFileOutput, MediaType } from "@root/types/media"
+import { getPaginatedDirectoryContents } from "@root/utils/files"
 import { getMediaFileInfo } from "@root/utils/media-utils"
 
 import GitFileCommitService from "./GitFileCommitService"
 import GitFileSystemService from "./GitFileSystemService"
 import GitHubService from "./GitHubService"
 import * as ReviewApi from "./review"
 
-const PLACEHOLDER_FILE_NAME = ".keep"
 const BRANCH_REF = config.get("github.branchRef")
 
-const getPaginatedDirectoryContents = (
-  directoryContents: GitDirectoryItem[],
-  page: number,
-  limit = 15,
-  search = ""
-): {
-  directories: GitDirectoryItem[]
-  files: GitDirectoryItem[]
-  total: number
-} => {
-  const subdirectories = directoryContents.filter((item) => item.type === "dir")
-  const files = directoryContents.filter(
-    (item) => item.type === "file" && item.name !== PLACEHOLDER_FILE_NAME
-  )
-
-  let sortedFiles = _(files)
-    // Note: We are sorting by name here to maintain compatibility for
-    // GitHub-login users, since it is very expensive to get the addedTime for
-    // each file from the GitHub API. The files will be sorted by addedTime in
-    // milliseconds for GGS users, so they will never see the alphabetical
-    // sorting.
-    .orderBy(
-      [(file) => file.addedTime, (file) => file.name.toLowerCase()],
-      ["desc", "asc"]
-    )
-
-  if (search) {
-    sortedFiles = sortedFiles.filter((file) =>
-      file.name.toLowerCase().includes(search.toLowerCase())
-    )
-  }
-  const totalLength = sortedFiles.value().length
-
-  const paginatedFiles = sortedFiles
-    .drop(page * limit)
-    .take(limit)
-    .value()
-
-  return {
-    directories: subdirectories,
-    files: paginatedFiles,
-    total: totalLength,
-  }
-}
-
 // TODO: update the typings here to remove `any`.
 // We can type as `unknown` if required.
 
@@ -343,18 +299,21 @@ export default class RepoService extends GitHubService {
     const { siteName } = sessionData
     const defaultBranch = STAGING_BRANCH
     logger.debug(`Reading media directory: ${directoryName}`)
-    let dirContent: GitDirectoryItem[] = []
+    let dirContent: DirectoryContents
 
     if (
       sessionData.growthbook?.getFeatureValue(
         FEATURE_FLAGS.IS_GGS_ENABLED,
         false
       )
     ) {
-      const result = await this.gitFileSystemService.listDirectoryContents(
+      const result = await this.gitFileSystemService.listPaginatedDirectoryContents(
         siteName,
         directoryName,
-        defaultBranch
+        defaultBranch,
+        page,
+        limit,
+        search
       )
 
       if (result.isErr()) {
@@ -363,17 +322,13 @@ export default class RepoService extends GitHubService {
 
       dirContent = result.value
     } else {
-      dirContent = await super.readDirectory(sessionData, {
+      const contents = await super.readDirectory(sessionData, {
         directoryName,
       })
+      dirContent = getPaginatedDirectoryContents(contents, page, limit, search)
     }
 
-    const { directories, files, total } = getPaginatedDirectoryContents(
-      dirContent,
-      page,
-      limit,
-      search
-    )
+    const { directories, files, total } = dirContent
 
     return {
       directories: directories.map(({ name, type }) => ({