diff --git a/src/fixtures/media.ts b/src/fixtures/media.ts new file mode 100644 index 000000000..94e49ad27 --- /dev/null +++ b/src/fixtures/media.ts @@ -0,0 +1,82 @@ +import { MediaType, MediaFile } from "@root/types" + +export const MEDIA_FILE_NAME = "test file" +export const MEDIA_SITE_NAME = "site" +export const MEDIA_DIRECTORY_NAME = "dir" +export const MEDIA_SUBDIRECTORY_NAME = "sub dir" +export const MEDIA_FILE_SHA = "sha" + +export const MEDIA_DIR: MediaFile = { + name: "directory", + type: "dir", + sha: MEDIA_FILE_SHA, + path: `${MEDIA_DIRECTORY_NAME}/directory`, +} + +const BASE_MEDIA_FILE: MediaFile = { + name: MEDIA_FILE_NAME, + type: "file", + sha: MEDIA_FILE_SHA, + path: `${MEDIA_DIRECTORY_NAME}/${MEDIA_FILE_NAME}`, +} + +export const NESTED_MEDIA_FILE: MediaFile = { + ...BASE_MEDIA_FILE, + path: `${MEDIA_DIRECTORY_NAME}/${MEDIA_SUBDIRECTORY_NAME}/${MEDIA_FILE_NAME}`, +} + +export const SVG_FILE = { + ...BASE_MEDIA_FILE, + name: `${MEDIA_FILE_NAME}.svg`, + path: `${MEDIA_DIRECTORY_NAME}/${MEDIA_FILE_NAME}.svg`, +} + +const BASE_INPUT = { + siteName: MEDIA_SITE_NAME, + directoryName: MEDIA_DIRECTORY_NAME, +} + +export const DIR_INPUT = { + ...BASE_INPUT, + file: MEDIA_DIR, + mediaType: "images" as MediaType, + isPrivate: false, +} + +export const IMAGE_FILE_PUBLIC_INPUT = { + ...BASE_INPUT, + file: BASE_MEDIA_FILE, + mediaType: "images" as MediaType, + isPrivate: false, +} + +export const NESTED_IMAGE_FILE_PUBLIC_INPUT = { + ...IMAGE_FILE_PUBLIC_INPUT, + file: NESTED_MEDIA_FILE, + directoryName: `${MEDIA_DIRECTORY_NAME}/${MEDIA_SUBDIRECTORY_NAME}`, +} + +export const SVG_FILE_PUBLIC_INPUT = { + ...IMAGE_FILE_PUBLIC_INPUT, + file: SVG_FILE, +} + +export const IMAGE_FILE_PRIVATE_INPUT = { + ...IMAGE_FILE_PUBLIC_INPUT, + isPrivate: true, +} + +export const SVG_FILE_PRIVATE_INPUT = { + ...SVG_FILE_PUBLIC_INPUT, + isPrivate: true, +} + +export const PDF_FILE_PUBLIC_INPUT = { + ...IMAGE_FILE_PUBLIC_INPUT, + mediaType: "files" as MediaType, +} + +export const PDF_FILE_PRIVATE_INPUT = { + ...PDF_FILE_PUBLIC_INPUT, + isPrivate: true, +} diff --git a/src/services/directoryServices/MediaDirectoryService.js b/src/services/directoryServices/MediaDirectoryService.js index 4cca8f777..a8ef0366f 100644 --- a/src/services/directoryServices/MediaDirectoryService.js +++ b/src/services/directoryServices/MediaDirectoryService.js @@ -1,12 +1,10 @@ -const { config } = require("@config/config") - const { BadRequestError } = require("@errors/BadRequestError") -const GITHUB_ORG_NAME = config.get("github.orgName") +const { isMediaPathValid } = require("@validators/validators") -const PLACEHOLDER_FILE_NAME = ".keep" +const { getMediaFileInfo } = require("@root/utils/media-utils") -const { isMediaPathValid } = require("@validators/validators") +const PLACEHOLDER_FILE_NAME = ".keep" class MediaDirectoryService { constructor({ baseDirectoryService, gitHubService }) { @@ -40,39 +38,32 @@ class MediaDirectoryService { const { private: isPrivate } = await this.gitHubService.getRepoInfo( sessionData ) - const files = await this.listWithDefault(sessionData, { directoryName }) - - const resp = [] - for (const curr of files) { - if (curr.type === "dir") { - resp.push({ - name: curr.name, - type: "dir", - }) - } - if (curr.type !== "file" || curr.name === PLACEHOLDER_FILE_NAME) continue - const fileData = { - mediaUrl: `https://raw.githubusercontent.com/${GITHUB_ORG_NAME}/${siteName}/staging/${curr.path - .split("/") - .map((v) => encodeURIComponent(v)) - .join("/")}${curr.path.endsWith(".svg") ? "?sanitize=true" : ""}`, - name: curr.name, - sha: curr.sha, - mediaPath: `${directoryName}/${curr.name}`, - type: curr.type, - } - if (mediaType === "images" && isPrivate) { - // Generate blob url - const imageExt = curr.name.slice(curr.name.lastIndexOf(".") + 1) - const contentType = `image/${imageExt === "svg" ? "svg+xml" : imageExt}` - const { content } = await this.gitHubService.readMedia(sessionData, { - fileSha: curr.sha, + const files = ( + await this.listWithDefault(sessionData, { directoryName }) + ).filter( + (file) => + (file.type === "file" || file.type === "dir") && + file.name !== PLACEHOLDER_FILE_NAME + ) + + const resp = await Promise.all( + files.map((curr) => { + if (curr.type === "dir") { + return { + name: curr.name, + type: "dir", + } + } + return getMediaFileInfo({ + file: curr, + siteName, + directoryName, + mediaType, + isPrivate, }) - const blobURL = `data:${contentType};base64,${content}` - fileData.mediaUrl = blobURL - } - resp.push(fileData) - } + }) + ) + return resp } diff --git a/src/services/directoryServices/__tests__/MediaDirectoryService.spec.js b/src/services/directoryServices/__tests__/MediaDirectoryService.spec.js index 2ef010bb2..107bdd19a 100644 --- a/src/services/directoryServices/__tests__/MediaDirectoryService.spec.js +++ b/src/services/directoryServices/__tests__/MediaDirectoryService.spec.js @@ -129,46 +129,6 @@ describe("Media Directory Service", () => { directoryName: imageDirectoryName, }) }) - mockGitHubService.getRepoInfo.mockResolvedValueOnce({ - private: true, - }) - mockBaseDirectoryService.list.mockResolvedValueOnce(readImgDirResp) - mockGitHubService.readMedia.mockResolvedValueOnce({ - content: mockContent1, - }) - mockGitHubService.readMedia.mockResolvedValueOnce({ - content: mockContent2, - }) - it("ListFiles for an image directory in a private repo returns all images properly formatted", async () => { - const expectedResp = [ - { - mediaUrl: `data:image/png;base64,${mockContent1}`, - name: testImg1.name, - sha: testImg1.sha, - mediaPath: `${imageDirectoryName}/${testImg1.name}`, - }, - { - mediaUrl: `data:image/svg+xml;base64,${mockContent2}`, - name: testImg2.name, - sha: testImg2.sha, - mediaPath: `${imageDirectoryName}/${testImg2.name}`, - }, - { - name: dir.name, - type: dir.type, - }, - ] - await expect( - service.listFiles(sessionData, { - mediaType: "images", - directoryName: imageDirectoryName, - }) - ).resolves.toMatchObject(expectedResp) - expect(mockGitHubService.getRepoInfo).toHaveBeenCalledWith(sessionData) - expect(mockBaseDirectoryService.list).toHaveBeenCalledWith(sessionData, { - directoryName: imageDirectoryName, - }) - }) mockGitHubService.getRepoInfo.mockResolvedValueOnce({ private: false, }) diff --git a/src/services/fileServices/MdPageServices/MediaFileService.js b/src/services/fileServices/MdPageServices/MediaFileService.js index 369e12e54..7ff1f98a8 100644 --- a/src/services/fileServices/MdPageServices/MediaFileService.js +++ b/src/services/fileServices/MdPageServices/MediaFileService.js @@ -1,12 +1,8 @@ -const { config } = require("@config/config") - const logger = require("@logger/logger") const { BadRequestError } = require("@errors/BadRequestError") const { MediaTypeError } = require("@errors/MediaTypeError") -const GITHUB_ORG_NAME = config.get("github.orgName") - const { validateAndSanitizeFileUpload, ALLOWED_FILE_EXTENSIONS, @@ -16,6 +12,7 @@ const { const { isMediaPathValid } = require("@validators/validators") const { getFileExt } = require("@root/utils/files") +const { getMediaFileInfo } = require("@root/utils/media-utils") class MediaFileService { constructor({ gitHubService }) { @@ -66,31 +63,17 @@ class MediaFileService { const targetFile = directoryData.find( (fileOrDir) => fileOrDir.name === fileName ) - const { sha } = targetFile const { private: isPrivate } = await this.gitHubService.getRepoInfo( sessionData ) - const fileData = { - mediaUrl: `https://raw.githubusercontent.com/${GITHUB_ORG_NAME}/${siteName}/staging/${directoryName - .split("/") - .map((v) => encodeURIComponent(v)) - .join("/")}/${fileName}${ - fileName.endsWith(".svg") ? "?sanitize=true" : "" - }`, - name: fileName, - sha, - mediaPath: `${directoryName}/${fileName}`, - } - if (mediaType === "images" && isPrivate) { - // Generate blob url - const imageExt = fileName.slice(fileName.lastIndexOf(".") + 1) - const contentType = `image/${imageExt === "svg" ? "svg+xml" : imageExt}` - const { content } = await this.gitHubService.readMedia(sessionData, { - fileSha: sha, - }) - const blobURL = `data:${contentType};base64,${content}` - fileData.mediaUrl = blobURL - } + const fileData = await getMediaFileInfo({ + file: targetFile, + siteName, + directoryName, + mediaType, + isPrivate, + }) + return fileData } diff --git a/src/services/fileServices/MdPageServices/__tests__/MediaFileService.spec.js b/src/services/fileServices/MdPageServices/__tests__/MediaFileService.spec.js index 9be58f861..8618dd5cf 100644 --- a/src/services/fileServices/MdPageServices/__tests__/MediaFileService.spec.js +++ b/src/services/fileServices/MdPageServices/__tests__/MediaFileService.spec.js @@ -8,7 +8,9 @@ describe("Media File Service", () => { const siteName = "test-site" const accessToken = "test-token" const imageName = "test image.png" + const imageEncodedName = encodeURIComponent(imageName) const fileName = "test file.pdf" + const fileEncodedName = encodeURIComponent(fileName) const directoryName = "images/subfolder" const mockContent = "schema, test" const mockSanitizedContent = "sanitized-test" @@ -90,20 +92,24 @@ describe("Media File Service", () => { const imageDirResp = [ { name: imageName, + path: `${directoryName}/${imageName}`, sha, }, { name: "image2.png", + path: `${directoryName}/image2.png`, sha: "image2sha", }, ] const fileDirResp = [ { name: fileName, + path: `${directoryName}/${fileName}`, sha, }, { name: "file2.pdf", + path: `${directoryName}/file2.pdf`, sha: "file2sha", }, ] @@ -116,7 +122,7 @@ describe("Media File Service", () => { }) it("Reading image files in public repos works correctly", async () => { const expectedResp = { - mediaUrl: `https://raw.githubusercontent.com/${GITHUB_ORG_NAME}/${siteName}/staging/${directoryName}/${imageName}`, + mediaUrl: `https://raw.githubusercontent.com/${GITHUB_ORG_NAME}/${siteName}/staging/${directoryName}/${imageEncodedName}`, name: imageName, sha, } @@ -139,6 +145,7 @@ describe("Media File Service", () => { ...imageDirResp, { sha, + path: `${directoryName}/${svgName}`, name: svgName, }, ]) @@ -166,34 +173,6 @@ describe("Media File Service", () => { ) expect(mockGithubService.getRepoInfo).toHaveBeenCalledWith(sessionData) }) - mockGithubService.readDirectory.mockResolvedValueOnce(imageDirResp) - mockGithubService.getRepoInfo.mockResolvedValueOnce({ - private: true, - }) - it("Reading image files in private repos works correctly", async () => { - const expectedResp = { - mediaUrl: `data:image/png;base64,${mockContent}`, - name: imageName, - sha, - } - await expect( - service.read(sessionData, { - fileName: imageName, - directoryName, - mediaType: "images", - }) - ).resolves.toMatchObject(expectedResp) - expect(mockGithubService.readDirectory).toHaveBeenCalledWith( - sessionData, - { - directoryName, - } - ) - expect(mockGithubService.getRepoInfo).toHaveBeenCalledWith(sessionData) - expect(mockGithubService.readMedia).toHaveBeenCalledWith(sessionData, { - fileSha: sha, - }) - }) mockGithubService.readDirectory.mockResolvedValueOnce(fileDirResp) mockGithubService.getRepoInfo.mockResolvedValueOnce({ private: false, @@ -203,7 +182,7 @@ describe("Media File Service", () => { }) it("Reading files in public repos works correctly", async () => { const expectedResp = { - mediaUrl: `https://raw.githubusercontent.com/${GITHUB_ORG_NAME}/${siteName}/staging/${directoryName}/${fileName}`, + mediaUrl: `https://raw.githubusercontent.com/${GITHUB_ORG_NAME}/${siteName}/staging/${directoryName}/${fileEncodedName}`, name: fileName, sha, } diff --git a/src/types/index.ts b/src/types/index.ts index e26218c88..c14656ed5 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -1,4 +1,5 @@ export * from "./axios" export * from "./error" +export * from "./media" export * from "./request" export * from "./amplify" diff --git a/src/types/media.ts b/src/types/media.ts new file mode 100644 index 000000000..13b2e4178 --- /dev/null +++ b/src/types/media.ts @@ -0,0 +1,25 @@ +export type ItemType = "dir" | "file" +export type MediaType = "images" | "files" + +export interface MediaFile { + name: string + type: ItemType + sha: string + path: string +} + +export interface MediaFileInput { + file: MediaFile + siteName: string + directoryName: string + mediaType: MediaType + isPrivate?: boolean +} + +export interface MediaFileOutput { + name: string + sha: string + mediaUrl: string + mediaPath: string + type: ItemType +} diff --git a/src/utils/__tests__/media-utils.spec.ts b/src/utils/__tests__/media-utils.spec.ts new file mode 100644 index 000000000..0b829e6e2 --- /dev/null +++ b/src/utils/__tests__/media-utils.spec.ts @@ -0,0 +1,151 @@ +import mockAxios from "jest-mock-axios" + +import { config } from "@config/config" + +import { + MEDIA_DIRECTORY_NAME, + MEDIA_FILE_NAME, + MEDIA_FILE_SHA, + MEDIA_SITE_NAME, + IMAGE_FILE_PRIVATE_INPUT, + IMAGE_FILE_PUBLIC_INPUT, + PDF_FILE_PRIVATE_INPUT, + PDF_FILE_PUBLIC_INPUT, + SVG_FILE_PRIVATE_INPUT, + SVG_FILE_PUBLIC_INPUT, + MEDIA_SUBDIRECTORY_NAME, + NESTED_IMAGE_FILE_PUBLIC_INPUT, +} from "@root/fixtures/media" + +import { getMediaFileInfo } from "../media-utils" + +const GITHUB_ORG_NAME = config.get("github.orgName") + +const mockGenericAxios = mockAxios.create() + +jest.mock("@utils/token-retrieval-utils", () => ({ + getAccessToken: jest.fn().mockResolvedValue("token"), +})) +mockGenericAxios.get.mockResolvedValue({ + data: "blahblah", + headers: { + "content-type": "blahblah", + }, +}) + +describe("Media utils test", () => { + it("should return mediaUrl as raw github information for images in public repos", async () => { + const expectedResp = { + mediaUrl: `https://raw.githubusercontent.com/${GITHUB_ORG_NAME}/${MEDIA_SITE_NAME}/staging/${MEDIA_DIRECTORY_NAME}/${encodeURIComponent( + MEDIA_FILE_NAME + )}`, + name: MEDIA_FILE_NAME, + sha: MEDIA_FILE_SHA, + mediaPath: `${MEDIA_DIRECTORY_NAME}/${MEDIA_FILE_NAME}`, + type: "file", + } + expect(await getMediaFileInfo(IMAGE_FILE_PUBLIC_INPUT)).toStrictEqual( + expectedResp + ) + }) + + it("should handle nested images in public repos", async () => { + const expectedResp = { + mediaUrl: `https://raw.githubusercontent.com/${GITHUB_ORG_NAME}/${MEDIA_SITE_NAME}/staging/${MEDIA_DIRECTORY_NAME}/${encodeURIComponent( + MEDIA_SUBDIRECTORY_NAME + )}/${encodeURIComponent(MEDIA_FILE_NAME)}`, + name: MEDIA_FILE_NAME, + sha: MEDIA_FILE_SHA, + mediaPath: `${MEDIA_DIRECTORY_NAME}/${MEDIA_SUBDIRECTORY_NAME}/${MEDIA_FILE_NAME}`, + type: "file", + } + expect( + await getMediaFileInfo(NESTED_IMAGE_FILE_PUBLIC_INPUT) + ).toStrictEqual(expectedResp) + }) + + it("should return mediaUrl as raw github information for svgs with sanitisation in public repos", async () => { + const expectedResp = { + mediaUrl: `https://raw.githubusercontent.com/${GITHUB_ORG_NAME}/${MEDIA_SITE_NAME}/staging/${MEDIA_DIRECTORY_NAME}/${encodeURIComponent( + MEDIA_FILE_NAME + )}.svg?sanitize=true`, + name: `${MEDIA_FILE_NAME}.svg`, + sha: MEDIA_FILE_SHA, + mediaPath: `${MEDIA_DIRECTORY_NAME}/${MEDIA_FILE_NAME}.svg`, + type: "file", + } + expect(await getMediaFileInfo(SVG_FILE_PUBLIC_INPUT)).toStrictEqual( + expectedResp + ) + }) + + it("should return mediaUrl as raw github information for files in public repos", async () => { + const expectedResp = { + mediaUrl: `https://raw.githubusercontent.com/${GITHUB_ORG_NAME}/${MEDIA_SITE_NAME}/staging/${MEDIA_DIRECTORY_NAME}/${encodeURIComponent( + MEDIA_FILE_NAME + )}`, + name: MEDIA_FILE_NAME, + sha: MEDIA_FILE_SHA, + mediaPath: `${MEDIA_DIRECTORY_NAME}/${MEDIA_FILE_NAME}`, + type: "file", + } + expect(await getMediaFileInfo(PDF_FILE_PUBLIC_INPUT)).toStrictEqual( + expectedResp + ) + }) + + it("should return the mediaUrl as a data url for images in private repos", async () => { + const expectedPartialResp = { + name: MEDIA_FILE_NAME, + sha: MEDIA_FILE_SHA, + mediaPath: `${MEDIA_DIRECTORY_NAME}/${MEDIA_FILE_NAME}`, + type: "file", + } + const resp = await getMediaFileInfo(IMAGE_FILE_PRIVATE_INPUT) + expect(resp).toStrictEqual(expect.objectContaining(expectedPartialResp)) + expect(resp.mediaUrl).toContain("data:") + expect( + mockGenericAxios.get + ).toHaveBeenCalledWith( + `https://token@raw.githubusercontent.com/${GITHUB_ORG_NAME}/${MEDIA_SITE_NAME}/staging/${MEDIA_DIRECTORY_NAME}/${encodeURIComponent( + MEDIA_FILE_NAME + )}`, + { responseType: "arraybuffer" } + ) + }) + + it("should return the mediaUrl as a data url for svgs and sanitise the svgs for svgs in private repos", async () => { + const expectedPartialResp = { + name: `${MEDIA_FILE_NAME}.svg`, + sha: MEDIA_FILE_SHA, + mediaPath: `${MEDIA_DIRECTORY_NAME}/${MEDIA_FILE_NAME}.svg`, + type: "file", + } + const resp = await getMediaFileInfo(SVG_FILE_PRIVATE_INPUT) + expect(resp).toStrictEqual(expect.objectContaining(expectedPartialResp)) + expect(resp.mediaUrl).toContain("data:") + expect( + mockGenericAxios.get + ).toHaveBeenCalledWith( + `https://token@raw.githubusercontent.com/${GITHUB_ORG_NAME}/${MEDIA_SITE_NAME}/staging/${MEDIA_DIRECTORY_NAME}/${encodeURIComponent( + MEDIA_FILE_NAME + )}.svg?sanitize=true`, + { responseType: "arraybuffer" } + ) + }) + + it("should return mediaUrl as raw github information information for files in private repos", async () => { + const expectedResp = { + mediaUrl: `https://raw.githubusercontent.com/${GITHUB_ORG_NAME}/${MEDIA_SITE_NAME}/staging/${MEDIA_DIRECTORY_NAME}/${encodeURIComponent( + MEDIA_FILE_NAME + )}`, + name: MEDIA_FILE_NAME, + sha: MEDIA_FILE_SHA, + mediaPath: `${MEDIA_DIRECTORY_NAME}/${MEDIA_FILE_NAME}`, + type: "file", + } + expect(await getMediaFileInfo(PDF_FILE_PRIVATE_INPUT)).toStrictEqual( + expectedResp + ) + }) +}) diff --git a/src/utils/media-utils.ts b/src/utils/media-utils.ts new file mode 100644 index 000000000..9943476d2 --- /dev/null +++ b/src/utils/media-utils.ts @@ -0,0 +1,71 @@ +import axios from "axios" + +import { config } from "@config/config" + +import logger from "@root/logger/logger" +import { MediaFileInput, MediaFileOutput } from "@root/types" +import { getAccessToken } from "@root/utils/token-retrieval-utils" + +const GITHUB_ORG_NAME = config.get("github.orgName") + +const getEncodedFilePathAsUriComponent = ( + siteName: string, + filePath: string, + accessToken?: string +) => { + const isSvg = filePath.endsWith(".svg") + const encodedFilePath = filePath + .split("/") + .map((v) => encodeURIComponent(v)) + .join("/") + return `https://${ + accessToken ? `${accessToken}@` : "" + }raw.githubusercontent.com/${GITHUB_ORG_NAME}/${siteName}/staging/${encodedFilePath}${ + isSvg ? "?sanitize=true" : "" + }` +} + +export const getMediaFileInfo = async ({ + file, + siteName, + directoryName, + mediaType, + isPrivate, +}: MediaFileInput): Promise => { + const baseFileData = { + name: file.name, + sha: file.sha, + mediaPath: `${directoryName}/${file.name}`, + type: file.type, + } + if (mediaType === "images" && isPrivate) { + try { + // Generate data url + const accessToken = await getAccessToken() + // Accessing images in this way avoids token usage + const endpoint = getEncodedFilePathAsUriComponent( + siteName, + file.path, + accessToken + ) + const resp = await axios.get(endpoint, { responseType: "arraybuffer" }) + const data = Buffer.from(resp.data, "binary").toString("base64") + const dataUri = `data:${resp.headers["content-type"]};base64,${data}` + return { + ...baseFileData, + mediaUrl: dataUri, + } + } catch (err) { + // If an error occurs while retrieving the image, we log but continue to return generic values + logger.error(err) + return { + ...baseFileData, + mediaUrl: "", + } + } + } + return { + ...baseFileData, + mediaUrl: getEncodedFilePathAsUriComponent(siteName, file.path), + } +}