diff --git a/README.md b/README.md index 4eea9d31bb..33b8d6acef 100644 --- a/README.md +++ b/README.md @@ -119,6 +119,12 @@ custom: With this approach you could have a CloudFront distribution in front of the bucket and use a custom domain in the assetPrefix. +| Plugin config key | Default Value | Description | +| ----------------- | ------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| assetsBucketName | \ | Creates an S3 bucket with the name provided. The bucket will be used for uploading next static assets | +| staticDir | \ | Directory with static assets to be uploaded to S3, typically a directory named `static`, but it can be any other name. Requires a bucket provided via the `assetPrefix` described above or the `assetsBucketName` plugin config. | +| uploadBuildAssets | true | In the unlikely event that you only want to upload the `staticDir`, set this to `false` | + ## Deploying `serverless deploy` diff --git a/__tests__/index.test.js b/__tests__/index.test.js index 86b4fca88d..11c57a4dff 100644 --- a/__tests__/index.test.js +++ b/__tests__/index.test.js @@ -1,9 +1,5 @@ -const path = require("path"); const ServerlessPluginBuilder = require("../utils/test/ServerlessPluginBuilder"); -const parsedNextConfigurationFactory = require("../utils/test/parsedNextConfigurationFactory"); -const uploadStaticAssetsToS3 = require("../lib/uploadStaticAssetsToS3"); const displayStackOutput = require("../lib/displayStackOutput"); -const parseNextConfiguration = require("../lib/parseNextConfiguration"); const build = require("../lib/build"); const NextPage = require("../classes/NextPage"); const PluginBuildDir = require("../classes/PluginBuildDir"); @@ -11,7 +7,6 @@ const PluginBuildDir = require("../classes/PluginBuildDir"); jest.mock("js-yaml"); jest.mock("../lib/build"); jest.mock("../lib/parseNextConfiguration"); -jest.mock("../lib/uploadStaticAssetsToS3"); jest.mock("../lib/displayStackOutput"); jest.mock("../utils/logger"); @@ -159,66 +154,6 @@ describe("ServerlessNextJsPlugin", () => { }); }); - describe("#uploadStaticAssets", () => { - it("should NOT call uploadStaticAssetsToS3 when there isn't a bucket available", () => { - parseNextConfiguration.mockReturnValueOnce( - parsedNextConfigurationFactory({}, null) - ); - - const plugin = new ServerlessPluginBuilder().build(); - - return plugin.uploadStaticAssets().then(() => { - expect(uploadStaticAssetsToS3).not.toBeCalled(); - }); - }); - - it("should call uploadStaticAssetsToS3 with bucketName and next static dir", () => { - const distDir = "build"; - parseNextConfiguration.mockReturnValueOnce( - parsedNextConfigurationFactory({ - distDir - }) - ); - - uploadStaticAssetsToS3.mockResolvedValueOnce("Assets Uploaded"); - - const plugin = new ServerlessPluginBuilder().build(); - - return plugin.uploadStaticAssets().then(() => { - expect(uploadStaticAssetsToS3).toBeCalledWith({ - staticAssetsPath: path.join("/path/to/next", distDir, "static"), - bucketName: "my-bucket", - providerRequest: expect.any(Function) - }); - }); - }); - - it("should call uploadStaticAssetsToS3 with bucketName from plugin config", () => { - const distDir = "build"; - parseNextConfiguration.mockReturnValueOnce( - parsedNextConfigurationFactory({ - distDir - }) - ); - - uploadStaticAssetsToS3.mockResolvedValueOnce("Assets Uploaded"); - - const plugin = new ServerlessPluginBuilder() - .withNextCustomConfig({ - assetsBucketName: "custom-bucket" - }) - .build(); - - return plugin.uploadStaticAssets().then(() => { - expect(uploadStaticAssetsToS3).toBeCalledWith({ - staticAssetsPath: path.join("/path/to/next", distDir, "static"), - bucketName: "custom-bucket", - providerRequest: expect.any(Function) - }); - }); - }); - }); - describe("#printStackOutput", () => { it("should call displayStackOutput with awsInfo", () => { const awsInfo = { diff --git a/index.js b/index.js index b40f4b1a30..711b4da4f1 100644 --- a/index.js +++ b/index.js @@ -1,12 +1,12 @@ "use strict"; const path = require("path"); -const uploadStaticAssetsToS3 = require("./lib/uploadStaticAssetsToS3"); const displayStackOutput = require("./lib/displayStackOutput"); const parseNextConfiguration = require("./lib/parseNextConfiguration"); const build = require("./lib/build"); const PluginBuildDir = require("./classes/PluginBuildDir"); const addAssetsBucketForDeployment = require("./lib/addAssetsBucketForDeployment"); +const uploadStaticAssets = require("./lib/uploadStaticAssets"); class ServerlessNextJsPlugin { constructor(serverless, options) { @@ -19,7 +19,7 @@ class ServerlessNextJsPlugin { this.pluginBuildDir = new PluginBuildDir(this.nextConfigDir); this.addAssetsBucketForDeployment = addAssetsBucketForDeployment.bind(this); - this.uploadStaticAssets = this.uploadStaticAssets.bind(this); + this.uploadStaticAssets = uploadStaticAssets.bind(this); this.printStackOutput = this.printStackOutput.bind(this); this.buildNextPages = this.buildNextPages.bind(this); this.removePluginBuildDir = this.removePluginBuildDir.bind(this); @@ -74,30 +74,6 @@ class ServerlessNextJsPlugin { this.serverless.service.setFunctionNames(); } - uploadStaticAssets() { - let { nextConfiguration, staticAssetsBucket } = this.configuration; - - const bucketNameFromConfig = this.getPluginConfigValue("assetsBucketName"); - - if (bucketNameFromConfig) { - staticAssetsBucket = bucketNameFromConfig; - } - - if (!staticAssetsBucket) { - return Promise.resolve(); - } - - return uploadStaticAssetsToS3({ - staticAssetsPath: path.join( - this.nextConfigDir, - nextConfiguration.distDir, - "static" - ), - providerRequest: this.providerRequest, - bucketName: staticAssetsBucket - }); - } - printStackOutput() { const awsInfo = this.serverless.pluginManager.getPlugins().find(plugin => { return plugin.constructor.name === "AwsInfo"; diff --git a/lib/__tests__/addAssetsBucketForDeployment.test.js b/lib/__tests__/addAssetsBucketForDeployment.test.js index 8c779d33e8..191a006307 100644 --- a/lib/__tests__/addAssetsBucketForDeployment.test.js +++ b/lib/__tests__/addAssetsBucketForDeployment.test.js @@ -27,6 +27,25 @@ describe("addAssetsBucketForDeployment", () => { jest.clearAllMocks(); }); + it("should error if staticDir provided but no bucket", () => { + expect.assertions(1); + + const pluginWithoutBucket = new ServerlessPluginBuilder() + .withNextCustomConfig({ + nextConfigDir: "./", + staticDir: "./static" + }) + .build(); + + parseNextConfiguration.mockReturnValueOnce( + parsedNextConfigurationFactory({ staticAssetsBucket: null }, null) + ); + + return addAssetsBucketForDeployment.call(pluginWithoutBucket).catch(err => { + expect(err.message).toContain("staticDir requires a bucket. See"); + }); + }); + it("should not call addS3BucketToResources if a staticAssetsBucket is not available", () => { expect.assertions(1); parseNextConfiguration.mockReturnValueOnce( diff --git a/lib/__tests__/uploadStaticAssetsToS3.test.js b/lib/__tests__/uploadStaticAssetsToS3.test.js deleted file mode 100644 index e3abac37ae..0000000000 --- a/lib/__tests__/uploadStaticAssetsToS3.test.js +++ /dev/null @@ -1,211 +0,0 @@ -const path = require("path"); -const fs = require("fs"); -const walkDir = require("klaw"); -const mime = require("mime"); -const stream = require("stream"); -const uploadStaticAssetsToS3 = require("../uploadStaticAssetsToS3"); -const logger = require("../../utils/logger"); - -jest.mock("../../utils/logger"); -jest.mock("fs"); -jest.mock("mime"); -jest.mock("klaw"); - -describe("uploadStaticAssetsToS3", () => { - let mockedStream; - - beforeEach(() => { - mockedStream = new stream.Readable(); - mockedStream._read = () => {}; - walkDir.mockReturnValueOnce(mockedStream); - fs.lstatSync.mockReturnValue({ isDirectory: () => false }); - }); - - afterEach(() => { - jest.clearAllMocks(); - }); - - it("should log when upload started", () => { - expect.assertions(1); - const bucketName = "my-bucket"; - - const promise = uploadStaticAssetsToS3({ - staticAssetsPath: path.join(".next", "static"), - bucketName, - providerRequest: () => {} - }).then(() => { - expect(logger.log).toBeCalledWith( - expect.stringContaining(`Uploading static assets to ${bucketName}`) - ); - }); - - mockedStream.emit("data", { - path: path.normalize("users/foo/static/chunks/foo.js") - }); - mockedStream.emit("end"); - - return promise; - }); - - it("should log when upload finished", () => { - expect.assertions(1); - - const promise = uploadStaticAssetsToS3({ - staticAssetsPath: path.join(".next", "static"), - providerRequest: () => {} - }).then(() => { - expect(logger.log).toBeCalledWith( - expect.stringContaining(`Upload finished`) - ); - }); - - mockedStream.emit("data", { - path: path.normalize("users/foo/static/chunks/foo.js") - }); - mockedStream.emit("end"); - - return promise; - }); - - it("should get a list of all static files to upload", () => { - expect.assertions(1); - - const promise = uploadStaticAssetsToS3({ - staticAssetsPath: path.join(".next", "static"), - providerRequest: () => {} - }).then(() => { - expect(walkDir).toBeCalledWith(path.join(".next", "static")); - }); - - mockedStream.emit("data", { - path: path.normalize("users/foo/static/chunks/foo.js") - }); - mockedStream.emit("end"); - - return promise; - }); - - it("should get a list of all static files to upload using the custom next build dir provided", () => { - expect.assertions(1); - - const promise = uploadStaticAssetsToS3({ - staticAssetsPath: path.join("build", "static"), - providerRequest: () => {} - }).then(() => { - expect(walkDir).toBeCalledWith(path.join("build", "static")); - }); - - mockedStream.emit("data", { - path: path.normalize("users/foo/static/chunks/foo.js") - }); - mockedStream.emit("end"); - - return promise; - }); - - it("should upload to S3 the next static assets with correct body", () => { - expect.assertions(2); - - fs.createReadStream.mockReturnValueOnce("FakeStream"); - - const providerRequest = jest.fn(); - const bucketName = "my-bucket"; - - mime.getType.mockImplementation(() => "application/foo"); - - const promise = uploadStaticAssetsToS3({ - staticAssetsPath: path.join("build", "static"), - providerRequest, - bucketName - }).then(() => { - expect(mime.getType).toBeCalledWith( - expect.stringContaining(path.normalize("chunks/foo.js")) - ); - expect(providerRequest).toBeCalledWith( - "S3", - "upload", - expect.objectContaining({ - ACL: "public-read", - Bucket: bucketName, - Key: "_next/static/chunks/foo.js", - ContentType: "application/foo" - }) - ); - }); - - mockedStream.emit("data", { - path: path.normalize("/users/foo/prj/.next/static/chunks/foo.js") - }); - mockedStream.emit("end"); - - return promise; - }); - - it("should not try to upload directories to S3 bucket", () => { - expect.assertions(1); - - fs.lstatSync.mockReturnValue({ isDirectory: () => true }); - - const providerRequest = jest.fn(); - - const promise = uploadStaticAssetsToS3({ - staticAssetsPath: path.join("build", "static"), - providerRequest - }).then(() => { - expect(providerRequest).not.toBeCalled(); - }); - - mockedStream.emit("data", { - path: path.normalize("/users/foo/prj/.next/static/chunks") - }); - mockedStream.emit("end"); - - return promise; - }); - - it("should resolve when all files have been uploaded and return files count", () => { - expect.assertions(1); - - const providerRequest = jest.fn().mockResolvedValue("OK"); - - const promise = uploadStaticAssetsToS3({ - staticAssetsPath: "build/static", - providerRequest - }).then(filesUploaded => { - expect(filesUploaded).toEqual(2); - }); - - mockedStream.emit("data", { - path: path.normalize("/users/foo/prj/.next/static/chunks/1.js") - }); - mockedStream.emit("data", { - path: path.normalize("/users/foo/prj/.next/static/chunks/2.js") - }); - mockedStream.emit("end"); - - return promise; - }); - - it("should reject when a file upload fails", () => { - expect.assertions(1); - - const providerRequest = jest.fn().mockRejectedValueOnce("Error"); - - const promise = uploadStaticAssetsToS3({ - staticAssetsPath: path.join("build", "static"), - providerRequest - }).catch(err => { - expect(err.message).toContain("File upload failed"); - }); - - mockedStream.emit("data", { - path: path.normalize("/users/foo/prj/.next/static/chunks/1.js") - }); - mockedStream.emit("data", { - path: path.normalize("/users/foo/prj/.next/static/chunks/2.js") - }); - mockedStream.emit("end"); - - return promise; - }); -}); diff --git a/lib/addAssetsBucketForDeployment.js b/lib/addAssetsBucketForDeployment.js index e5e8ae5265..594f68ea27 100644 --- a/lib/addAssetsBucketForDeployment.js +++ b/lib/addAssetsBucketForDeployment.js @@ -16,9 +16,10 @@ const getCFTemplatesWithBucket = async function(bucketName) { }; module.exports = async function() { - let { staticAssetsBucket } = parseNextConfiguration( - this.getPluginConfigValue("nextConfigDir") - ); + const nextConfigDir = this.getPluginConfigValue("nextConfigDir"); + const staticDir = this.getPluginConfigValue("staticDir"); + + let { staticAssetsBucket } = parseNextConfiguration(nextConfigDir); const bucketNameFromConfig = this.getPluginConfigValue("assetsBucketName"); @@ -29,7 +30,15 @@ module.exports = async function() { } if (!staticAssetsBucket) { - return; + if (staticDir) { + return Promise.reject( + new Error( + "staticDir requires a bucket. See https://github.com/danielcondemarin/serverless-nextjs-plugin#hosting-static-assets" + ) + ); + } + + return Promise.resolve(); } logger.log(`Found bucket "${staticAssetsBucket}"`); diff --git a/lib/uploadStaticAssets/__tests__/index.test.js b/lib/uploadStaticAssets/__tests__/index.test.js new file mode 100644 index 0000000000..ae1338fd5b --- /dev/null +++ b/lib/uploadStaticAssets/__tests__/index.test.js @@ -0,0 +1,127 @@ +const path = require("path"); +const uploadStaticAssets = require(".."); +const parseNextConfiguration = require("../../parseNextConfiguration"); +const parsedNextConfigurationFactory = require("../../../utils/test/parsedNextConfigurationFactory"); +const ServerlessPluginBuilder = require("../../../utils/test/ServerlessPluginBuilder"); +const uploadDirToS3Factory = require("../../../utils/s3/upload"); + +jest.mock("../../../utils/s3/upload"); +jest.mock("../../parseNextConfiguration"); + +describe("uploadStaticAssets", () => { + let uploadDirToS3; + + beforeEach(() => { + uploadDirToS3 = jest.fn().mockResolvedValue(); + uploadDirToS3Factory.mockReturnValue(uploadDirToS3); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + it("should NOT upload build assets when there isn't a bucket available", () => { + parseNextConfiguration.mockReturnValueOnce( + parsedNextConfigurationFactory({}, null) + ); + + const plugin = new ServerlessPluginBuilder().build(); + + return uploadStaticAssets.call(plugin).then(() => { + expect(uploadDirToS3).not.toBeCalled(); + }); + }); + + it("should upload next build assets", () => { + const distDir = "build"; + parseNextConfiguration.mockReturnValueOnce( + parsedNextConfigurationFactory({ + distDir + }) + ); + + const plugin = new ServerlessPluginBuilder().build(); + + return uploadStaticAssets.call(plugin).then(() => { + expect(uploadDirToS3).toBeCalledTimes(1); + expect(uploadDirToS3).toBeCalledWith( + path.join("/path/to/next", distDir, "static"), + { + bucket: "my-bucket", + prefix: "static", + rootPrefix: "_next" + } + ); + }); + }); + + it("should upload next build assets using bucketName from plugin config", () => { + const distDir = "build"; + parseNextConfiguration.mockReturnValueOnce( + parsedNextConfigurationFactory({ + distDir + }) + ); + + const plugin = new ServerlessPluginBuilder() + .withNextCustomConfig({ + assetsBucketName: "custom-bucket" + }) + .build(); + + return uploadStaticAssets.call(plugin).then(() => { + expect(uploadDirToS3).toBeCalledWith( + path.join("/path/to/next", distDir, "static"), + { + bucket: "custom-bucket", + prefix: "static", + rootPrefix: "_next" + } + ); + }); + }); + + it("should upload staticDir", () => { + const staticDir = "/path/to/assets"; + + parseNextConfiguration.mockReturnValueOnce( + parsedNextConfigurationFactory() + ); + + const plugin = new ServerlessPluginBuilder() + .withNextCustomConfig({ + staticDir: "/path/to/assets" + }) + .build(); + + return uploadStaticAssets.call(plugin).then(() => { + expect(uploadDirToS3).toBeCalledWith(staticDir, { + bucket: "my-bucket", + prefix: "assets" + }); + }); + }); + + it("should not upload build assets", () => { + const staticDir = "/path/to/assets"; + + parseNextConfiguration.mockReturnValueOnce( + parsedNextConfigurationFactory() + ); + + const plugin = new ServerlessPluginBuilder() + .withNextCustomConfig({ + uploadBuildAssets: false, + staticDir: "/path/to/assets" + }) + .build(); + + return uploadStaticAssets.call(plugin).then(() => { + expect(uploadDirToS3).toBeCalledTimes(1); + expect(uploadDirToS3).toBeCalledWith(staticDir, { + bucket: "my-bucket", + prefix: "assets" + }); + }); + }); +}); diff --git a/lib/uploadStaticAssets/index.js b/lib/uploadStaticAssets/index.js new file mode 100644 index 0000000000..e8e5a96a47 --- /dev/null +++ b/lib/uploadStaticAssets/index.js @@ -0,0 +1,45 @@ +const path = require("path"); +const uploadDirToS3Factory = require("../../utils/s3/upload"); + +module.exports = function() { + const uploadDirToS3 = uploadDirToS3Factory(this.providerRequest); + + let { nextConfiguration, staticAssetsBucket } = this.configuration; + + const bucketNameFromConfig = this.getPluginConfigValue("assetsBucketName"); + const staticDir = this.getPluginConfigValue("staticDir"); + const uploadBuildAssets = this.getPluginConfigValue("uploadBuildAssets"); + + if (bucketNameFromConfig) { + staticAssetsBucket = bucketNameFromConfig; + } + + if (!staticAssetsBucket) { + return Promise.resolve(); + } + + const uploadPromises = []; + + if (uploadBuildAssets !== false) { + const buildAssetsUpload = uploadDirToS3( + path.join(this.nextConfigDir, nextConfiguration.distDir, "static"), + { + bucket: staticAssetsBucket, + prefix: "static", + rootPrefix: "_next" + } + ); + uploadPromises.push(buildAssetsUpload); + } + + if (staticDir) { + const staticDirUpload = uploadDirToS3(staticDir, { + bucket: staticAssetsBucket, + prefix: path.basename(staticDir) + }); + + uploadPromises.push(staticDirUpload); + } + + return Promise.all(uploadPromises); +}; diff --git a/lib/uploadStaticAssetsToS3.js b/lib/uploadStaticAssetsToS3.js deleted file mode 100644 index 9cf8b96e07..0000000000 --- a/lib/uploadStaticAssetsToS3.js +++ /dev/null @@ -1,55 +0,0 @@ -const fs = require("fs"); -const path = require("path"); -const walkDir = require("klaw"); -const mime = require("mime"); -const logger = require("../utils/logger"); -const toPosix = require("../utils/pathToPosix"); - -const uploadStaticAssetsToS3 = ({ - staticAssetsPath, - bucketName, - providerRequest -}) => { - return new Promise((resolve, reject) => { - const uploadPromises = []; - - logger.log(`Uploading static assets to ${bucketName} ...`); - - walkDir(staticAssetsPath) - .on("data", item => { - const itemPath = item.path; - const isFile = !fs.lstatSync(itemPath).isDirectory(); - const posixItemPath = toPosix(item.path); - - if (isFile) { - uploadPromises.push( - providerRequest("S3", "upload", { - ACL: "public-read", - Bucket: bucketName, - Key: path.posix.join( - "_next", - posixItemPath.substring( - posixItemPath.indexOf("/static"), - posixItemPath.length - ) - ), - ContentType: mime.getType(itemPath), - Body: fs.createReadStream(itemPath) - }) - ); - } - }) - .on("end", () => { - Promise.all(uploadPromises) - .then(results => { - logger.log("Upload finished"); - resolve(results.length); - }) - .catch(() => { - reject(new Error("File upload failed")); - }); - }); - }); -}; - -module.exports = uploadStaticAssetsToS3; diff --git a/package.json b/package.json index 81ed02be56..7960cad181 100644 --- a/package.json +++ b/package.json @@ -41,6 +41,7 @@ "dependencies": { "@mapbox/s3urls": "^1.5.3", "chalk": "^2.4.2", + "debug": "^4.1.1", "fs-extra": "^7.0.1", "js-yaml": "^3.12.1", "klaw": "^3.0.0", diff --git a/utils/s3/__tests__/get.test.js b/utils/s3/__tests__/get.test.js new file mode 100644 index 0000000000..50372476d5 --- /dev/null +++ b/utils/s3/__tests__/get.test.js @@ -0,0 +1,123 @@ +const getFactory = require("../get"); + +describe("s3 get", () => { + let awsProvider; + let get; + + beforeEach(() => { + awsProvider = jest.fn().mockResolvedValue({ + Contents: [] + }); + get = getFactory(awsProvider); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + it("should list objects using bucket and prefix", () => { + expect.assertions(2); + + const key = "/path/to/smile.jpg"; + const bucket = "my-bucket"; + + awsProvider.mockResolvedValueOnce({ + Contents: [ + { + Key: key + } + ] + }); + + return get(key, bucket).then(object => { + expect(object.Key).toEqual(key); + expect(awsProvider).toBeCalledWith("S3", "listObjectsV2", { + Bucket: bucket, + Prefix: "/path/to" + }); + }); + }); + + it("should not list objects again if key already in cache", () => { + expect.assertions(3); + + const key = "/path/to/ironman.jpg"; + const bucket = "my-bucket"; + + awsProvider.mockResolvedValueOnce({ + Contents: [ + { + Key: key + } + ] + }); + + return get(key, bucket) + .then(() => get(key, bucket)) + .then(object => { + expect(object.Key).toEqual(key); + expect(awsProvider).toBeCalledTimes(1); + expect(awsProvider).toBeCalledWith("S3", "listObjectsV2", { + Bucket: bucket, + Prefix: "/path/to" + }); + }); + }); + + it("should handle paginated response", () => { + expect.assertions(4); + + const key = "/path/to/batman.jpg"; + const bucket = "my-bucket"; + const continuationToken = "123"; + + awsProvider.mockResolvedValueOnce({ + IsTruncated: true, + Contents: [{ Key: "/path/to/bar.jpg" }], + NextContinuationToken: continuationToken + }); + + return get(key, bucket).then(object => { + expect(object).toEqual(undefined); + expect(awsProvider).toBeCalledTimes(2); + expect(awsProvider).toBeCalledWith("S3", "listObjectsV2", { + Bucket: bucket, + Prefix: "/path/to" + }); + expect(awsProvider).toBeCalledWith("S3", "listObjectsV2", { + Bucket: bucket, + Prefix: "/path/to", + ContinuationToken: continuationToken + }); + }); + }); + + it("should not list objects again if key already in cache after paginated response", () => { + expect.assertions(4); + + const bucket = "my-bucket"; + const key = "/path/to/ironman.jpg"; + + awsProvider.mockResolvedValueOnce({ + IsTruncated: true, + Contents: [{ Key: key }], + NextContinuationToken: "123" + }); + + return get(key, bucket) + .then(() => get(key, bucket)) + .then(object => { + expect(object.Key).toEqual(key); + expect(awsProvider).toBeCalledTimes(2); + expect(awsProvider).toBeCalledWith("S3", "listObjectsV2", { + Bucket: bucket, + Prefix: "/path/to" + }); + expect(awsProvider).toBeCalledWith("S3", "listObjectsV2", { + Bucket: bucket, + Prefix: "/path/to", + ContinuationToken: "123" + }); + }); + }); +}); diff --git a/utils/s3/__tests__/upload.test..js b/utils/s3/__tests__/upload.test..js new file mode 100644 index 0000000000..87649e11be --- /dev/null +++ b/utils/s3/__tests__/upload.test..js @@ -0,0 +1,322 @@ +const stream = require("stream"); +const walkDir = require("klaw"); +const fse = require("fs-extra"); +const path = require("path"); +const s3Upload = require("../upload"); +const getFactory = require("../get"); +const logger = require("../../logger"); + +jest.mock("fs-extra"); +jest.mock("klaw"); +jest.mock("../get"); +jest.mock("../../logger"); + +describe("s3Upload", () => { + let upload; + let walkStream; + let awsProvider; + let get; + + beforeEach(() => { + get = jest.fn(); + awsProvider = jest.fn(); + getFactory.mockReturnValue(get); + walkStream = new stream.Readable(); + walkStream._read = () => {}; + walkDir.mockReturnValueOnce(walkStream); + fse.lstat.mockResolvedValue({ isDirectory: () => false }); + fse.createReadStream.mockReturnValue("readStream"); + + upload = s3Upload(awsProvider); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + it("should read from the directory given", () => { + expect.assertions(1); + + const dir = "/path/to/dir"; + + const r = upload(dir, { bucket: "my-bucket" }).then(() => { + expect(walkDir).toBeCalledWith(dir); + }); + + walkStream.emit("end"); + + return r; + }); + + it("should upload files to S3 with correct parameters and resolve with file count", () => { + expect.assertions(5); + + const bucket = "my-bucket"; + + const r = upload("/path/to/dir", { + bucket + }).then(result => { + expect(logger.log).toBeCalledWith( + `Uploading /path/to/dir to ${bucket} ...` + ); + + expect(awsProvider).toBeCalledWith("S3", "upload", { + ContentType: "application/javascript", + ACL: "public-read", + Bucket: bucket, + Key: "/path/to/foo.js", + Body: "readStream" + }); + + expect(awsProvider).toBeCalledWith("S3", "upload", { + ContentType: "text/css", + ACL: "public-read", + Bucket: bucket, + Key: "/path/to/bar.css", + Body: "readStream" + }); + + expect(awsProvider).toBeCalledWith("S3", "upload", { + ContentType: "text/plain", + ACL: "public-read", + Bucket: bucket, + Key: "/path/to/readme.txt", + Body: "readStream" + }); + + expect(result.count).toEqual(3); + }); + + walkStream.emit("data", { + path: "/path/to/foo.js" + }); + + walkStream.emit("data", { + path: "/path/to/bar.css" + }); + + walkStream.emit("data", { + path: "/path/to/readme.txt" + }); + + walkStream.emit("end"); + + return r; + }); + + it("should not try uploading directories", () => { + expect.assertions(1); + + fse.lstat.mockResolvedValue({ isDirectory: () => true }); + + const r = upload("/path/to/dir", { + bucket: "my-bucket" + }).then(() => { + expect(awsProvider).not.toBeCalledWith( + "S3", + "upload", + expect.objectContaining({ + Key: "/path/to/dir/subdir" + }) + ); + }); + + walkStream.emit("data", { + path: "/path/to/dir/subdir" + }); + + walkStream.emit("end"); + + return r; + }); + + it("should handle windows paths", () => { + expect.assertions(1); + + const r = upload("/path/to/dir", { + bucket: "my-bucket" + }).then(() => { + expect(awsProvider).toBeCalledWith( + "S3", + "upload", + expect.objectContaining({ + Key: "/path/to/foo.js" + }) + ); + }); + + walkStream.emit("data", { + path: path.win32.normalize("/path/to/foo.js") + }); + + walkStream.emit("end"); + + return r; + }); + + it("should reject when a file upload fails", () => { + expect.assertions(1); + + awsProvider.mockRejectedValueOnce(new Error("Boom!")); + + const promise = upload("/path/to/dir", { + bucket: "my-bucket" + }).catch(err => { + expect(err.message).toContain("Boom"); + }); + + walkStream.emit("data", { + path: "/path/to/foo.js" + }); + walkStream.emit("end"); + + return promise; + }); + + it("S3 Key should use prefix", () => { + expect.assertions(2); + + const promise = upload("/path/to/dir", { + bucket: "my-bucket", + prefix: "to" + }).then(() => { + expect(awsProvider).toBeCalledWith( + "S3", + "upload", + expect.objectContaining({ + Key: "to/foo.js" + }) + ); + expect(awsProvider).toBeCalledWith( + "S3", + "upload", + expect.objectContaining({ + Key: "to/bar.js" + }) + ); + }); + + walkStream.emit("data", { + path: "/some/path/to/foo.js" + }); + + walkStream.emit("data", { + path: path.win32.normalize("/some/path/to/bar.js") + }); + + walkStream.emit("end"); + + return promise; + }); + + it("S3 Key should use rootPrefix", () => { + expect.assertions(2); + + const promise = upload("/path/to/dir", { + bucket: "my-bucket", + prefix: "/to", + rootPrefix: "blah" + }).then(() => { + expect(awsProvider).toBeCalledWith( + "S3", + "upload", + expect.objectContaining({ + Key: "blah/to/foo.js" + }) + ); + expect(awsProvider).toBeCalledWith( + "S3", + "upload", + expect.objectContaining({ + Key: "blah/to/bar.js" + }) + ); + }); + + walkStream.emit("data", { + path: "/some/path/to/foo.js" + }); + + walkStream.emit("data", { + path: path.win32.normalize("/some/path/to/bar.js") + }); + + walkStream.emit("end"); + + return promise; + }); + + it("should not try to upload file that is already uploaded with same file size", () => { + expect.assertions(2); + + const size = 100; + const key = "/path/to/happyface.jpg"; + + fse.lstat.mockResolvedValueOnce({ + isDirectory: () => false, + size + }); + + get.mockResolvedValue({ + ETag: '"70ee1738b6b21e2c8a43f3a5ab0eee71"', + Key: key, + LastModified: "", + Size: size, + StorageClass: "STANDARD" + }); + + const bucket = "my-bucket"; + + const promise = upload("/path/to/dir", { + bucket + }).then(() => { + expect(get).toBeCalledWith(key, bucket); + expect(awsProvider).not.toBeCalledWith("S3", "upload", expect.anything()); + }); + + walkStream.emit("data", { + path: "/path/to/happyface.jpg" + }); + + walkStream.emit("end"); + + return promise; + }); + + it("should upload file that is already uploaded with with different file size", () => { + expect.assertions(2); + + const size = 100; + const key = "/path/to/happyface.jpg"; + + fse.lstat.mockResolvedValueOnce({ + isDirectory: () => false, + size + }); + + get.mockResolvedValue({ + ETag: '"70ee1738b6b21e2c8a43f3a5ab0eee71"', + Key: key, + LastModified: "", + Size: size + 1, + StorageClass: "STANDARD" + }); + + const bucket = "my-bucket"; + + const promise = upload("/path/to/dir", { + bucket + }).then(() => { + expect(get).toBeCalledWith(key, bucket); + expect(awsProvider).toBeCalledWith("S3", "upload", expect.anything()); + }); + + walkStream.emit("data", { + path: "/path/to/happyface.jpg" + }); + + walkStream.emit("end"); + + return promise; + }); +}); diff --git a/utils/s3/get.js b/utils/s3/get.js new file mode 100644 index 0000000000..93e7c5ed25 --- /dev/null +++ b/utils/s3/get.js @@ -0,0 +1,65 @@ +const path = require("path"); +const debug = require("debug")("sls-next:s3"); + +module.exports = awsProvider => { + const cache = {}; + + const addPrefixToCache = (prefix, listPromise) => { + const updateCache = prefixCache => + listPromise.then(({ Contents }) => { + Contents.forEach(x => (prefixCache[x.Key] = x)); + return prefixCache; + }); + + if (cache[prefix]) { + // already objects have been cached for this prefix + cache[prefix] = cache[prefix].then(prefixCache => + updateCache(prefixCache) + ); + } else { + cache[prefix] = updateCache({}); + } + }; + + return (key, bucket) => { + async function getWithCache(nextContinuationToken) { + const prefix = path.dirname(key); + + const shouldReturnFromCache = cache[prefix] && !nextContinuationToken; + + if (shouldReturnFromCache) { + debug(`cache hit - ${key}`); + const objects = await cache[prefix]; + return objects[key]; + } + + const listParams = { + Bucket: bucket, + Prefix: prefix + }; + + if (nextContinuationToken) { + listParams.ContinuationToken = nextContinuationToken; + } + + debug(`listObjects - ${listParams.Prefix}`); + + const listPromise = awsProvider("S3", "listObjectsV2", listParams); + + addPrefixToCache(prefix, listPromise); + + const { NextContinuationToken, IsTruncated } = await listPromise; + + if (IsTruncated) { + await getWithCache(NextContinuationToken); + } + + const keys = await cache[prefix]; + return keys[key]; + } + + const object = getWithCache(null); + + return object; + }; +}; diff --git a/utils/s3/upload.js b/utils/s3/upload.js new file mode 100644 index 0000000000..185f46df38 --- /dev/null +++ b/utils/s3/upload.js @@ -0,0 +1,76 @@ +const mime = require("mime"); +const walkDir = require("klaw"); +const fse = require("fs-extra"); +const path = require("path"); +const pathToPosix = require("../pathToPosix"); +const get = require("./get"); +const logger = require("../logger"); +const debug = require("debug")("sls-next:s3"); + +const getUploadParameters = (bucket, filePath, prefix, rootPrefix) => { + let key = pathToPosix(filePath); + + if (prefix) { + key = key.substring(key.indexOf(prefix), key.length); + } + + if (rootPrefix) { + key = path.posix.join(rootPrefix, key); + } + + return { + ACL: "public-read", + Bucket: bucket, + Key: key, + ContentType: mime.getType(key), + Body: fse.createReadStream(filePath) + }; +}; + +const filesAreEqual = (s3Object, fStats) => + s3Object && fStats.size === s3Object.Size; + +module.exports = awsProvider => ( + dir, + { bucket, prefix = null, rootPrefix = null } +) => { + const getObjectFromS3 = get(awsProvider); + const promises = []; + + logger.log(`Uploading ${dir} to ${bucket} ...`); + + return new Promise((resolve, reject) => { + walkDir(dir) + .on("data", item => { + const p = fse.lstat(item.path).then(async stats => { + if (!stats.isDirectory()) { + const uploadParams = getUploadParameters( + bucket, + item.path, + prefix, + rootPrefix + ); + + const s3Object = await getObjectFromS3(uploadParams.Key, bucket); + + if (filesAreEqual(s3Object, stats)) { + debug(`no need to upload ${uploadParams.Key}`); + return Promise.resolve(); + } + + debug(`uploading to s3 - ${uploadParams.Key}`); + return awsProvider("S3", "upload", uploadParams); + } + }); + + promises.push(p); + }) + .on("end", () => { + Promise.all(promises) + .then(() => { + resolve({ count: promises.length }); + }) + .catch(reject); + }); + }); +};