-
Notifications
You must be signed in to change notification settings - Fork 394
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Migrate deployment method from the blog repo (#1105)
* Migrate deployment process to s3 from the blog * Add trailing slash redirect to redirects list * Allow all node versions great or equal than 12.0.0 * Migrate cloudflare worker redirects to redirects-list.json * Allow second redirect to remove trailing slash. Add tests for new redirects * Exclude redirects-list.json file from links checker Co-authored-by: Pavel Grinchenko <[email protected]>
- Loading branch information
1 parent
bb36d06
commit 847d4ad
Showing
22 changed files
with
529 additions
and
95 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,21 +1,29 @@ | ||
| [ | ||
| "^https://blog\\.dvc\\.org/blog/(.*?)/?$ https://dvc.org/blog/$1", | ||
| "^https://blog\\.dvc\\.org/(.*?)/?$ https://dvc.org/blog/$1", | ||
| "^https://(?:www\\.)dvc\\.org/(.+)? https://dvc.org/$1", | ||
| "^https://man\\.dvc\\.org/(.+)? https://dvc.org/doc/command-reference/$1 303", | ||
| "^https://error\\.dvc\\.org/(.+)? https://dvc.org/doc/user-guide/troubleshooting#$1 303", | ||
| "^https://(code|data|remote)\\.dvc\\.org/(.+) https://s3-us-east-2.amazonaws.com/dvc-public/$1/$2 303", | ||
| "^/((?:deb|rpm)/.+) https://s3-us-east-2.amazonaws.com/dvc-s3-repo/$1 303", | ||
| "^/(?:help|chat)/?$ https://discordapp.com/invite/dvwXA2N 303", | ||
| "^/(?:docs|documentation)(/.*)?$ /doc$1", | ||
| "^/doc/?$ /doc/home 307", | ||
| "^/doc/get-started(/.*)?$ /doc/tutorials/get-started$1", | ||
| "^/doc/tutorials/get-started/?$ /doc/tutorials/get-started/agenda 307", | ||
| "^/doc/tutorial/?$ /doc/tutorials", | ||
| "^/doc/tutorial/(.*)? /doc/tutorials/deep/$1", | ||
| "^/doc/commands-reference(/.*)?$ /doc/command-reference$1", | ||
| "^/doc/use-cases/data-and-model-files-versioning/?$ /doc/use-cases/versioning-data-and-model-files", | ||
| "^/doc/user-guide/contributing/?$ /doc/user-guide/contributing/core 307", | ||
| "^/doc/understanding-dvc/?$ /doc/understanding-dvc/collaboration-issues 307", | ||
| "^/doc/changelog/?$ /doc/changelog/0.18 307" | ||
| "^https://(?:www\\.)dataversioncontrol\\.com(.*)? https://dataversioncontrol.com$1 301", | ||
| "^https://blog\\.dataversioncontrol\\.com/data-version-control-tutorial-9146715eda46 https://dvc.org/doc/tutorials 301", | ||
| "^https://blog\\.dataversioncontrol\\.com/data-version-control-beta-release-iterative-machine-learning-a7faf7c8be67 https://dvc.org/doc/tutorials 301", | ||
| "^https://blog\\.dataversioncontrol\\.com/dvc-heartbeat-6301aebf5c96 https://dvc.org/blog/march-19-dvc-heartbeat 301", | ||
| "^https://blog\\.dataversioncontrol\\.com/april19-dvc-heartbeat-296c71a59be4 https://dvc.org/blog/april-19-dvc-heartbeat 301", | ||
| "^https://blog\\.dataversioncontrol\\.com/dvc-0-8-5-release-f66ef3b10684 https://github.com/iterative/dvc/releases 301", | ||
| "^https://blog\\.dataversioncontrol\\.com(.+)-[a-z0-9]{12}$ https://dvc.org/blog$1 301", | ||
| "^https://blog\\.dvc\\.org/blog/(.*?)/?$ https://dvc.org/blog/$1", | ||
| "^https://blog\\.dvc\\.org/(.*?)/?$ https://dvc.org/blog/$1", | ||
| "^https://(?:www\\.)dvc\\.org(.*)? https://dvc.org$1", | ||
| "^https://man\\.dvc\\.org(.*)? https://dvc.org/doc/command-reference$1 303", | ||
| "^https://error\\.dvc\\.org/(.+)? https://dvc.org/doc/user-guide/troubleshooting#$1 303", | ||
| "^https://(code|data|remote)\\.dvc\\.org/(.+) https://s3-us-east-2.amazonaws.com/dvc-public/$1/$2 303", | ||
| "^/((?:deb|rpm)/.+) https://s3-us-east-2.amazonaws.com/dvc-s3-repo/$1 303", | ||
| "^/(?:help|chat)/?$ https://discordapp.com/invite/dvwXA2N 303", | ||
| "^/(?:docs|documentation)(/.*)?$ /doc$1", | ||
| "^/doc/?$ /doc/home 307", | ||
| "^/doc/get-started(/.*)?$ /doc/tutorials/get-started$1", | ||
| "^/doc/tutorials/get-started/?$ /doc/tutorials/get-started/agenda 307", | ||
| "^/doc/tutorial/?$ /doc/tutorials", | ||
| "^/doc/tutorial/(.*)? /doc/tutorials/deep/$1", | ||
| "^/doc/commands-reference(/.*)?$ /doc/command-reference$1", | ||
| "^/doc/use-cases/data-and-model-files-versioning/?$ /doc/use-cases/versioning-data-and-model-files", | ||
| "^/doc/user-guide/contributing/?$ /doc/user-guide/contributing/core 307", | ||
| "^/doc/understanding-dvc/?$ /doc/understanding-dvc/collaboration-issues 307", | ||
| "^/doc/changelog/?$ /doc/changelog/0.18 307", | ||
| "^/(.+)/$ /$1 301" | ||
| ] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,130 @@ | ||
| #!/usr/bin/env node | ||
| 'use strict' | ||
|
|
||
| /** | ||
| * Build gatsby site and deploy public/ to s3. | ||
| * | ||
| * The S3 path of the deployment depends on the HEROKU_APP_NAME variable, | ||
| * which is passed to PRs by heroku, but you can set locally too. | ||
| * | ||
| * With HEROKU_APP_NAME: /dvc-org-pulls/$HEROKU_APP_NAME | ||
| * Without HEROKU_APP_NAME: /dvc-org-prod | ||
| * | ||
| * Needs following environment variables: | ||
| * | ||
| * - S3_BUCKET: name of the bucket | ||
| * - AWS_REGION: region of the bucket | ||
| * - AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY: auth token to access the bucket. | ||
| * - HEROKU_APP_NAME: (optional) app name to specify the ID of the PR if any. | ||
| **/ | ||
|
|
||
| const path = require('path') | ||
| const { execSync } = require('child_process') | ||
| const { remove, move, ensureDir } = require('fs-extra') | ||
| const { s3Prefix, s3Bucket, s3Client } = require('./s3-utils') | ||
|
|
||
| const rootDir = path.join(__dirname, '..') | ||
| const cacheDir = path.join(rootDir, '.cache') | ||
| const publicDir = path.join(rootDir, 'public') | ||
|
|
||
| function run(command) { | ||
| execSync(command, { | ||
| stdio: ['pipe', process.stdout, process.stderr] | ||
| }) | ||
| } | ||
|
|
||
| function syncCall(method, ...args) { | ||
| return new Promise((resolve, reject) => { | ||
| const synchroniser = s3Client[method](...args) | ||
| synchroniser.on('error', reject) | ||
| synchroniser.on('end', resolve) | ||
| }) | ||
| } | ||
|
|
||
| async function prefixIsEmpty(prefix) { | ||
| try { | ||
| await s3Client.s3 | ||
| .headObject({ | ||
| Bucket: s3Bucket, | ||
| Prefix: prefix + '/index.html' | ||
| }) | ||
| .promise() | ||
| return false | ||
| } catch (e) { | ||
| return true | ||
| } | ||
| } | ||
|
|
||
| async function downloadFromS3(prefix) { | ||
| try { | ||
| const staticDir = path.join(publicDir, 'static') | ||
| const staticPrefix = prefix + '/static' | ||
| await ensureDir(staticDir) | ||
|
|
||
| console.log( | ||
| `downloading public/static from s3://${s3Bucket}/${staticPrefix}` | ||
| ) | ||
| console.time('download from s3') | ||
| await syncCall('downloadDir', { | ||
| localDir: staticDir, | ||
| s3Params: { | ||
| Bucket: s3Bucket, | ||
| Prefix: staticPrefix | ||
| } | ||
| }) | ||
| console.timeEnd('download from s3') | ||
| } catch (downloadError) { | ||
| console.error('Error downloading initial data', downloadError) | ||
| // Don't propagate. It's just a cache warming step | ||
| } | ||
| } | ||
|
|
||
| async function uploadToS3() { | ||
| console.log(`Uploading public/ to s3://${s3Bucket}/${s3Prefix}`) | ||
| console.time('upload to s3') | ||
| await syncCall('uploadDir', { | ||
| localDir: publicDir, | ||
| deleteRemoved: true, | ||
| s3Params: { | ||
| Bucket: s3Bucket, | ||
| Prefix: s3Prefix | ||
| } | ||
| }) | ||
| console.timeEnd('upload to s3') | ||
| } | ||
|
|
||
| async function main() { | ||
| const emptyPrefix = await prefixIsEmpty(s3Prefix) | ||
|
|
||
| // First build of a PR is slow because it can't reuse cache. | ||
| // But we can download from prod to warm cache up. | ||
| const cacheWarmPrefix = emptyPrefix ? 'dvc-org-prod' : s3Prefix | ||
|
|
||
| await downloadFromS3(cacheWarmPrefix) | ||
|
|
||
| try { | ||
| run('yarn build') | ||
| } catch (buildError) { | ||
| // Sometimes gatsby build fails because of bad cache. | ||
| // Clear it and try again. | ||
|
|
||
| console.error('------------------------\n\n') | ||
| console.error(buildError) | ||
| console.error('\nAssuming bad cache and retrying:\n') | ||
|
|
||
| await remove(cacheDir) | ||
| await remove(publicDir) | ||
| run('yarn build') | ||
| } | ||
|
|
||
| await move(path.join(publicDir, '404.html'), path.join(rootDir, '404.html'), { | ||
| overwrite: true | ||
| }) | ||
| await uploadToS3() | ||
| await remove(publicDir) | ||
| } | ||
|
|
||
| main().catch(e => { | ||
| console.error(e) | ||
| process.exit(1) | ||
| }) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,32 @@ | ||
| 'use strict' | ||
|
|
||
| const { s3Prefix, s3Bucket } = require('../src/server/config') | ||
| const s3 = require('s3-client') | ||
|
|
||
| const { | ||
| AWS_REGION, | ||
| AWS_ACCESS_KEY_ID, | ||
| AWS_SECRET_ACCESS_KEY, | ||
| HEROKU_APP_NAME | ||
| } = process.env | ||
|
|
||
| const s3Client = s3.createClient({ | ||
| maxAsyncS3: 50, | ||
| region: AWS_REGION, | ||
| accessKeyId: AWS_ACCESS_KEY_ID, | ||
| secretAccessKey: AWS_SECRET_ACCESS_KEY | ||
| }) | ||
|
|
||
| console.log({ | ||
| AWS_REGION, | ||
| HEROKU_APP_NAME, | ||
| s3Bucket, | ||
| s3Prefix, | ||
| hasCreds: Boolean(AWS_ACCESS_KEY_ID && AWS_SECRET_ACCESS_KEY) | ||
| }) | ||
|
|
||
| module.exports = { | ||
| s3Bucket, | ||
| s3Prefix, | ||
| s3Client | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,11 @@ | ||
| 'use strict' | ||
|
|
||
| const { AWS_REGION, S3_BUCKET, HEROKU_APP_NAME } = process.env | ||
|
|
||
| const s3Prefix = HEROKU_APP_NAME | ||
| ? `dvc-org-pulls/${HEROKU_APP_NAME}` | ||
| : 'dvc-org-prod' | ||
| const s3Bucket = S3_BUCKET | ||
| const s3Url = `http://${s3Bucket}.s3-website.${AWS_REGION}.amazonaws.com/${s3Prefix}` | ||
|
|
||
| module.exports = { s3Prefix, s3Bucket, s3Url } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,43 @@ | ||
| /* | ||
| * Production server. Proxies to S3 depending on HEROKU_APP_NAME (see | ||
| * scripts/deploy-with-s3.js) | ||
| * | ||
| * NOTE: This file doesn't go through babel or webpack. Make sure the syntax and | ||
| * sources this file requires are compatible with the current node version you | ||
| * are running. | ||
| * | ||
| * Required environment variables: | ||
| * | ||
| * - S3_BUCKET: name of the bucket | ||
| * - AWS_REGION: region of the bucket | ||
| * - AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY: IAM token to access bucket | ||
| * - HEROKU_APP_NAME: If this is a PR, an ID of the PR. Don't add this for | ||
| * production. | ||
| */ | ||
|
|
||
| const express = require('express') | ||
| const compression = require('compression') | ||
| const { s3Url } = require('./config') | ||
| const { isProduction } = require('./utils') | ||
|
|
||
| const port = process.env.PORT || 3000 | ||
| const app = express() | ||
|
|
||
| const apiMiddleware = require('./middleware/api') | ||
| const redirectsMiddleware = require('./middleware/redirects') | ||
| const serveMiddleware = require('./middleware/serve') | ||
|
|
||
| app.use(compression()) | ||
| app.use(redirectsMiddleware) | ||
| app.use('/api', apiMiddleware) | ||
| app.use(serveMiddleware) | ||
|
|
||
| app.listen(port, () => { | ||
| console.log(`Listening on http://0.0.0.0:${port}/`) | ||
|
|
||
| if (isProduction) { | ||
| console.log(`Proxying to ${s3Url}`) | ||
| } else { | ||
| console.log('Serving static files from local') | ||
| } | ||
| }) |
Oops, something went wrong.