Deploy electricityMap static assets to GCS (electricitymaps#2869)

* Deploy electricityMap static assets to GCS

* update dockerfile paths

* Revert

* Fix dockerfile

* Add config

* update container path

* No need to clean dir

* rename env

.circleci/config.yml

@@ -80,7 +80,7 @@ jobs:
             docker-compose build
             # Make sure files are available outside of container
             CONTAINER_ID=$(docker create eu.gcr.io/tmrow-152415/electricitymap_web:latest)
-            docker cp $CONTAINER_ID:/home/web/public/dist web/public/dist
+            docker cp $CONTAINER_ID:/home/src/electricitymap/contrib/web/public/dist web/public/dist
           no_output_timeout: 30m
       - run:
           name: Start environment

docker-compose.yml

@@ -22,18 +22,18 @@ services:
     ports: ['8000:8000']
     volumes:
       - './config:/home/config'
-      - './web/.eslintrc:/home/web/.eslintrc'
-      - './web/generate-geometries.js:/home/web/generate-geometries.js'
-      - './web/locales:/home/web/locales'
-      - './web/locales-config.json:/home/web/locales-config.json'
-      - './web/package.json:/home/web/package.json'
-      - './web/public:/home/web/public'
-      - './web/server.js:/home/web/server.js'
-      - './web/src:/home/web/src'
-      - './web/third_party_maps:/home/web/third_party_maps'
-      - './web/topogen.sh:/home/web/topogen.sh'
-      - './web/views:/home/web/views'
-      - './web/webpack.config.js:/home/web/webpack.config.js'
+      - './web/.eslintrc:/home/src/electricitymap/contrib/web/.eslintrc'
+      - './web/generate-geometries.js:/home/src/electricitymap/contrib/web/generate-geometries.js'
+      - './web/locales:/home/src/electricitymap/contrib/web/locales'
+      - './web/locales-config.json:/home/src/electricitymap/contrib/web/locales-config.json'
+      - './web/package.json:/home/src/electricitymap/contrib/web/package.json'
+      - './web/public:/home/src/electricitymap/contrib/web/public'
+      - './web/server.js:/home/src/electricitymap/contrib/web/server.js'
+      - './web/src:/home/src/electricitymap/contrib/web/src'
+      - './web/third_party_maps:/home/src/electricitymap/contrib/web/third_party_maps'
+      - './web/topogen.sh:/home/src/electricitymap/contrib/web/topogen.sh'
+      - './web/views:/home/src/electricitymap/contrib/web/views'
+      - './web/webpack.config.js:/home/src/electricitymap/contrib/web/webpack.config.js'
   web-watch:
     build:
       context: .
@@ -48,15 +48,15 @@ services:
       - NODE_ENV=development
     volumes:
       - './config:/home/config'
-      - './web/.eslintrc:/home/web/.eslintrc'
-      - './web/generate-geometries.js:/home/web/generate-geometries.js'
-      - './web/locales:/home/web/locales'
-      - './web/locales-config.json:/home/web/locales-config.json'
-      - './web/package.json:/home/web/package.json'
-      - './web/public:/home/web/public'
-      - './web/server.js:/home/web/server.js'
-      - './web/src:/home/web/src'
-      - './web/third_party_maps:/home/web/third_party_maps'
-      - './web/topogen.sh:/home/web/topogen.sh'
-      - './web/views:/home/web/views'
-      - './web/webpack.config.js:/home/web/webpack.config.js'
+      - './web/.eslintrc:/home/src/electricitymap/contrib/web/.eslintrc'
+      - './web/generate-geometries.js:/home/src/electricitymap/contrib/web/generate-geometries.js'
+      - './web/locales:/home/src/electricitymap/contrib/web/locales'
+      - './web/locales-config.json:/home/src/electricitymap/contrib/web/locales-config.json'
+      - './web/package.json:/home/src/electricitymap/contrib/web/package.json'
+      - './web/public:/home/src/electricitymap/contrib/web/public'
+      - './web/server.js:/home/src/electricitymap/contrib/web/server.js'
+      - './web/src:/home/src/electricitymap/contrib/web/src'
+      - './web/third_party_maps:/home/src/electricitymap/contrib/web/third_party_maps'
+      - './web/topogen.sh:/home/src/electricitymap/contrib/web/topogen.sh'
+      - './web/views:/home/src/electricitymap/contrib/web/views'
+      - './web/webpack.config.js:/home/src/electricitymap/contrib/web/webpack.config.js'

web/BUILD.yaml

@@ -9,18 +9,36 @@ steps:
     commands:
       - yarn
 
-  # This script is used to generate zonegeometries.json
   build:
+    environment:
+      ELECTRICITYMAP_PUBLIC_TOKEN: ${BRICK_ELECTRICITYMAP_PUBLIC_TOKEN}
     inputs:
       # geometries
       - generate-geometries.js
       - topogen.sh
       - third_party_maps
       # source code
       - ../config/{exchanges,zones}.json
+      - locales
+      - ./{locales-config.json,translation-status.js}
+      - public/{css,fonts,images,apple-app-site-association,browserconfig.xml,manifest.json}
+      - src
+      - views
+      - ./{.babelrc,.eslintrc,server.js,webpack.config.js}
     commands:
       - mkdir -p public/dist
-      - mkdir -p src
       - bash topogen.sh
+      - yarn build-release
     outputs:
-      - public/dist/zonegeometries.json
+      - src/world.json
+      - public/dist
+    tag: eu.gcr.io/tmrow-152415/electricitymap_web
+
+  deploy:
+    image: google/cloud-sdk:243.0.0
+    commands:
+      - gsutil -m cp -a public-read -r public/* gs://static.electricitymap.org/public_web
+    secrets:
+      gcloud:
+        src: ~/.config/gcloud
+        target: /root/.config/gcloud

web/Dockerfile

@@ -1,26 +1,26 @@
 FROM node:12.13.1
-WORKDIR /home/web
+WORKDIR /home/src/electricitymap/contrib/web
 
 # Install dependencies
 RUN apt-get update && apt-get install -y jq unzip
-ADD web/package.json /home/web/package.json
-ADD web/yarn.lock /home/web/yarn.lock
+ADD web/package.json ./package.json
+ADD web/yarn.lock ./yarn.lock
 RUN yarn
 
 # Generate map
 RUN mkdir -p public/dist/
-ADD web/build /home/web/build
-ADD web/third_party_maps /home/web/third_party_maps
-ADD web/generate-geometries.js web/topogen.sh /home/web/
-ADD web/src/world.json /home/web/src/world.json
+ADD web/build ./build
+ADD web/third_party_maps ./third_party_maps
+ADD web/generate-geometries.js web/topogen.sh ./
+ADD web/src/world.json ./src/world.json
 RUN bash topogen.sh
 
 ARG ELECTRICITYMAP_PUBLIC_TOKEN
 
 # Build
 # (note: this will override the world.json that was previously created)
-ADD config /home/config
-ADD web /home/web
+ADD config /home/src/electricitymap/contrib/config
+ADD web ./
 RUN yarn lint && yarn build-release
 
 EXPOSE 8000

web/package.json

@@ -60,7 +60,6 @@
     "babel-polyfill": "^6.26.0",
     "babel-preset-es2015": "^6.24.1",
     "babel-runtime": "^6.26.0",
-    "clean-webpack-plugin": "^3.0.0",
     "colors": "^1.3.2",
     "css-loader": "^0.28.10",
     "eslint": "^5.5.0",

web/server.js

@@ -15,7 +15,7 @@ const {
   getSingleTranslationStatusJSON,
   getTranslationStatusJSON,
   getTranslationStatusSVG,
-} = require(__dirname + '/translation-status');
+} = require(`${__dirname}/translation-status`);
 const {
   localeToFacebookLocale,
   supportedFacebookLocales,
@@ -26,7 +26,7 @@ const app = express();
 const server = http.Server(app);
 
 // Constants
-const STATIC_PATH = process.env['STATIC_PATH'] || (__dirname + '/public');
+const STATIC_PATH = process.env.STATIC_PATH || (`${__dirname}/public`);
 
 // * Common
 app.use(compression()); // Cloudflare already does gzip but we do it anyway
@@ -46,7 +46,7 @@ i18n.configure({
   // where to store json files - defaults to './locales' relative to modules directory
   // note: detected locales are always lowercase
   locales,
-  directory: __dirname + '/locales',
+  directory: `${__dirname}/locales`,
   defaultLocale: 'en',
   queryParameter: 'lang',
   objectNotation: true,
@@ -85,24 +85,18 @@ function translateWithLocale(locale, keyStr) {
 // * Long-term caching
 function getHash(key, ext, obj) {
   let filename;
-  if (typeof obj.assetsByChunkName[key] == 'string') {
+  if (typeof obj.assetsByChunkName[key] === 'string') {
     filename = obj.assetsByChunkName[key];
   } else {
     // assume list
     filename = obj.assetsByChunkName[key]
-      .filter((d) => d.match(new RegExp('\.' + ext + '$')))[0]
+      .filter(d => d.match(new RegExp(`\.${ext}$`)))[0];
   }
-  return filename.replace('.' + ext, '').replace(key + '.', '');
+  return filename.replace(`.${ext}`, '').replace(`${key}.`, '');
 }
 
 const manifest = JSON.parse(fs.readFileSync(`${STATIC_PATH}/dist/manifest.json`));
 
-// * Error handling
-function handleError(err) {
-  if (!err) return;
-  console.error(err);
-}
-
 app.get('/health', (req, res) => res.json({ status: 'ok' }));
 app.get('/clientVersion', (req, res) => res.send(version));
 
@@ -115,10 +109,8 @@ app.get('/translationstatus', (req, res) => res.json(getTranslationStatusJSON(lo
 app.get('/translationstatus/:language', (req, res) => res.json(getSingleTranslationStatusJSON(req.params.language)));
 
 // API
-app.get('/v1/*', (req, res) =>
-  res.redirect(301, `https://api.electricitymap.org${req.originalUrl}`));
-app.get('/v2/*', (req, res) =>
-  res.redirect(301, `https://api.electricitymap.org${req.originalUrl}`));
+app.get('/v1/*', (req, res) => res.redirect(301, `https://api.electricitymap.org${req.originalUrl}`));
+app.get('/v2/*', (req, res) => res.redirect(301, `https://api.electricitymap.org${req.originalUrl}`));
 
 // Source maps
 app.all('/dist/*.map', (req, res, next) => {
@@ -143,20 +135,20 @@ app.use('/', (req, res) => {
   // redirect everyone except the Facebook crawler,
   // else, we will lose all likes
   const isTmrowCo = req.get('host').indexOf('electricitymap.tmrow') !== -1;
-  const isNonWWW = req.get('host') === 'electricitymap.org' ||
-    req.get('host') === 'live.electricitymap.org';
+  const isNonWWW = req.get('host') === 'electricitymap.org'
+    || req.get('host') === 'live.electricitymap.org';
   const isStaging = req.get('host') === 'staging.electricitymap.org';
   const isHTTPS = req.secure;
   const isLocalhost = req.hostname === 'localhost'; // hostname is without port
 
   // Redirect all non-facebook, non-staging, non-(www.* or *.tmrow.co)
   if (!isStaging && (isNonWWW || isTmrowCo) && (req.headers['user-agent'] || '').indexOf('facebookexternalhit') == -1) {
-    res.redirect(301, 'https://www.electricitymap.org' + req.originalUrl);
+    res.redirect(301, `https://www.electricitymap.org${req.originalUrl}`);
   // Redirect all non-HTTPS and non localhost
   // Warning: this can't happen here because Cloudfare is the HTTPS proxy.
   // Node only receives HTTP traffic.
   } else if (false && !isHTTPS && !isLocalhost) {
-    res.redirect(301, 'https://www.electricitymap.org' + req.originalUrl);
+    res.redirect(301, `https://www.electricitymap.org${req.originalUrl}`);
   } else {
     // Set locale if facebook requests it
     if (req.query.fb_locale) {
@@ -166,7 +158,7 @@ app.use('/', (req, res) => {
       res.setLocale(lr[0]);
     }
     const { locale } = res;
-    const fullUrl = 'https://www.electricitymap.org' + req.originalUrl;
+    const fullUrl = `https://www.electricitymap.org${req.originalUrl}`;
 
     // basic auth for premium access
     if (process.env.BASIC_AUTH_CREDENTIALS) {
@@ -186,19 +178,17 @@ app.use('/', (req, res) => {
         res.end('Access denied');
         return;
       }
-      res.cookie('electricitymap-token', process.env['ELECTRICITYMAP_TOKEN']);
+      res.cookie('electricitymap-token', process.env.ELECTRICITYMAP_TOKEN);
     }
     res.render('pages/index', {
-      alternateUrls: locales.map(function(l) {
+      alternateUrls: locales.map((l) => {
         if (fullUrl.indexOf('lang') !== -1) {
-          return fullUrl.replace('lang=' + req.query.lang, 'lang=' + l)
-        } else {
-          if (Object.keys(req.query).length) {
-            return fullUrl + '&lang=' + l;
-          } else {
-            return fullUrl.replace('?', '') + '?lang=' + l;
-          }
+          return fullUrl.replace(`lang=${req.query.lang}`, `lang=${l}`);
         }
+        if (Object.keys(req.query).length) {
+          return `${fullUrl}&lang=${l}`;
+        }
+        return `${fullUrl.replace('?', '')}?lang=${l}`;
       }),
       bundleHash: getHash('bundle', 'js', manifest),
       vendorHash: getHash('vendor', 'js', manifest),
@@ -207,14 +197,22 @@ app.use('/', (req, res) => {
       // Make the paths absolute as that's required for BrowserHistory routing
       // to work normally and it's also ok when used with the https:// protocol
       // as resources are mounted to a fixed location.
-      resolvePath: function(relativePath) { return '/' + relativePath; },
+      // Note: `resolvePath` is executed on the client as well,
+      // as it is used in react components. We can't therefore include any variables
+      // in its closure. It would be better to pass a `pathPrefix` instead.
+      resolvePath: (!isProduction || isStaging)
+        ? relativePath => `/${relativePath}`
+        : relativePath =>
+          // Note we here point to static hosting in order to make
+          // sure we can serve older bundle versions
+          `https://static.electricitymap.org/public_web/${relativePath}`,
       fullUrl,
       locale,
-      locales: { en: localeConfigs['en'], [locale]: localeConfigs[locale] },
+      locales: { en: localeConfigs.en, [locale]: localeConfigs[locale] },
       supportedLocales: locales,
       FBLocale: localeToFacebookLocale[locale],
       supportedFBLocales: supportedFacebookLocales,
-      '__': function() {
+      __() {
         const argsArray = Array.prototype.slice.call(arguments);
         // Prepend the first argument which is the locale
         argsArray.unshift(locale);
@@ -224,7 +222,13 @@ app.use('/', (req, res) => {
   }
 });
 
+if (isProduction) {
+  app.get('/*', (req, res) =>
+    // Redirect all requests except root to static
+    res.redirect(`https://static.electricitymap.org/public_web${req.originalUrl}`));
+}
+
 // Start the application
-server.listen(process.env['PORT'], () => {
-  console.log(`Listening on *:${process.env['PORT']}`);
+server.listen(process.env.PORT, () => {
+  console.log(`Listening on *:${process.env.PORT}`);
 });

web/webpack.config.js

@@ -4,7 +4,6 @@ const autoprefixer = require('autoprefixer');
 
 const MiniCssExtractPlugin = require('mini-css-extract-plugin');
 const OptimizeCssAssetsPlugin = require('optimize-css-assets-webpack-plugin');
-const { CleanWebpackPlugin } = require('clean-webpack-plugin');
 
 const isProduction = process.env.NODE_ENV === 'production';
 
@@ -41,7 +40,6 @@ module.exports = {
     ],
   },
   plugins: [
-    new CleanWebpackPlugin(),
     new OptimizeCssAssetsPlugin(),
     new MiniCssExtractPlugin({
       filename: '[name].' + (isProduction ? '[chunkhash]' : 'dev') + '.css',