Merge branch 'master' into update-dockerfile-naming

docs/using_lagoon/drupal/lagoonize.md

@@ -2,9 +2,9 @@
 
 ## 1. Lagoon Drupal Setting Files
 
-In order for Drupal to work with Lagoon we need to teach Drupal about Lagoon and Lagoon about Drupal. This happens with copying specific YAML and PHP Files into your Git Repository.
+In order for Drupal to work with Lagoon we need to teach Drupal about Lagoon and Lagoon about Drupal. This happens by copying specific YAML and PHP Files into your Git repository.
 
-You find these Files [here](https://github.com/amazeeio/lagoon/tree/master/docs/using_lagoon/drupal). The easiest way is to download them as [ZIP File](https://minhaskamal.github.io/DownGit/#/home?url=https://github.com/amazeeio/lagoon/tree/master/docs/using_lagoon/drupal) and copy them into your Git Repository. For each Drupal Version and Database Type you will find an individual folder. A short overview of what they are:
+You find [these Files in our GitHub repository](https://github.com/amazeeio/lagoon/tree/master/docs/using_lagoon/drupal); the easiest way is to [download these files as a ZIP file](https://minhaskamal.github.io/DownGit/#/home?url=https://github.com/amazeeio/lagoon/tree/master/docs/using_lagoon/drupal) and copy them into your Git repository. For each Drupal version and database type you will find an individual folder. A short overview of what they are:
 
 - `.lagoon.yml` - The main file that will be used by Lagoon to understand what should be deployed and many more things. This file has some sensible Drupal defaults, if you would like to edit or modify, please check the specific [Documentation for .lagoon.yml](/using_lagoon/lagoon_yml.md)
 - `docker-compose.yml`, `.dockerignore`, and `*.dockerfile` (or `Dockerfile`) - These files are used to run your Local Drupal Development environment, they tell docker which services to start and how to build them. They contain sensible defaults and many commented lines, it should be pretty much self describing. If you would like to find out more, see [Documentation for docker-compose.yml]()

images/oc-build-deploy-dind/build-deploy-docker-compose.sh

@@ -460,7 +460,7 @@ else
 fi
 
 # If restic backups are supported by this cluster we create the schedule definition
-if oc auth --insecure-skip-tls-verify can-i create schedules.backup.appuio.ch -q > /dev/null; then
+if oc auth --insecure-skip-tls-verify -n ${OPENSHIFT_PROJECT} can-i create schedules.backup.appuio.ch -q > /dev/null; then
 
   if ! oc --insecure-skip-tls-verify -n ${OPENSHIFT_PROJECT} get secret baas-repo-pw &> /dev/null; then
     # Create baas-repo-pw secret based on the project secret
@@ -750,7 +750,7 @@ do
   fi
 
   # Generate Backup Definitions are supported and if service type defines one
-  if oc auth --insecure-skip-tls-verify can-i create prebackuppod.backup.appuio.ch -q > /dev/null; then
+  if oc auth --insecure-skip-tls-verify -n ${OPENSHIFT_PROJECT} can-i create prebackuppod.backup.appuio.ch -q > /dev/null; then
     OPENSHIFT_SERVICES_TEMPLATE="/oc-build-deploy/openshift-templates/${SERVICE_TYPE}/prebackuppod.yml"
     if [ -f $OPENSHIFT_SERVICES_TEMPLATE ]; then
       OPENSHIFT_TEMPLATE=$OPENSHIFT_SERVICES_TEMPLATE

images/oc-build-deploy-dind/build-deploy.sh

@@ -45,6 +45,8 @@ DEPLOYER_TOKEN=$(cat /var/run/secrets/lagoon/deployer/token)
 oc login --insecure-skip-tls-verify --token="${DEPLOYER_TOKEN}" https://kubernetes.default.svc
 set -x
 
+oc project --insecure-skip-tls-verify $OPENSHIFT_PROJECT
+
 ADDITIONAL_YAMLS=($(cat .lagoon.yml | shyaml keys additional-yaml || echo ""))
 
 for ADDITIONAL_YAML in "${ADDITIONAL_YAMLS[@]}"

images/oc-build-deploy-dind/openshift-templates/cli-persistent/deployment.yml

@@ -120,6 +120,8 @@ objects:
                 requests:
                   cpu: 10m
                   memory: 10Mi
+                limits:
+                  memory: 8Gi
       test: false
       triggers:
         - type: ConfigChange

images/oc-build-deploy-dind/openshift-templates/cli/deployment.yml

@@ -101,6 +101,8 @@ objects:
                 requests:
                   cpu: 10m
                   memory: 10Mi
+                limits:
+                  memory: 8Gi
       test: false
       triggers:
         - type: ConfigChange

node-packages/commons/package.json

@@ -26,6 +26,7 @@
     "jsonwebtoken": "^8.0.1",
     "kubernetes-client": "^3.15.0",
     "lokka": "^1.7.0",
+    "node-fetch": "^2.6.0",
     "ramda": "^0.25.0",
     "winston": "^2.4.0",
     "winston-logstash": "^0.4.0"

node-packages/commons/src/api.js

@@ -11,7 +11,7 @@ import type {
 } from './types';
 
 const { Lokka } = require('lokka');
-const { Transport } = require('@lagoon/lokka-transport-http');
+const { Transport } = require('./lokka-transport-http-retry');
 const R = require('ramda');
 const { createJWTWithoutUserId } = require('./jwt');
 const { logger } = require('./local-logging');
@@ -249,6 +249,25 @@ const getAllEnvironmentBackups = (): Promise<Project[]> =>
 `,
   );
 
+const getEnvironmentBackups = (openshiftProjectName: string): Promise<Project[]> =>
+  graphqlapi.query(
+    `
+  query environmentByOpenshiftProjectName($openshiftProjectName: String!) {
+    environmentByOpenshiftProjectName(openshiftProjectName: $openshiftProjectName) {
+      id
+      name
+      openshiftProjectName
+      project {
+        name
+      }
+      backups {
+        ...${backupFragment}
+      }
+    }
+  }
+`, { openshiftProjectName }
+  );
+
 const updateCustomer = (id: number, patch: CustomerPatch): Promise<Object> =>
   graphqlapi.mutate(
     `
@@ -965,6 +984,7 @@ module.exports = {
   deleteBackup,
   updateRestore,
   getAllEnvironmentBackups,
+  getEnvironmentBackups,
   updateUser,
   deleteUser,
   addUserToCustomer,

node-packages/commons/src/lokka-transport-http-retry.js

@@ -0,0 +1,58 @@
+const {
+  Transport: LokkaTransportHttp,
+} = require('@lagoon/lokka-transport-http');
+const fetchUrl = require('node-fetch');
+
+class NetworkError extends Error {}
+class ApiError extends Error {}
+
+// Retries the fetch if operational/network errors occur
+const retryFetch = (endpoint, options, retriesLeft = 5, interval = 1000) =>
+  new Promise((resolve, reject) =>
+    fetchUrl(endpoint, options)
+      .then(response => {
+        if (response.status !== 200 && response.status !== 400) {
+          throw new NetworkError(`Invalid status code: ${response.status}`);
+        }
+
+        return response.json();
+      })
+      .then(({ data, errors }) => {
+        if (errors) {
+          const error = new ApiError(`GraphQL Error: ${errors[0].message}`);
+          error.rawError = errors;
+          error.rawData = data;
+          throw error;
+        }
+
+        resolve(data);
+      })
+      .catch(error => {
+        // Don't retry if limit is reached or the error was not network/operational
+        if (retriesLeft === 1 || error instanceof ApiError) {
+          reject(error);
+          return;
+        }
+
+        setTimeout(() => {
+          retryFetch(endpoint, options, retriesLeft - 1).then(resolve, reject);
+        }, interval);
+      }),
+  );
+
+class Transport extends LokkaTransportHttp {
+  constructor(endpoint, options = {}) {
+    super(endpoint, options);
+  }
+
+  send(query, variables, operationName) {
+    const payload = { query, variables, operationName };
+    const options = this._buildOptions(payload);
+
+    return retryFetch(this.endpoint, options);
+  }
+}
+
+module.exports = {
+  Transport,
+};

services/ui/package.json

@@ -19,7 +19,7 @@
     "apollo-link-ws": "^1.0.10",
     "dotenv-extended": "^2.2.0",
     "express": "^4.16.4",
-    "git-url-parse": "^10.0.1",
+    "git-url-parse": "^11.1.2",
     "graphql": "^0.13.2",
     "graphql-tag": "^2.10.0",
     "isomorphic-unfetch": "^2.1.1",

services/webhooks2tasks/src/handlers/resticbackupSnapshotFinished.js

@@ -5,7 +5,7 @@ const moment = require('moment');
 const { sendToLagoonLogs } = require('@lagoon/commons/src/logs');
 const {
   addBackup,
-  getAllEnvironmentBackups
+  getEnvironmentBackups
 } = require('@lagoon/commons/src/api');
 const R = require('ramda');
 
@@ -50,35 +50,30 @@ async function resticbackupSnapshotFinished(webhook: WebhookRequestData) {
   const { webhooktype, event, uuid, body } = webhook;
 
   try {
-    // Get a list of all environments and their existing backups.
-    const allEnvironmentsResult = await getAllEnvironmentBackups();
-    const allEnvironments = R.prop('allEnvironments', allEnvironmentsResult)
-    const existingBackupIds = R.pipe(
-      R.map(env => env.backups),
-      R.flatten(),
-      R.map(backup => backup.backupId)
-    )(allEnvironments);
-
     // The webhook contains all existing and new snapshots made for all
-    // environments. Filter out snapshots that have already been recorded and
-    // group remaining (new) by hostname.
+    // environments. Group by hostname.
     const incomingSnapshots = R.prop('snapshots', body);
-    const newSnapshots = R.pipe(
-      R.reject(snapshot => R.contains(snapshot.id, existingBackupIds)),
+    const snapshotsByHostname = R.pipe(
       // Remove pod names suffix from hostnames.
       R.map(R.over(R.lensProp('hostname'), R.replace(/(-cli|-mariadb|-nginx|-solr|-node|-elasticsearch|-redis|-[\w]+-prebackuppod)$/, ''))),
       R.groupBy(snapshot => snapshot.hostname),
       R.toPairs()
     )(incomingSnapshots);
 
-    for (const [hostname, snapshots] of newSnapshots) {
-      const environment = R.find(R.propEq('openshiftProjectName', hostname), allEnvironments);
+    for (const [hostname, snapshots] of snapshotsByHostname) {
+      // Get environment and existing backups.
+      const environmentResult = await getEnvironmentBackups(hostname);
+      const environment = R.prop('environmentByOpenshiftProjectName', environmentResult)
 
       if (!environment) {
         continue;
       }
 
-      for (const snapshot of snapshots) {
+      // Filter out snapshots that have already been recorded.
+      const existingBackupIds = R.pluck('backupId', environment.backups);
+      const newSnapshots = R.reject(snapshot => R.contains(snapshot.id, existingBackupIds), snapshots);
+
+      for (const snapshot of newSnapshots) {
         try {
           const newBackupResult = await saveSnapshotAsBackup(
             snapshot,

yarn.lock

@@ -4918,20 +4918,20 @@ getpass@^0.1.1:
   dependencies:
     assert-plus "^1.0.0"
 
-git-up@^2.0.0:
-  version "2.1.0"
-  resolved "https://registry.yarnpkg.com/git-up/-/git-up-2.1.0.tgz#2f14cfe78327e7c4a2b92fcac7bfc674fdfad40c"
-  integrity sha512-MJgwfcSd9qxgDyEYpRU/CDxNpUadrK80JHuEQDG4Urn0m7tpSOgCBrtiSIa9S9KH8Tbuo/TN8SSQmJBvsw1HkA==
+git-up@^4.0.0:
+  version "4.0.1"
+  resolved "https://registry.yarnpkg.com/git-up/-/git-up-4.0.1.tgz#cb2ef086653640e721d2042fe3104857d89007c0"
+  integrity sha512-LFTZZrBlrCrGCG07/dm1aCjjpL1z9L3+5aEeI9SBhAqSc+kiA9Or1bgZhQFNppJX6h/f5McrvJt1mQXTFm6Qrw==
   dependencies:
     is-ssh "^1.3.0"
-    parse-url "^3.0.2"
+    parse-url "^5.0.0"
 
-git-url-parse@^10.0.1:
-  version "10.1.0"
-  resolved "https://registry.yarnpkg.com/git-url-parse/-/git-url-parse-10.1.0.tgz#a27813218f8777e91d15f1c121b83bf14721b67e"
-  integrity sha512-goZOORAtFjU1iG+4zZgWq+N7It09PqS3Xsy43ZwhP5unDD0tTSmXTpqULHodMdJXGejm3COwXIhIRT6Z8DYVZQ==
+git-url-parse@^11.1.2:
+  version "11.1.2"
+  resolved "https://registry.yarnpkg.com/git-url-parse/-/git-url-parse-11.1.2.tgz#aff1a897c36cc93699270587bea3dbcbbb95de67"
+  integrity sha512-gZeLVGY8QVKMIkckncX+iCq2/L8PlwncvDFKiWkBn9EtCfYDbliRTTp6qzyQ1VMdITUfq7293zDzfpjdiGASSQ==
   dependencies:
-    git-up "^2.0.0"
+    git-up "^4.0.0"
 
 glob-base@^0.3.0:
   version "0.3.0"
@@ -5963,7 +5963,7 @@ is-path-inside@^1.0.0:
   dependencies:
     path-is-inside "^1.0.1"
 
-is-plain-obj@^1.0.0, is-plain-obj@^1.1.0:
+is-plain-obj@^1.1.0:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/is-plain-obj/-/is-plain-obj-1.1.0.tgz#71a50c8429dfca773c92a390a4a03b39fcd51d3e"
   integrity sha1-caUMhCnfync8kqOQpKA7OfzVHT4=
@@ -7746,6 +7746,11 @@ node-fetch@^2.1.2, node-fetch@^2.2.0:
   resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.3.0.tgz#1a1d940bbfb916a1d3e0219f037e89e71f8c5fa5"
   integrity sha512-MOd8pV3fxENbryESLgVIeaGKrdl+uaYhCSSVkjeOb/31/njTpcis5aWfdqgNlHIrKOLRbMnfPINPOML2CIFeXA==
 
+node-fetch@^2.6.0:
+  version "2.6.0"
+  resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.0.tgz#e633456386d4aa55863f676a7ab0daa8fdecb0fd"
+  integrity sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==
+
 node-int64@^0.4.0:
   version "0.4.0"
   resolved "https://registry.yarnpkg.com/node-int64/-/node-int64-0.4.0.tgz#87a9065cdb355d3182d8f94ce11188b825c68a3b"
@@ -7891,17 +7896,7 @@ normalize-path@^3.0.0:
   resolved "https://registry.yarnpkg.com/normalize-path/-/normalize-path-3.0.0.tgz#0dcd69ff23a1c9b11fd0978316644a0388216a65"
   integrity sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==
 
-normalize-url@^1.9.1:
-  version "1.9.1"
-  resolved "https://registry.yarnpkg.com/normalize-url/-/normalize-url-1.9.1.tgz#2cc0d66b31ea23036458436e3620d85954c66c3c"
-  integrity sha1-LMDWazHqIwNkWENuNiDYWVTGbDw=
-  dependencies:
-    object-assign "^4.0.1"
-    prepend-http "^1.0.0"
-    query-string "^4.1.0"
-    sort-keys "^1.0.0"
-
-normalize-url@^3.1.0:
+normalize-url@^3.1.0, normalize-url@^3.3.0:
   version "3.3.0"
   resolved "https://registry.yarnpkg.com/normalize-url/-/normalize-url-3.3.0.tgz#b2e1c4dc4f7c6d57743df733a4f5978d18650559"
   integrity sha512-U+JJi7duF1o+u2pynbp2zXDW2/PADgC30f0GsHZtRh+HOcXHnw137TrNlyxxRvWW5fjKd3bcLHPxofWuCjaeZg==
@@ -8341,22 +8336,22 @@ parse-passwd@^1.0.0:
   resolved "https://registry.yarnpkg.com/parse-passwd/-/parse-passwd-1.0.0.tgz#6d5b934a456993b23d37f40a382d6f1666a8e5c6"
   integrity sha1-bVuTSkVpk7I9N/QKOC1vFmao5cY=
 
-parse-path@^3.0.1:
-  version "3.0.4"
-  resolved "https://registry.yarnpkg.com/parse-path/-/parse-path-3.0.4.tgz#a48b7b529da41f34d9d1428602a39b29fc7180e4"
-  integrity sha512-wP70vtwv2DyrM2YoA7ZHVv4zIXa4P7dGgHlj+VwyXNDduLLVJ7NMY1zsFxjUUJ3DAwJLupGb1H5gMDDiNlJaxw==
+parse-path@^4.0.0:
+  version "4.0.1"
+  resolved "https://registry.yarnpkg.com/parse-path/-/parse-path-4.0.1.tgz#0ec769704949778cb3b8eda5e994c32073a1adff"
+  integrity sha512-d7yhga0Oc+PwNXDvQ0Jv1BuWkLVPXcAoQ/WREgd6vNNoKYaW52KI+RdOFjI63wjkmps9yUE8VS4veP+AgpQ/hA==
   dependencies:
     is-ssh "^1.3.0"
     protocols "^1.4.0"
 
-parse-url@^3.0.2:
-  version "3.0.2"
-  resolved "https://registry.yarnpkg.com/parse-url/-/parse-url-3.0.2.tgz#602787a7063a795d72b8673197505e72f60610be"
-  integrity sha1-YCeHpwY6eV1yuGcxl1BecvYGEL4=
+parse-url@^5.0.0:
+  version "5.0.1"
+  resolved "https://registry.yarnpkg.com/parse-url/-/parse-url-5.0.1.tgz#99c4084fc11be14141efa41b3d117a96fcb9527f"
+  integrity sha512-flNUPP27r3vJpROi0/R3/2efgKkyXqnXwyP1KQ2U0SfFRgdizOdWfvrrvJg1LuOoxs7GQhmxJlq23IpQ/BkByg==
   dependencies:
     is-ssh "^1.3.0"
-    normalize-url "^1.9.1"
-    parse-path "^3.0.1"
+    normalize-url "^3.3.0"
+    parse-path "^4.0.0"
     protocols "^1.4.0"
 
 [email protected]:
@@ -8682,7 +8677,7 @@ prelude-ls@~1.1.2:
   resolved "https://registry.yarnpkg.com/prelude-ls/-/prelude-ls-1.1.2.tgz#21932a549f5e52ffd9a827f570e04be62a97da54"
   integrity sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ=
 
-prepend-http@^1.0.0, prepend-http@^1.0.1:
+prepend-http@^1.0.1:
   version "1.0.4"
   resolved "https://registry.yarnpkg.com/prepend-http/-/prepend-http-1.0.4.tgz#d4f4562b0ce3696e41ac52d0e002e57a635dc6dc"
   integrity sha1-1PRWKwzjaW5BrFLQ4ALlemNdxtw=
@@ -8940,14 +8935,6 @@ [email protected], qs@~6.5.1, qs@~6.5.2:
   resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.2.tgz#cb3ae806e8740444584ef154ce8ee98d403f3e36"
   integrity sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==
 
-query-string@^4.1.0:
-  version "4.3.4"
-  resolved "https://registry.yarnpkg.com/query-string/-/query-string-4.3.4.tgz#bbb693b9ca915c232515b228b1a02b609043dbeb"
-  integrity sha1-u7aTucqRXCMlFbIosaArYJBD2+s=
-  dependencies:
-    object-assign "^4.1.0"
-    strict-uri-encode "^1.0.0"
-
 querystring-es3@^0.2.0:
   version "0.2.1"
   resolved "https://registry.yarnpkg.com/querystring-es3/-/querystring-es3-0.2.1.tgz#9ec61f79049875707d69414596fd907a4d711e73"
@@ -9980,13 +9967,6 @@ [email protected]:
   dependencies:
     hoek "4.x.x"
 
-sort-keys@^1.0.0:
-  version "1.1.2"
-  resolved "https://registry.yarnpkg.com/sort-keys/-/sort-keys-1.1.2.tgz#441b6d4d346798f1b4e49e8920adfba0e543f9ad"
-  integrity sha1-RBttTTRnmPG05J6JIK37oOVD+a0=
-  dependencies:
-    is-plain-obj "^1.0.0"
-
 source-list-map@^2.0.0:
   version "2.0.1"
   resolved "https://registry.yarnpkg.com/source-list-map/-/source-list-map-2.0.1.tgz#3993bd873bfc48479cca9ea3a547835c7c154b34"
@@ -10213,11 +10193,6 @@ strftime@~0.10.0:
   resolved "https://registry.yarnpkg.com/strftime/-/strftime-0.10.0.tgz#b3f0fa419295202a5a289f6d6be9f4909a617193"
   integrity sha1-s/D6QZKVICpaKJ9ta+n0kJphcZM=
 
-strict-uri-encode@^1.0.0:
-  version "1.1.0"
-  resolved "https://registry.yarnpkg.com/strict-uri-encode/-/strict-uri-encode-1.1.0.tgz#279b225df1d582b1f54e65addd4352e18faa0713"
-  integrity sha1-J5siXfHVgrH1TmWt3UNS4Y+qBxM=
-
 [email protected]:
   version "1.1.3"
   resolved "https://registry.yarnpkg.com/string-hash/-/string-hash-1.1.3.tgz#e8aafc0ac1855b4666929ed7dd1275df5d6c811b"