From 77f8507c88c701ba2b7f2b9723ef33025fc09ae8 Mon Sep 17 00:00:00 2001 From: Michael Schmid Date: Tue, 23 Jul 2019 17:21:22 -0400 Subject: [PATCH 1/4] add 1 second delays for responses --- tests/files/nginx/first/Dockerfile | 1 + tests/files/nginx/first/docker-compose.yml | 4 ++-- tests/files/nginx/first/static-files.conf | 13 +++++++++++++ tests/files/node6_subfolder/subfolder/index.js | 4 ++++ tests/files/node8/index.js | 3 +++ 5 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 tests/files/nginx/first/static-files.conf diff --git a/tests/files/nginx/first/Dockerfile b/tests/files/nginx/first/Dockerfile index f0bff5c4ee..e5c17ed791 100644 --- a/tests/files/nginx/first/Dockerfile +++ b/tests/files/nginx/first/Dockerfile @@ -1,6 +1,7 @@ ARG IMAGE_REPO FROM ${IMAGE_REPO:-amazeeio}/nginx +COPY static-files.conf /etc/nginx/conf.d/app.conf COPY redirects-map.conf /etc/nginx/redirects-map.conf RUN fix-permissions /etc/nginx/redirects-map.conf diff --git a/tests/files/nginx/first/docker-compose.yml b/tests/files/nginx/first/docker-compose.yml index 841c67fef7..789de94ab9 100644 --- a/tests/files/nginx/first/docker-compose.yml +++ b/tests/files/nginx/first/docker-compose.yml @@ -12,8 +12,8 @@ services: lagoon.deployment.strategy: Recreate volumes: - ./app:/app:delegated - expose: - - "8080" + ports: + - "8080:8080" environment: - AMAZEEIO_URL=nginx.docker.amazee.io nginx-basic-auth: diff --git a/tests/files/nginx/first/static-files.conf b/tests/files/nginx/first/static-files.conf new file mode 100644 index 0000000000..3a93f2f294 --- /dev/null +++ b/tests/files/nginx/first/static-files.conf @@ -0,0 +1,13 @@ +server { + + listen 8080 default_server; + + include /etc/nginx/helpers/*.conf; + + location / { + echo_sleep 1.0; # 1 sec delay + index index.html index.htm; + try_files $uri $uri/ =404; + } + +} \ No newline at end of file diff --git a/tests/files/node6_subfolder/subfolder/index.js b/tests/files/node6_subfolder/subfolder/index.js index 1979962b69..1c89ab0535 100644 --- a/tests/files/node6_subfolder/subfolder/index.js +++ b/tests/files/node6_subfolder/subfolder/index.js @@ -1,6 +1,10 @@ const express = require('express') const app = express() +// Adds a 1 sec delay for all requests +app.use((req, res, next) => setTimeout(next, 1000)) + + app.get('/', function (req, res) { let result = [] Object.keys(process.env).map(key => { diff --git a/tests/files/node8/index.js b/tests/files/node8/index.js index db2e482814..eaf17e00c7 100644 --- a/tests/files/node8/index.js +++ b/tests/files/node8/index.js @@ -1,6 +1,9 @@ const express = require('express') const app = express() +// Adds a 1 sec delay for all requests +app.use((req, res, next) => setTimeout(next, 1000)) + app.get('/', function (req, res) { let result = [] Object.keys(process.env).map(key => { From e5aaa3604a61d3baaead0b871be0ea1711cd08d7 Mon Sep 17 00:00:00 2001 From: Michael Schmid Date: Tue, 23 Jul 2019 17:22:26 -0400 Subject: [PATCH 2/4] Gracefull shutdown for nodejs/expressjs --- tests/files/node6_subfolder/subfolder/index.js | 12 +++++++++++- tests/files/node8/Dockerfile | 2 +- tests/files/node8/index.js | 12 +++++++++++- 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/tests/files/node6_subfolder/subfolder/index.js b/tests/files/node6_subfolder/subfolder/index.js index 1c89ab0535..8a361b4936 100644 --- a/tests/files/node6_subfolder/subfolder/index.js +++ b/tests/files/node6_subfolder/subfolder/index.js @@ -15,6 +15,16 @@ app.get('/', function (req, res) { res.send(result.join("
")) }) -app.listen(3000, function () { +const server = app.listen(3000, function () { console.log('Example app listening on port 3000!') }) + +const startGracefulShutdown = () => { + console.log('Starting shutdown of express...'); + server.close(function () { + console.log('Express shut down.'); + }); +} + +process.on('SIGTERM', startGracefulShutdown); +process.on('SIGINT', startGracefulShutdown); diff --git a/tests/files/node8/Dockerfile b/tests/files/node8/Dockerfile index 198681d50a..5d531aef1a 100644 --- a/tests/files/node8/Dockerfile +++ b/tests/files/node8/Dockerfile @@ -33,4 +33,4 @@ ENV LAGOON_PR_TITLE_BUILDTIME ${LAGOON_PR_TITLE} EXPOSE 3000 -CMD ["yarn", "run", "start"] +CMD ["node", "index.js"] diff --git a/tests/files/node8/index.js b/tests/files/node8/index.js index eaf17e00c7..abe11e090c 100644 --- a/tests/files/node8/index.js +++ b/tests/files/node8/index.js @@ -17,6 +17,16 @@ app.get('/', function (req, res) { res.send(result.join("
")) }) -app.listen(3000, function () { +const server = app.listen(3000, function () { console.log('Example app listening on port 3000!') }) + +const startGracefulShutdown = () => { + console.log('Starting shutdown of express...'); + server.close(function () { + console.log('Express shut down.'); + }); +} + +process.on('SIGTERM', startGracefulShutdown); +process.on('SIGINT', startGracefulShutdown); From 512b09c21b2ad17bc4a024294a511a285166b0a3 Mon Sep 17 00:00:00 2001 From: Michael Schmid Date: Tue, 23 Jul 2019 17:22:33 -0400 Subject: [PATCH 3/4] easier testing --- tests/files/node8/docker-compose.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/files/node8/docker-compose.yml b/tests/files/node8/docker-compose.yml index 4d76b7ab01..44d26bacba 100644 --- a/tests/files/node8/docker-compose.yml +++ b/tests/files/node8/docker-compose.yml @@ -9,10 +9,8 @@ services: dockerfile: Dockerfile labels: lagoon.type: node - volumes: - - ./app:/app:delegated - expose: - - "8080" + ports: + - "3000:3000" environment: - AMAZEEIO_URL=node.docker.amazee.io networks: From d03acdd6e552db11d4b400b404984551297196e2 Mon Sep 17 00:00:00 2001 From: Michael Schmid Date: Mon, 29 Jul 2019 17:55:02 -0400 Subject: [PATCH 4/4] document how to gracefull shutdown within node.js containers --- .../using_lagoon/nodejs/gracefull_shutdown.md | 107 ++++++++++++++++++ mkdocs.yml | 2 + 2 files changed, 109 insertions(+) create mode 100644 docs/using_lagoon/nodejs/gracefull_shutdown.md diff --git a/docs/using_lagoon/nodejs/gracefull_shutdown.md b/docs/using_lagoon/nodejs/gracefull_shutdown.md new file mode 100644 index 0000000000..67c03cca5d --- /dev/null +++ b/docs/using_lagoon/nodejs/gracefull_shutdown.md @@ -0,0 +1,107 @@ +# Gracefull Shutdown with Node.js + +Node.js has integrated web server capabilities plus with [Express](https://expressjs.com/) these can be extended even more. + +Unfortunately Node.js does not handle the shutdown of itself very nicely out of the box and this causes many issues with containerized systems. The biggest one being that when a Node.js container is told to shut down, it will immediatelly kill all active connections and does not allow them to gracefully stop. + +This part explains how you can teach node.js how to behave like a real webserver: Finishing active requests and gradefully shut down. + +As an example we use a super simple Node.js server with Express: + +``` +const express = require('express'); +const app = express(); + +// Adds a 5 sec delay for all requests +app.use((req, res, next) => setTimeout(next, 5000)); + +app.get('/', function (req, res) { + res.send("Hello World"); +}) + +const server = app.listen(3000, function () { + console.log('Example app listening on port 3000!'); +}) +``` + +This will just show "Hello World" in when the webserver is visited at localhost:3000. Note the 5 second delay in the response in order to simulate a request that takes some computing time. + +## Part A: Allow requests to be finished. + +If we run the above example and stop the Node.js process while the request is handled (within the 5 seconds), we will see that the Node.js server immediatelly kills the connection and our Browser will show an error. + +We can easily explain our Node.js that it should wait for all the requests to be finished before actually stopping itself, we just add the following code: + + +``` +const startGracefulShutdown = () => { + console.log('Starting shutdown of express...'); + server.close(function () { + console.log('Express shut down.'); + }); +} + +process.on('SIGTERM', startGracefulShutdown); +process.on('SIGINT', startGracefulShutdown); +``` + +This basically calls `server.close()`, which will instruct the http server of Node.js to: +1. Not accept any more requests +2. Finish all running requests + +It will do this on `SIGINT` (when you press `CTRL + C`) or on `SIGTERM` (the standard signal for a process to terminate). + +With this small addition our Node.js will wait until all requests are finished and then stop itself. + +Remark: If we would not run Node.js in a containerized environment we would probably like some additional code that actually kills the Node.js after a couple of seconds as it is technically possible that some requests are either taking very long or are never stopped. As this is though running in a containerized system and Docker and Kubernetes will run a `SIGKILL` (which cannot be handled by the process itself) after a couple of seconds (usually 30) if the container is not stopped this is not a concern for us. + +## Part B: Yarn and NPM children spawning issues + +If we would just implement Part A, we would have a nice experience out of the box. In the real world many Node.js Systems are built with Yarn or NPM which provide not only package management systems to Node.js but also script management. + +With these script functionalities we simplify the start of our application. We can see many `package.json` that look like: + +``` +{ + "name": "node", + "version": "1.0.0", + "main": "index.js", + "license": "MIT", + "dependencies": { + "express": "^4.15.3" + }, + "scripts": { + "start": "node index.js" + } +} +``` + +and with the defined `scripts` section we can run our application just with: + +``` +yarn start +``` + +or + +``` +npm start +``` + +This is nice and makes the life of developers easier. So we also end up using the same within Dockerfiles: + +``` +CMD ["yarn", "start"] +``` + +Unfortunately there is a big problem with this: + +If yarn or npm get a `SIGINT` or `SIGTERM` signal they correctly forward the signal to spawned child process (in this case `node index.js`) but it do not wait for the child processes to stop. Instead yarn/npm immediatelly stop themselves, this signals to Kubernetes/Docker that the container is finished and Kubernetes/Docker will kill all children processes immediatelly. There are issues open for [Yarn](https://github.com/yarnpkg/yarn/issues/4667) and [NPM](https://github.com/npm/npm/issues/4603) but unfortunately they are not solved yet. + +The solution for the problem is to not use Yarn or NPM to start your application and instead use `node` directly: + +``` +CMD ["node", "index.js"] +``` + +This allows Node.js to properly terminate and Kubernetes/Docker will wait for Node to be finished. \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml index a8dea5b024..8c1c9e21ec 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -31,6 +31,8 @@ pages: - Solr: using_lagoon/drupal/services/solr.md - Varnish: using_lagoon/drupal/services/varnish.md - Redis: using_lagoon/drupal/services/redis.md + - Node.js: + - Gracefull Shutdown: using_lagoon/nodejs/gracefull_shutdown.md - Migrations: - From amazee.io: using_lagoon/migrations/amazeeio.md - GoLive: using_lagoon/golive.md