Merge pull request #30 from City-of-Helsinki/UHF-9986

UHF-9986: Logging improvements
City-of-Helsinki · Apr 23, 2024 · fb350bc · fb350bc
2 parents 3c8354e + 9f175de
commit fb350bc
Show file tree

Hide file tree

Showing 14 changed files with 100 additions and 75 deletions.
diff --git a/local/drupal/Dockerfile b/local/drupal/Dockerfile
@@ -4,9 +4,7 @@ FROM druidfi/drupal-web:php-${PHP_VERSION} as base
 
 COPY --from=ghcr.io/city-of-helsinki/drupal-oc-cli:latest /usr/bin/oc /usr/bin/oc
 
-RUN sudo apk add --no-cache jq \
-  # This is required by mysql 8 (caching_sha2_password)
-  mariadb-connector-c
+RUN sudo apk add --no-cache jq
 
 # Autostart drush server and syslog services.
 COPY entrypoints/ /entrypoints
@@ -16,7 +14,9 @@ COPY files/ /
 # Override default fpm pool conf to run nginx and php-fpm as same user.
 COPY php-fpm-pool.conf /etc/php${PHP_SHORT_VERSION}/php-fpm.d/www.conf
 
-RUN grep -qxF 'xdebug.log' /etc/php${PHP_SHORT_VERSION}/conf.d/xdebug.ini || echo 'xdebug.log=/tmp/xdebug.log' | sudo tee -a /etc/php${PHP_SHORT_VERSION}/conf.d/xdebug.ini
+RUN grep -qxF 'xdebug.log' /etc/php${PHP_SHORT_VERSION}/conf.d/xdebug.ini || echo 'xdebug.log=/tmp/xdebug.log' | sudo tee -a /etc/php${PHP_SHORT_VERSION}/conf.d/xdebug.ini && \
+  sudo -u root touch /tmp/xdebug.log && \
+  sudo chmod 666 /tmp/xdebug.log
 
 # Add druid user to nginx group to prevent permission issues with drush and 'asset://'
 # files.
@@ -39,4 +39,4 @@ FROM base as php83
 RUN sudo apk --no-cache --repository=https://dl-cdn.alpinelinux.org/alpine/edge/testing \
     --repository=https://dl-cdn.alpinelinux.org/alpine/edge/main  \
     --repository=https://dl-cdn.alpinelinux.org/alpine/edge/community \
-    --repository=https://dl-cdn.alpinelinux.org/alpine/edge/releases add php82-pecl-imagick php83-pecl-pcov
+    --repository=https://dl-cdn.alpinelinux.org/alpine/edge/releases add php83-pecl-imagick php83-pecl-pcov
diff --git a/local/drupal/entrypoints/15-syslog.sh b/local/drupal/entrypoints/15-syslog.sh
@@ -1,4 +1,11 @@
 #!/bin/sh
-sudo touch /tmp/drupal.log && sudo chmod a+rw /tmp/drupal.log
 
-tail -f /tmp/drupal.log &
+cd /app
+
+# @todo Remove this once all projects use monolog.
+if ! composer show drupal/monolog -q 2>/dev/null; then
+  sudo touch /tmp/drupal.log && sudo chmod a+rw /tmp/drupal.log
+  tail -f /tmp/drupal.log &
+else
+  echo "Found drupal/monolog. Skipping logger entrypoint ..."
+fi
diff --git a/local/drupal/files/etc/nginx/http.d/default.conf b/local/drupal/files/etc/nginx/http.d/default.conf
@@ -128,4 +128,13 @@ server {
     empty_gif;
   }
 
+  location ~ ^/(?:.*)-assets/(.*)$ {
+    proxy_redirect off;
+    proxy_set_header Host $host;
+    proxy_set_header X-Forwarded-Proto https;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_pass http://127.0.0.1:8080/$1$is_args$args;
+  }
+
 }
diff --git a/local/drupal/files/etc/nginx/nginx.conf b/local/drupal/files/etc/nginx/nginx.conf
@@ -126,8 +126,13 @@ http {
   # Specifies the logger-json log format.
   log_format logger-json escape=json '{"source": "nginx", "time": $msec, "resp_body_size": $body_bytes_sent, "host": "$http_host", "address": "$remote_addr", "request_length": $request_length, "method": "$request_method", "uri": "$request_uri", "status": $status,  "user_agent": "$http_user_agent", "resp_time": $request_time, "upstream_addr": "$upstream_addr", "x_forwarderd_for": "$http_x_forwarded_for"}';
 
+  map $request_uri $loggable {
+    /health 0;
+    default 1;
+  }
+
   # Sets the path, format, and configuration for a buffered log write.
-  access_log  /dev/stdout logger-json;
+  access_log  /dev/stdout logger-json if=$loggable;
 
   # Includes virtual hosts configs.
   include /etc/nginx/http.d/*.conf;

diff --git a/local/drupal/php-fpm-pool.conf b/local/drupal/php-fpm-pool.conf
@@ -2,7 +2,7 @@
 ; Log to stderr
 error_log = /dev/stderr
 ; Error log level. Possible values: alert, error, warning, notice, debug. Default value: notice.
-log_level = notice
+log_level = warning
 
 [www]
 listen = [::]:9000
@@ -11,10 +11,10 @@ listen = [::]:9000
 clear_env = no
 
 pm = dynamic
-pm.max_children = 8
+pm.max_children = 60
 pm.start_servers = 2
-pm.min_spare_servers = 1
-pm.max_spare_servers = 3
+pm.min_spare_servers = 2
+pm.max_spare_servers = 20
 pm.max_requests = 500
 
 user=nginx
@@ -24,7 +24,7 @@ group=nginx
 catch_workers_output = yes
 
 ; Remove the 'child 10 said into stderr' prefix in the log and only show the actual message
-; decorate_workers_output = no
+decorate_workers_output = no
 
 php_value[memory_limit] = 512M
 
diff --git a/openshift/drupal/Dockerfile b/openshift/drupal/Dockerfile
@@ -40,12 +40,16 @@ RUN wget https://www.digicert.com/CACerts/BaltimoreCyberTrustRoot.crt.pem -O /tm
   cat /tmp/baltimore.crt.pem >> $AZURE_SQL_SSL_CA_PATH && \
   rm /tmp/*.crt.pem
 
+# Expose nginx
+EXPOSE 8080
+
+# Copy configuration files and scripts
+COPY files/ /
+RUN chmod +x /entrypoints/* && \
+  chmod +x /usr/local/bin/entrypoint
+
 # Create missing nginx folder.
 RUN mkdir -p /run/nginx
-COPY drupal.conf /etc/nginx/http.d/default.conf
-COPY fastcgi.conf /etc/nginx/fastcgi.conf
-COPY nginx.conf /etc/nginx/nginx.conf
-COPY nginx-custom.locations /etc/nginx/conf.d/custom.locations
 
 RUN \
   # COMPOSER_HOME permissions.
@@ -57,18 +61,8 @@ RUN \
   # Correct permissions to nginx folders.
   chmod -R a+rwx /var/lib/nginx && \
   chmod -R a+rwx /var/log/nginx && \
-  chmod -R a+rwx /etc/nginx/conf.d && \
-  # Redirect log output to docker log collector.
-  ln -sf /proc/1/fd/1 /var/log/nginx/access.log && \
-  ln -sf /proc/1/fd/2 /var/log/nginx/error.log
+  chmod -R a+rwx /etc/nginx/conf.d
 
-# Expose nginx
-EXPOSE 8080
-
-# Copy configuration files and scripts
-COPY files/ /
-RUN chmod +x /entrypoints/* && \
-  chmod +x /usr/local/bin/entrypoint
 ENTRYPOINT ["/sbin/tini", "--"]
 
 # Default command: Start up multiple services via entrypoint
@@ -168,36 +162,8 @@ RUN set -eux; \
 
 FROM php${PHP_SHORT_VERSION} as final
 
-ENV NEW_RELIC_SHA256SUM d50b763b5f24d1bfb63e83e346b9c2c4b007e58290f03f190cbee211ce529124
-ENV NEW_RELIC_URL https://download.newrelic.com/php_agent/archive/10.16.0.5/newrelic-php5-10.16.0.5-linux-musl.tar.gz
-RUN set -eux; \
-  curl $NEW_RELIC_URL > newrelic-php.tar.gz && \
-  echo "$NEW_RELIC_SHA256SUM *newrelic-php.tar.gz" | sha256sum -c - && \
-  gzip -dc newrelic-php.tar.gz | tar xf - && \
-  rm newrelic-php.tar.gz && \
-  cd newrelic-php* && \
-  NR_INSTALL_SILENT=true NR_INSTALL_USE_CP_NOT_LN=1 ./newrelic-install install && \
-  cd .. && \
-  rm -rf newrelic-php* && \
-  rm -r /tmp/*
-
-RUN touch /tmp/php_agent.log /tmp/newrelic-daemon.log && \
-  chmod a+rw /tmp/php_agent.log /tmp/newrelic-daemon.log;
-
 ARG PHP_SHORT_VERSION
 
-# Newrelic configuration.
-RUN { \
-    echo 'extension = "newrelic.so"'; \
-    echo '[newrelic]'; \
-    echo 'newrelic.enabled = true'; \
-    echo 'newrelic.logfile = "/tmp/php_agent.log"'; \
-    echo 'newrelic.daemon.logfile = "/tmp/newrelic-daemon.log"'; \
-    echo 'newrelic.license = "${NEW_RELIC_LICENSE_KEY}"'; \
-    echo 'newrelic.appname = "${NEW_RELIC_APPNAME}"'; \
-    echo 'newrelic.daemon.address = "${NEW_RELIC_DAEMON_ADDRESS}"'; \
-} > /etc/php${PHP_SHORT_VERSION}/conf.d/newrelic.ini
-
 # Set recommended PHP.ini settings
 # see https://secure.php.net/manual/en/opcache.installation.php
 # we set enable_file_override as files won't change while container is up.

diff --git a/openshift/drupal/files/entrypoints/15-syslog.sh b/openshift/drupal/files/entrypoints/15-syslog.sh
@@ -1,4 +1,12 @@
 #!/bin/sh
-touch /tmp/drupal.log && chmod a+rw /tmp/drupal.log
 
-tail -f /tmp/drupal.log &
+cd /var/www/html
+
+# @todo Remove this once all projects use monolog.
+if ! composer show drupal/monolog -q 2>/dev/null; then
+  sudo touch /tmp/drupal.log && sudo chmod a+rw /tmp/drupal.log
+  tail -f /tmp/drupal.log &
+else
+  echo "Found drupal/monolog. Skipping logger entrypoint ..."
+fi
+
diff --git a/openshift/drupal/nginx-custom.locations → ...l/files/etc/nginx/conf.d/custom.locations b/openshift/drupal/nginx-custom.locations → ...l/files/etc/nginx/conf.d/custom.locations
diff --git a/openshift/drupal/fastcgi.conf → ...shift/drupal/files/etc/nginx/fastcgi.conf b/openshift/drupal/fastcgi.conf → ...shift/drupal/files/etc/nginx/fastcgi.conf
diff --git a/openshift/drupal/drupal.conf → ...rupal/files/etc/nginx/http.d/default.conf b/openshift/drupal/drupal.conf → ...rupal/files/etc/nginx/http.d/default.conf
@@ -6,6 +6,23 @@ server {
   root /var/www/html/public;
   index index.php;
 
+  # Support for the helfi_proxy module
+  # https://github.com/City-of-Helsinki/drupal-module-helfi-proxy
+  location ~ ^/(?:.*)-assets/(.*)$ {
+    proxy_redirect off;
+    proxy_set_header Host $host;
+    proxy_set_header X-Forwarded-Proto https;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    # Make sure nginx doesn't buffer files to a temporary file. The value must be larger
+    # than the largest asset file.
+    # At the moment, the largest assets are .woff fonts from GIN theme.
+    proxy_buffers 240 340k;
+    proxy_busy_buffers_size 340k;
+    proxy_buffer_size 340k;
+    proxy_pass http://127.0.0.1:8080/$1$is_args$args;
+  }
+
   # Allow locations to be added without overriding the whole conf.
   include conf.d/custom.locations;
 
@@ -119,15 +136,4 @@ server {
     expires 30d;
     empty_gif;
   }
-
-  # Support for the helfi_proxy module
-  # https://github.com/City-of-Helsinki/drupal-module-helfi-proxy
-  location ~ ^/(?:.*)-assets/(.*)$ {
-    proxy_redirect off;
-    proxy_set_header Host $host;
-    proxy_set_header X-Forwarded-Proto https;
-    proxy_set_header X-Real-IP $remote_addr;
-    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-    proxy_pass http://127.0.0.1:8080/$1$is_args$args;
-  }
 }
diff --git a/openshift/drupal/nginx.conf → openshift/drupal/files/etc/nginx/nginx.conf b/openshift/drupal/nginx.conf → openshift/drupal/files/etc/nginx/nginx.conf
@@ -7,7 +7,7 @@ worker_processes auto;
 pcre_jit on;
 
 # Configures default error logger.
-error_log /var/log/nginx/error.log warn;
+error_log /dev/stderr warn;
 
 pid /tmp/nginx.pid;
 
@@ -121,8 +121,16 @@ http {
       '$status $body_bytes_sent "$http_referer" '
       '"$http_user_agent" "$http_x_forwarded_for"';
 
+  # Specifies the logger-json log format.
+  log_format logger-json escape=json '{"source": "nginx", "time": $msec, "resp_body_size": $body_bytes_sent, "host": "$http_host", "address": "$remote_addr", "request_length": $request_length, "method": "$request_method", "uri": "$request_uri", "status": $status,  "user_agent": "$http_user_agent", "resp_time": $request_time, "upstream_addr": "$upstream_addr", "x_forwarderd_for": "$http_x_forwarded_for"}';
+
+  map $request_uri $loggable {
+    /health 0;
+    default 1;
+  }
+
   # Sets the path, format, and configuration for a buffered log write.
-  access_log /var/log/nginx/access.log main;
+  access_log  /dev/stdout logger-json if=$loggable;
 
   # Includes virtual hosts configs.
   include /etc/nginx/http.d/*.conf;

diff --git a/openshift/drupal/php-fpm-pool.conf b/openshift/drupal/php-fpm-pool.conf
@@ -2,7 +2,7 @@
 ; Log to stderr
 error_log = /dev/stderr
 ; Error log level. Possible values: alert, error, warning, notice, debug. Default value: notice.
-log_level = notice
+log_level = warning
 
 [www]
 listen = [::]:9000
@@ -24,7 +24,7 @@ group=nginx
 catch_workers_output = yes
 
 ; Remove the 'child 10 said into stderr' prefix in the log and only show the actual message
-; decorate_workers_output = no
+decorate_workers_output = no
 
 php_value[memory_limit] = 512M
 
diff --git a/openshift/drupal/tests/nginx.yaml b/openshift/drupal/tests/nginx.yaml
@@ -8,3 +8,19 @@ commandTests:
     command: nginx
     args: ['-t']
 
+fileExistenceTests:
+- name: 'Nginx fastcgi.conf'
+  path: '/etc/nginx/fastcgi.conf'
+  shouldExist: true
+
+- name: 'Nginx nginx.conf'
+  path: '/etc/nginx/nginx.conf'
+  shouldExist: true
+
+- name: 'Nginx default.conf'
+  path: '/etc/nginx/http.d/default.conf'
+  shouldExist: true
+
+- name: 'Nginx custom.locations'
+  path: '/etc/nginx/conf.d/custom.locations'
+  shouldExist: true
diff --git a/openshift/drupal/tests/php.yaml b/openshift/drupal/tests/php.yaml
@@ -9,8 +9,8 @@ commandTests:
     args:
       - -c
       - |
-        php -m | grep -e json -e apcu -e curl -e gd -e mbstring -e openssl -e pdo -e redis -e sodium -e "Zend OPcache" -e tokenizer -e xml -e SimpleXML -e ctype -e newrelic -e imagick | sort -uf
-    expectedOutput: ["apcu\nctype\ncurl\ngd\nimagick\njson\nlibxml\nmbstring\nnewrelic\nopenssl\npdo_mysql\nredis\nSimpleXML\nsodium\ntokenizer\nxml\nxmlreader\nxmlwriter\nZend OPcache"]
+        php -m | grep -e json -e apcu -e curl -e gd -e mbstring -e openssl -e pdo -e redis -e sodium -e "Zend OPcache" -e tokenizer -e xml -e SimpleXML -e ctype -e imagick | sort -uf
+    expectedOutput: ["apcu\nctype\ncurl\ngd\nimagick\njson\nlibxml\nmbstring\nopenssl\npdo_mysql\nredis\nSimpleXML\nsodium\ntokenizer\nxml\nxmlreader\nxmlwriter\nZend OPcache"]
   - name: 'Git is installed'
     command: which
     args: ['git']