nginx: add test matrix, release builds

.github/workflows/nginx.yml

@@ -3,70 +3,85 @@ name: nginx instrumentation CI
 on:
   push:
     branches: "*"
+    paths:
+      - 'instrumentation/nginx/**'
+      - '.github/workflows/nginx.yml'
   pull_request:
     branches: [ main ]
 
 jobs:
-  test-nginx-agent:
-    name: Test nginx agent
+  nginx-build-test:
+    name: nginx
     runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        os: [ubuntu-20.10, ubuntu-20.04, ubuntu-18.04]
+        nginx-rel: [mainline, stable]
     steps:
       - name: checkout otel nginx
         uses: actions/checkout@v2
       - name: setup
         run: |
-          sudo ./instrumentation/nginx/ci/setup_test_environment.sh
-      - name: build docker images
-        run: |
-          cd instrumentation/nginx
-          docker build -t otel-nginx-test/nginx -f test/Dockerfile .
-          docker build -t otel-nginx-test/express-backend -f test/backend/simple_express/Dockerfile test/backend/simple_express
-      - name: run tests
+          sudo ./instrumentation/nginx/ci/setup_environment.sh
+      - name: generate dockerfile
         run: |
           cd instrumentation/nginx/test/instrumentation
           mix local.hex --force --if-missing
           mix local.rebar --force --if-missing
           mix deps.get
-          mix test
-  build-nginx-agent:
-    name: Build nginx agent
-    runs-on: ubuntu-20.04
-    timeout-minutes: 30
-    steps:
-      - name: checkout otel nginx
-        uses: actions/checkout@v2
-      - name: setup
-        run: |
-          sudo ./instrumentation/nginx/ci/setup_ci_environment.sh
-      - name: checkout grpc
-        uses: actions/checkout@v2
-        with: 
-          repository: "grpc/grpc"
-          ref: "v1.35.0"
-          fetch-depth: 1
-          path: "instrumentation/nginx/grpc"
-          submodules: "recursive"
-      - name: build grpc
+          mix dockerfiles .. ${{ matrix.os }}:${{ matrix.nginx-rel }}
+      - name: setup buildx
+        id: buildx
+        uses: docker/setup-buildx-action@master
+        with:
+          install: true
+      - name: cache docker layers
+        uses: actions/cache@v2
+        with:
+          path: /tmp/buildx-cache/
+          key: nginx-${{ matrix.os }}-${{ matrix.nginx-rel }}-${{ github.sha }}
+          restore-keys: |
+            nginx-${{ matrix.os }}-${{ matrix.nginx-rel }}
+      - name: build express backend docker
         run: |
           cd instrumentation/nginx
-          ./ci/grpc.sh
-      - name: checkout opentelemetry-cpp
-        uses: actions/checkout@v2
-        with: 
-          repository: "open-telemetry/opentelemetry-cpp"
-          fetch-depth: 1
-          path: "instrumentation/nginx/opentelemetry-cpp"
-          submodules: "recursive"
-      - name: build opentelemetry-cpp
+          docker buildx build -t otel-nginx-test/express-backend \
+            -f test/backend/simple_express/Dockerfile \
+            --cache-from type=local,src=/tmp/buildx-cache/express \
+            --cache-to type=local,dest=/tmp/buildx-cache/express-new \
+            --load \
+            test/backend/simple_express
+      - name: build nginx docker
         run: |
           cd instrumentation/nginx
-          ./ci/otel-cpp.sh 
-      - name: build nginx module
+          docker buildx build -t otel-nginx-test/nginx \
+            --build-arg image=$(echo ${{ matrix.os }} | sed s/-/:/) \
+            -f test/Dockerfile.${{ matrix.os }}.${{ matrix.nginx-rel }} \
+            --cache-from type=local,src=/tmp/buildx-cache/nginx \
+            --cache-to type=local,dest=/tmp/buildx-cache/nginx-new \
+            --load \
+            .
+      - name: update cache
+        run: |
+          rm -rf /tmp/buildx-cache/express
+          rm -rf /tmp/buildx-cache/nginx
+          mv /tmp/buildx-cache/express-new /tmp/buildx-cache/express
+          mv /tmp/buildx-cache/nginx-new /tmp/buildx-cache/nginx
+      - name: run tests
+        run: |
+          cd instrumentation/nginx/test/instrumentation
+          mix test
+      - name: copy artifacts
+        id: artifacts
         run: |
           cd instrumentation/nginx
-          ./ci/build_module.sh
-      - name: Upload module artifact
+          mkdir -p /tmp/otel_ngx/
+          docker buildx build -f test/Dockerfile.${{ matrix.os }}.${{ matrix.nginx-rel}} \
+          --target export \
+          --cache-from type=local,src=/tmp/.buildx-cache \
+          --output type=local,dest=/tmp/otel_ngx .
+      - name: upload artifacts
         uses: actions/upload-artifact@v2
         with:
-          name: otel-nginx
-          path: ./instrumentation/nginx/build/otel_ngx_module.so
+          name: otel_ngx_module-${{ matrix.os }}-${{ matrix.nginx-rel }}.so
+          path: /tmp/otel_ngx/otel_ngx_module.so

instrumentation/nginx/README.md

@@ -8,15 +8,34 @@ Supported propagation types:
 
 ## Requirements
 
-* OS: Linux
-* Nginx
-  * latest stable - [1.18.0](http://nginx.org/en/download.html)
+* OS: Linux. Test suite currently runs on Ubuntu 18.04, 20.04, 20.10.
+* [Nginx](http://nginx.org/en/download.html)
+  * both stable (`1.18.0`) and mainline (`1.19.8`)
 * Nginx modules
   * ngx_http_upstream_module (proxy_pass)
   * ngx_http_fastcgi_module (fastcgi_pass)
 
 Additional platforms and/or versions coming soon.
 
+## Dependencies (for building)
+
+1. [gRPC](https://github.com/grpc/grpc) - currently the only supported exporter is OTLP. This requirement will be lifted
+   once more exporters become available.
+2. [opentelemetry-cpp](https://github.com/open-telemetry/opentelemetry-cpp) - opentelemetry-cpp needs to be built with
+   position independent code and OTLP support, e.g.:
+```
+cmake -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DWITH_OTLP=ON ..
+```
+
+## Building
+
+```
+mkdir build
+cd build
+cmake ..
+make
+```
+
 ## Usage
 
 Modify nginx.conf, or see the [example](test/conf/nginx.conf)
@@ -133,25 +152,6 @@ List of exported attributes and their corresponding nginx variables if applicabl
 - `http.scheme` - `$scheme`
 - `http.server_name` - From the `server_name` directive
 
-## Dependencies
-
-1. [gRPC](https://github.com/grpc/grpc) - currently the only supported exporter is OTLP. This requirement will be lifted
-   once more exporters become available.
-2. [opentelemetry-cpp](https://github.com/open-telemetry/opentelemetry-cpp) - opentelemetry-cpp needs to be built with
-   position independent code and OTLP support, e.g.:
-```
-cmake -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DWITH_OTLP=ON ..
-```
-
-## Building
-
-```
-mkdir build
-cd build
-cmake ..
-make
-```
-
 ## Testing
 
 Dependencies:
@@ -160,8 +160,16 @@ Dependencies:
 * [Docker Compose](https://docs.docker.com/compose/install/)
 
 ```
-docker build -t otel-nginx-test/nginx -f test/Dockerfile .
+cd test/instrumentation
+mix .. dockerfiles ubuntu-20.04:mainline
+cd ../..
+docker build -t otel-nginx-test/nginx -f test/Dockerfile.ubuntu-20.04.mainline .
 docker build -t otel-nginx-test/express-backend -f test/backend/simple_express/Dockerfile test/backend/simple_express
 cd test/instrumentation
 mix test
 ```
+
+## Troubleshooting
+
+### `otel_ngx_module.so is not binary compatible`
+- Make sure your nginx is compiled with `--with-compat` (`nginx -V`). On Ubuntu 18.04 the default nginx (`1.14.0`) from apt does not have compatibility enabled. nginx provides [repositories](https://docs.nginx.com/nginx/admin-guide/installing-nginx/installing-nginx-open-source/#prebuilt_ubuntu) to install more up to date versions.

instrumentation/nginx/nginx.cmake

@@ -1,12 +1,33 @@
 include(ExternalProject)
 
-set(NGINX_VER "1.18.0" CACHE STRING "Nginx version to compile against")
+if (NOT NGINX_VERSION)
+  find_program(NGINX_BIN nginx REQUIRED)
+  execute_process(COMMAND ${NGINX_BIN} -v
+    ERROR_VARIABLE NGINX_VERSION_STRING
+  )
+
+  string(REGEX MATCH "[0-9]+\\.\[0-9]+\\.[0-9]+" NGINX_VER ${NGINX_VERSION_STRING})
+else()
+  set(NGINX_VER "1.18.0")
+endif()
+
+set(NGINX_VERSION ${NGINX_VER} CACHE STRING "Nginx version to compile against")
+message(STATUS "nginx: using version ${NGINX_VERSION}")
+
+option(NGINX_WITH_COMPAT "Enable --with-compat for the nginx module" ON)
+
+if (NGINX_WITH_COMPAT)
+  message(STATUS "nginx: --with-compat enabled")
+  set(NGINX_CONFIGURE_ARGS "--with-compat")
+else()
+  message(STATUS "nginx: --with-compat disabled")
+endif()
 
 ExternalProject_Add(project_nginx
-  URL "http://nginx.org/download/nginx-${NGINX_VER}.tar.gz"
+  URL "http://nginx.org/download/nginx-${NGINX_VERSION}.tar.gz"
   PREFIX "nginx"
   BUILD_IN_SOURCE 1
-  CONFIGURE_COMMAND ./configure --with-compat
+  CONFIGURE_COMMAND ./configure ${NGINX_CONFIGURE_ARGS}
   BUILD_COMMAND ""
   INSTALL_COMMAND ""
 )

instrumentation/nginx/test/conf/fastcgi_params

@@ -0,0 +1,17 @@
+fastcgi_param  GATEWAY_INTERFACE  CGI/1.1;
+fastcgi_param  SERVER_SOFTWARE    nginx;
+fastcgi_param  QUERY_STRING       $query_string;
+fastcgi_param  REQUEST_METHOD     $request_method;
+fastcgi_param  CONTENT_TYPE       $content_type;
+fastcgi_param  CONTENT_LENGTH     $content_length;
+fastcgi_param  SCRIPT_FILENAME    $document_root$fastcgi_script_name;
+fastcgi_param  SCRIPT_NAME        $fastcgi_script_name;
+fastcgi_param  REQUEST_URI        $request_uri;
+fastcgi_param  DOCUMENT_URI       $document_uri;
+fastcgi_param  DOCUMENT_ROOT      $document_root;
+fastcgi_param  SERVER_PROTOCOL    $server_protocol;
+fastcgi_param  REMOTE_ADDR        $remote_addr;
+fastcgi_param  REMOTE_PORT        $remote_port;
+fastcgi_param  SERVER_ADDR        $server_addr;
+fastcgi_param  SERVER_PORT        $server_port;
+fastcgi_param  SERVER_NAME        $server_name;

instrumentation/nginx/test/conf/nginx.conf

@@ -1,4 +1,4 @@
-load_module modules/otel_ngx_module.so;
+load_module /usr/share/nginx/modules/otel_ngx_module.so;
 
 events {}
 
@@ -49,11 +49,11 @@ http {
     }
 
     location ~ \.php$ {
+      include /etc/nginx/fastcgi_params;
       root /var/www/html/php;
       opentelemetry_operation_name php_fpm_backend;
       opentelemetry_propagate;
       fastcgi_pass php-backend:9000;
-      include fastcgi.conf;
     }
   } 
 }

instrumentation/nginx/test/docker-compose.yml

@@ -9,6 +9,7 @@ services:
     image: otel-nginx-test/nginx:latest
     volumes:
       - ./conf/nginx.conf:/etc/nginx/nginx.conf
+      - ./conf/fastcgi_params:/etc/nginx/fastcgi_params
       - ./conf/otel-nginx.toml:/conf/otel-nginx.toml
       - ./backend/files:/var/www/html/files
     ports: