Merge pull request #65 from samply/develop

Using beam library, some CQL replacement changes

.dockerignore

@@ -0,0 +1,2 @@
+*
+!artifacts/

.github/workflows/rust.yml

@@ -2,6 +2,7 @@ name: Build with rust and docker
 
 on:
   push:
+  workflow_dispatch:
   pull_request:
   schedule:
     # Fetch new base image updates every night at 1am
@@ -14,15 +15,15 @@ env:
 jobs:
   pre-check:
     name: Security, License Check
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
 
     steps:
     - uses: actions/checkout@v3
     - uses: EmbarkStudios/cargo-deny-action@v1
 
   build-rust:
     name: Build (Rust)
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
 
     strategy:
       matrix:
@@ -70,18 +71,34 @@ jobs:
         with:
           key: ${{ matrix.arch }}-${{ env.PROFILE }}
           prefix-key: "v1-rust" # Increase to invalidate old caches.
-      - name: Build (${{ matrix.arch }})
+      - name: Build (cross to ${{ matrix.arch }})
+        if: env.is_cross == 'true'
         uses: actions-rs/cargo@v1
         with:
           use-cross: ${{ env.is_cross }}
           command: build
-          args: --target ${{ env.rustarch }} ${{ env.profilestr }}
-      - name: Upload Artifact
+          args: --target ${{ env.rustarch }} ${{ matrix.features && format('--features {0}', matrix.features) }} ${{ env.profilestr }}
+      - name: Build (native)
+        if: env.is_cross == 'false'
+        run: |
+          BINS=$(cargo build --tests --bins --message-format=json --target ${{ env.rustarch }} ${{ matrix.features && format('--features {0}', matrix.features) }} ${{ env.profilestr }} | jq -r 'select(.profile.test == true) | .executable | select(. != null)')
+          mkdir -p testbinaries/
+          for testbin in $BINS; do
+            mv -v $testbin testbinaries/
+          done
+      - name: Upload (bins)
         uses: actions/upload-artifact@v3
         with:
           name: binaries-${{ matrix.arch }}
           path: |
             target/${{ env.rustarch }}/${{ env.PROFILE }}/focus
+      - name: Upload (test, native only)
+        if: matrix.arch == 'amd64'
+        uses: actions/upload-artifact@v3
+        with:
+          name: testbinaries-${{ matrix.arch }}
+          path: |
+            testbinaries/*
 
   test:
     name: Run tests
@@ -92,13 +109,17 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/download-artifact@v3
         with:
-          name: binaries-amd64
-          path: artifacts/binaries-amd64/
-      - run: cargo test
+          name: testbinaries-amd64
+          path: testbinaries/
+      - run: |
+          for testbin in testbinaries/*; do
+            chmod +x $testbin
+            $testbin
+          done
 
   docker-focus:
     needs: [ build-rust, pre-check, test ]
-    if: github.ref_protected == true
+    if: github.ref_protected == true || github.event_name == 'workflow_dispatch'
 
     # This workflow defines how a maven package is built, tested and published.
     # Visit: https://github.com/samply/github-workflows/blob/develop/.github/workflows/docker-ci.yml, for more information
@@ -111,7 +132,7 @@ jobs:
       # Define the build context of your image, typically default '.' will be enough
       # build-context: '.'
       # Define the Dockerfile of your image, typically default './Dockerfile' will be enough
-      build-file: './Dockerfile.ci'
+      build-file: './Dockerfile'
       # NOTE: This doesn't work currently
       # A list of build arguments, passed to the docker build
 #      build-args: |

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "focus"
-version = "0.1.0"
+version = "0.1.1"
 edition = "2021"
 license = "Apache-2.0"
 
@@ -9,20 +9,20 @@ license = "Apache-2.0"
 [dependencies]
 base64 = { version = "0.21.0", default_features = false }
 http = "0.2"
-reqwest = { version = "0.11.14", default_features = false, features = ["json", "default-tls"] }
+reqwest = { version = "0.11", default_features = false, features = ["json", "default-tls"] }
 serde = { version = "1.0.152", features = ["serde_derive"] }
-serde_json = "1.0.91"
+serde_json = "1.0"
 thiserror = "1.0.38"
 tokio = { version = "1.25.0", default_features = false, features = ["signal", "rt-multi-thread", "macros"] }
-uuid = { version = "1.3.0", default_features = false, features = ["serde"]}
+beam-lib = { git = "https://github.com/samply/beam", branch = "develop", features = ["http-util"] }
 laplace_rs = {version = "0.2.0", git = "https://github.com/samply/laplace-rs.git", branch = "main" }
 
 # Logging
 tracing = { version = "0.1.37", default_features = false }
 tracing-subscriber = { version = "0.3.11", default_features = false, features = ["env-filter", "fmt"] }
 
 # Global variables
-static_init = "1.0.2"
+once_cell = "1.18"
 
 # Command Line Interface
 clap = { version = "4.0", default_features = false, features = ["std", "env", "derive"] }

Dockerfile

@@ -1,11 +1,16 @@
-# This assumes binaries are present, see COPY directive.
+# This Dockerfile is infused with magic to speedup the build.
+# In particular, it requires built binaries to be present (see COPY directive).
+#
+# tl;dr: To make this build work, run
+#   ./dev/focusdev build
+# and find your freshly built images tagged with the `localbuild` tag.
 
 FROM alpine AS chmodder
 ARG TARGETARCH
 COPY /artifacts/binaries-$TARGETARCH/focus /app/
 RUN chmod +x /app/*
 
-FROM alpine
+FROM gcr.io/distroless/cc-debian12
 COPY --from=chmodder /app/* /usr/local/bin/
 ENTRYPOINT [ "/usr/local/bin/focus" ]
 

README.md

@@ -16,16 +16,16 @@ To run a standalone Focus, you need at least one running [Samply.Beam.Proxy](htt
 You can compile and run this application via Cargo, however, we encourage the usage of the pre-compiled [docker images](https://hub.docker.com/r/samply/focus):
 
 ```bash
-docker run --rm -e BEAM_BASE_URL=http://localhost:8081 -e BLAZE_BASE_URL=http://localhost:8089/fhir -e PROXY_ID=proxy1.broker -e API_KEY=App1Secret -e BEAM_APP_ID_LONG=app1.broker.example.com samply/focus:latest
+docker run --rm -e BEAM_PROXY_URL=http://localhost:8081 -e BLAZE_URL=http://localhost:8089/fhir -e PROXY_ID=proxy1.broker -e API_KEY=App1Secret -e BEAM_APP_ID_LONG=app1.broker.example.com samply/focus:latest
 ```
 
 ## Configuration
 
 The following environment variables are mandatory for the usage of Focus. If compiling and running Focus yourself, they are provided as command line options as well. See `focus  --help` for details.
 
 ```bash
-BEAM_BASE_URL = "http://localhost:8081" 
-BLAZE_BASE_URL = "http://localhost:8089/fhir"
+BEAM_PROXY_URL = "http://localhost:8081" 
+BLAZE_URL = "http://localhost:8089/fhir"
 PROXY_ID = "proxy1.broker"
 API_KEY = "App1Secret"
 BEAM_APP_ID_LONG = "app1.broker.example.com"
@@ -36,16 +36,16 @@ BEAM_APP_ID_LONG = "app1.broker.example.com"
 ```bash
 RETRY_COUNT = "32" # The maximum number of retries for beam and blaze healthchecks, default value: 32
 OBFUSCATE = "yes" # Should the results be obfuscated - the "master switch", allowed values: "yes", "no", default value: "yes"
-OBFUSCATE-BELOW-10 = "1" # The mode of obfuscating values below 10: 0 - return zero, 1 - return ten, 2 - obfuscate using Laplace distribution and rounding, has no effect if OBFUSCATE = "no", default value: 1
-DELTA-PATIENT = "1." # Sensitivity parameter for obfuscating the counts in the Patient stratifier, has no effect if OBFUSCATE = "no", default value: 1
-DELTA-SPECIMEN = "20." # Sensitivity parameter for obfuscating the counts in the Specimen stratifier, has no effect if OBFUSCATE = "no", default value: 20
-DELTA-DIAGNOSIS = "3." # Sensitivity parameter for obfuscating the counts in the Diagnosis stratifier, has no effect if OBFUSCATE = "no", default value: 3
+OBFUSCATE_BELOW_10_MODE = "1" # The mode of obfuscating values below 10: 0 - return zero, 1 - return ten, 2 - obfuscate using Laplace distribution and rounding, has no effect if OBFUSCATE = "no", default value: 1
+DELTA_PATIENT = "1." # Sensitivity parameter for obfuscating the counts in the Patient stratifier, has no effect if OBFUSCATE = "no", default value: 1
+DELTA_SPECIMEN = "20." # Sensitivity parameter for obfuscating the counts in the Specimen stratifier, has no effect if OBFUSCATE = "no", default value: 20
+DELTA_DIAGNOSIS = "3." # Sensitivity parameter for obfuscating the counts in the Diagnosis stratifier, has no effect if OBFUSCATE = "no", default value: 3
 EPSILON = "0.1" # Privacy budget parameter for obfuscating the counts in the stratifiers, has no effect if OBFUSCATE = "no", default value: 0.1
-ROUNDING-STEP = "10" # The granularity of the rounding of the obfuscated values, has no effect if OBFUSCATE = "no", default value: 10
+ROUNDING_STEP = "10" # The granularity of the rounding of the obfuscated values, has no effect if OBFUSCATE = "no", default value: 10
 QUERIES_TO_CACHE_FILE_PATH = "resources/bbmri" # The path to the file containing BASE64 encoded queries whose results are to be cached, if not set, no results are cached
 ```
 
-Obfuscating zero counts is by default switched off. To enable obfuscating zero counts, set the env. variable `OBFUSCATE-ZERO`. 
+Obfuscating zero counts is by default switched off. To enable obfuscating zero counts, set the env. variable `OBFUSCATE_ZERO`. 
 
 Optionally, you can provide the `TLS_CA_CERTIFICATES_DIR` environment variable to add additional trusted certificates, e.g., if you have a TLS-terminating proxy server in place. The application respects the `HTTP_PROXY`, `HTTPS_PROXY`, `ALL_PROXY`, `NO_PROXY`, and their respective lowercase equivalents.
 

dev/docker-compose.yml

@@ -0,0 +1,106 @@
+version: "3.7"
+services:
+  vault:
+    image: hashicorp/vault
+    ports:
+      - 127.0.0.1:8200:8200
+    environment:
+      VAULT_DEV_ROOT_TOKEN_ID: ${VAULT_TOKEN}
+      VAULT_DEV_LISTEN_ADDRESS: 0.0.0.0:8200
+    volumes:
+      - ./pki:/pki
+    networks:
+      - default
+  broker:
+    depends_on: [vault]
+    image: samply/beam-broker:develop
+    ports:
+      - 8080:8080
+      - 8090:8090
+    environment:
+      BROKER_URL: ${BROKER_URL}
+      PKI_ADDRESS: http://vault:8200
+      no_proxy: vault
+      NO_PROXY: vault
+      PRIVKEY_FILE: /run/secrets/dummy.pem
+      BIND_ADDR: 0.0.0.0:8080
+      MONITORING_API_KEY: ${BROKER_MONITORING_KEY}
+      RUST_LOG: ${RUST_LOG}
+      ALL_PROXY: http://mitmproxy:8080
+    secrets:
+      - pki.secret
+      - dummy.pem
+      - root.crt.pem
+  mitmproxy:
+    image: mitmproxy/mitmproxy
+    stop_signal: SIGKILL
+    command: mitmweb --web-host 0.0.0.0 --web-port 9090
+    ports:
+      - 9090:9090
+  proxy1:
+    depends_on: [broker]
+    image: samply/beam-proxy:develop
+    ports:
+      - 8081:8081
+    environment:
+      BROKER_URL: ${BROKER_URL}
+      PROXY_ID: ${PROXY1_ID}
+      APP_app1_KEY: ${APP_KEY}
+      APP_app2_KEY: ${APP_KEY}
+      PRIVKEY_FILE: /run/secrets/proxy1.pem
+      BIND_ADDR: 0.0.0.0:8081
+      RUST_LOG: ${RUST_LOG}
+      ALL_PROXY: http://mitmproxy:8080
+    secrets:
+      - proxy1.pem
+      - root.crt.pem
+  proxy2:
+    depends_on: [broker]
+    image: samply/beam-proxy:develop
+    ports:
+      - 8082:8082
+    environment:
+      BROKER_URL: ${BROKER_URL}
+      PROXY_ID: ${PROXY2_ID}
+      APP_app1_KEY: ${APP_KEY}
+      APP_app2_KEY: ${APP_KEY}
+      PRIVKEY_FILE: /run/secrets/proxy2.pem
+      BIND_ADDR: 0.0.0.0:8082
+      RUST_LOG: ${RUST_LOG}
+      ALL_PROXY: http://mitmproxy:8080
+    secrets:
+      - proxy2.pem
+      - root.crt.pem
+  focus:
+    depends_on:
+      - proxy1
+      - blaze
+    build:
+      context: ../
+      dockerfile: Dockerfile
+    image: samply/focus:${TAG}
+    environment:
+      API_KEY: ${APP_KEY}
+      BEAM_APP_ID_LONG: app1.${PROXY1_ID}
+      BLAZE_URL: "http://blaze:8080/fhir/"
+      BEAM_PROXY_URL: http://proxy1:8081
+      RETRY_COUNT: 30
+      OBFUSCATE: "no"
+  blaze:
+    image: samply/blaze
+    volumes:
+      - "blaze-data:/app/data"
+secrets:
+  pki.secret:
+    file: ./pki/pki.secret
+  proxy1.pem:
+    file: ./pki/${PROXY1_ID_SHORT}.priv.pem
+  proxy2.pem:
+    file: ./pki/${PROXY2_ID_SHORT}.priv.pem
+  dummy.pem:
+    file: ./pki/dummy.priv.pem
+  root.crt.pem:
+    file: ./pki/root.crt.pem
+
+volumes:
+  blaze-data: