Standardize LOGS_REDACTION feature

README.md

@@ -606,12 +606,13 @@ The metrics are collected by the [New Relic Java Agent](https://docs.newrelic.co
 
 To enable the integration you must provide the following variables:
 
-| Environment variable    | Value example                                  | Default                  | Description                                                                                                                            |
-|-------------------------|------------------------------------------------|--------------------------|----------------------------------------------------------------------------------------------------------------------------------------|
-| `NEW_RELIC_LICENSE_KEY` | `api_key`                                      | -                        | License Key or API Key ([docs](https://docs.newrelic.com/docs/apis/intro-apis/new-relic-api-keys/))                                    |
-| `NEW_RELIC_METRICS_URI` | `https://metric-api.eu.newrelic.com/metric/v1` | -                        | Metrics endpoint API ([docs](https://docs.newrelic.com/docs/data-apis/ingest-apis/metric-api/report-metrics-metric-api/#api-endpoint)) |
-| `NEW_RELIC_LOGS_URI`    | `https://log-api.eu.newrelic.com/log/v1`       | -                        | Logs endpoint API ([docs](https://docs.newrelic.com/docs/logs/log-api/introduction-log-api/))                                          |
-| `NEW_RELIC_APP_NAME`    | `MyApp`                                        | application domain name  | Optional. Mendix App name shown on New Relic                                                                                           |
+| Environment variable    | Value example                                  | Default                 | Description                                                                                                                            |
+|-------------------------|------------------------------------------------|-------------------------|----------------------------------------------------------------------------------------------------------------------------------------|
+| `NEW_RELIC_LICENSE_KEY` | `api_key`                                      | -                       | License Key or API Key ([docs](https://docs.newrelic.com/docs/apis/intro-apis/new-relic-api-keys/))                                    |
+| `NEW_RELIC_METRICS_URI` | `https://metric-api.eu.newrelic.com/metric/v1` | -                       | Metrics endpoint API ([docs](https://docs.newrelic.com/docs/data-apis/ingest-apis/metric-api/report-metrics-metric-api/#api-endpoint)) |
+| `NEW_RELIC_LOGS_URI`    | `https://log-api.eu.newrelic.com/log/v1`       | -                       | Logs endpoint API ([docs](https://docs.newrelic.com/docs/logs/log-api/introduction-log-api/))                                          |
+| `NEW_RELIC_APP_NAME`    | `MyApp`                                        | application domain name | Optional. Mendix App name shown on New Relic                                                                                           |
+| `LOGS_REDACTION`        | `true`                                         | `true`                  | Optional. Enables email address redaction from logs                                                                                    |
 
 :warning: For the first usage of the New Relic integration, the Mendix app should be redeployed after setting the variables up.
 
@@ -657,12 +658,13 @@ To enable Splunk integration for a Mendix application, following environment var
 
 :warning: For the first usage of Splunk integration the Mendix app should be **redeployed** after setting the variables up.
 
-| Environment variable | Value example | Default | Description |
-|-|-|-|-|
-| `SPLUNK_HOST` | `test.splunkcloud.com` | - | Host of Splunk Cloud without 'http://' |
-| `SPLUNK_PORT` | `8088` | `8088` | Port of Splunk Cloud |
-| `SPLUNK_TOKEN`¹ | `uuid token` | - | Token from Splunk Cloud dashboard |
-| `SPLUNK_LOGS_REDACTION` | `true` | `true` | If `true` emails in log message are redacted |
+| Environment variable    | Value example          | Default | Description                                                                                |
+|-------------------------|------------------------|---------|--------------------------------------------------------------------------------------------|
+| `SPLUNK_HOST`           | `test.splunkcloud.com` | -       | Host of Splunk Cloud without 'http://'                                                     |
+| `SPLUNK_PORT`           | `8088`                 | `8088`  | Port of Splunk Cloud                                                                       |
+| `SPLUNK_TOKEN`¹         | `uuid token`           | -       | Token from Splunk Cloud dashboard                                                          |
+| `SPLUNK_LOGS_REDACTION` | `true`                 | `true`  | **DEPRECATED** - If `true` emails in log message are redacted - Use LOGS_REDACTION instead |
+| `LOGS_REDACTION`        | `true`                 | `true`  | Enables email address redaction from logs                                                  |
 
 1) To create new token on Splunk Cloud dashboard go to `Settings -> Data Input -> HTTP Event Collector` and push
 button `New Token` in the top-right corner of the page.
@@ -770,7 +772,8 @@ Additionally, the following integration-specific variables are available:
 | ------------------------------------- | ------------- | ------------------------------------------------------------------------------------------------------------------------------------------------- |
 | `DATADOG_DATABASE_DISKSTORAGE_METRIC` | `true`        | Enables a metric denoting the disk storage size available to the database. This metric is set in the `DATABASE_DISKSTORAGE` environment variable. |
 | `DATADOG_DATABASE_RATE_COUNT_METRICS` | `false`       | Enables additional rate / count database metrics currently not compatible with the Datadog PostgreSQL integration                                 |
-| `DATADOG_LOGS_REDACTION`              | `true`        | Enables email address redaction from logs                                                                                                         |
+| `DATADOG_LOGS_REDACTION`              | `true`        | **DEPRECATED** - Enables email address redaction from logs - Use LOGS_REDACTION instead                                                           |
+| `LOGS_REDACTION`                      | `true`        | Enables email address redaction from logs                                                                                                         |
 
 To receive metrics from the runtime, the Mendix Java Agent is added to the runtime as Java agent. This agent can be configured by passing a JSON in the environment variable `METRICS_AGENT_CONFIG` as described in [Datadog for v4 Mendix Cloud](https://docs.mendix.com/developerportal/operate/datadog-metrics).
 

buildpack/telemetry/datadog.py

@@ -80,6 +80,15 @@ def _is_tracing_enabled():
 
 # Toggles logs redaction (email addresses are replaced by a generic string)
 def _is_logs_redaction_enabled():
+    """Check if logs should be redacted."""
+
+    # Use this, if it is set
+    logs_redaction = os.getenv("LOGS_REDACTION")
+    if logs_redaction is not None:
+        return strtobool(logs_redaction)
+
+    # Turned on by default
+    # DEPRECATED - Datadog-specific LOGS_REDACTION variable
     return strtobool(os.environ.get("DATADOG_LOGS_REDACTION", "true"))
 
 

buildpack/telemetry/fluentbit.py

@@ -9,7 +9,7 @@
 
 from buildpack import util
 from buildpack.telemetry import newrelic, splunk
-
+from lib.m2ee.util import strtobool
 
 NAMESPACE = "fluentbit"
 CONF_FILENAME = f"{NAMESPACE}.conf"
@@ -148,6 +148,8 @@ def _set_up_environment(model_version, runtime_version):
     env_vars["FLUENTBIT_APP_RUNTIME_VERSION"] = str(runtime_version)
     env_vars["FLUENTBIT_APP_MODEL_VERSION"] = model_version
 
+    env_vars["LOGS_REDACTION"] = str(_is_logs_redaction_enabled())
+
     fluentbit_env_vars.update(env_vars)
     return fluentbit_env_vars
 
@@ -169,3 +171,19 @@ def _print_logs() -> Tuple:
     if FLUENTBIT_ENV_VARS["FLUENTBIT_LOG_LEVEL"] == "debug":
         return tuple()
     return "-l", "/dev/null"
+
+
+def _is_logs_redaction_enabled() -> bool:
+    """Check if logs should be redacted."""
+
+    # Use this, if it is set
+    logs_redaction = os.getenv("LOGS_REDACTION")
+    if logs_redaction is not None:
+        return bool(strtobool(logs_redaction))
+
+    # DEPRECATED - Splunk-specific LOGS_REDACTION variable
+    if splunk.is_splunk_enabled():
+        return bool(strtobool(os.getenv("SPLUNK_LOGS_REDACTION", "true")))
+
+    # Turned on by default
+    return True

buildpack/telemetry/newrelic.py

@@ -63,7 +63,7 @@ def update_config(m2ee, app_name):
 
 def _get_new_relic_license_key() -> Optional[str]:
     """Get the New Relic's license key."""
-    # Service-binding based integration (on-prem only)
+    # DEPRECATED - Service-binding integration (on-prem only)
     vcap_services = util.get_vcap_services_data()
     if vcap_services and "newrelic" in vcap_services:
         return vcap_services["newrelic"][0]["credentials"]["licenseKey"]

etc/fluentbit/redaction.lua

@@ -1,19 +1,15 @@
 function apply_redaction(tag, timestamp, record)
 
-    local stringtoboolean={ ["true"]=true, ["false"]=false }
+    local stringtoboolean={ ["True"]=true, ["False"]=false }
 
     local patterns = {
         '\'jdbc:postgresql://(.*)\'',
         'S3 storage, bucket location: (.*)',
         'Endpoint set to: s3-(.*)',
     }
 
-    local is_logs_redaction = os.getenv("SPLUNK_LOGS_REDACTION")
-    if is_logs_redaction == nil then
-        is_logs_redaction = true
-    else
-        is_logs_redaction = stringtoboolean[is_logs_redaction]
-    end
+    local is_logs_redaction = os.getenv("LOGS_REDACTION")
+    is_logs_redaction = stringtoboolean[is_logs_redaction]
 
     if is_logs_redaction then
         table.insert(patterns, '[%w+%.%-_]+@[%w+%.%-_]+%.%a%a+') --email