From bf9126c026f46858170e11c9c2c32eb2f266e1d0 Mon Sep 17 00:00:00 2001 From: Joey den Broeder <34917186+jeohist@users.noreply.github.com> Date: Thu, 26 Oct 2023 12:11:52 +0200 Subject: [PATCH 1/3] Bump GitHub actions to latest versions (#694) --- .github/workflows/build.yml | 6 +++--- .github/workflows/test.yml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 558adcac..82e4493f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -23,7 +23,7 @@ jobs: version_tag: ${{ steps.autogenerate-version.outputs.version_tag }} current_version_tag: ${{ steps.get-current-version.outputs.current_version_tag}} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 - name: Get current version @@ -33,7 +33,7 @@ jobs: echo "::set-output name=current_version_tag::${CURRENT_VERSION_TAG}" - name: Auto-generate future version id: autogenerate-version - uses: paulhatch/semantic-version@v5.0.3 + uses: paulhatch/semantic-version@v5.3.0 with: tag_prefix: "v" major_pattern: "(major)" @@ -44,7 +44,7 @@ jobs: needs: get-version runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v4 with: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 46a47d70..f6ea8ed3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -23,7 +23,7 @@ jobs: steps: - name: Check if workflow should be skipped id: skip-check - uses: fkirc/skip-duplicate-actions@v3 + uses: fkirc/skip-duplicate-actions@v5 with: github_token: ${{ github.token }} paths_ignore: '["**.md", "dev/**", "LICENSE"]' From 410ba70d1888cecaa6d44785f9792c7024bafd45 Mon Sep 17 00:00:00 2001 From: Matheus Svolenski Date: Fri, 13 Oct 2023 15:21:10 +0200 Subject: [PATCH 2/3] New Relic Integration --- README.md | 46 +++++++++++++ buildpack/stage.py | 2 +- buildpack/telemetry/fluentbit.py | 104 ++++++++++++++++++----------- buildpack/telemetry/metrics.py | 3 +- buildpack/telemetry/newrelic.py | 92 +++++++++++++++++++++---- buildpack/telemetry/splunk.py | 24 ++----- buildpack/telemetry/telegraf.py | 18 ++++- dependencies.yml | 2 +- etc/fluentbit/fluentbit.conf | 12 +--- etc/fluentbit/metadata.lua | 8 +-- etc/fluentbit/output_newrelic.conf | 6 ++ etc/fluentbit/output_splunk.conf | 10 +++ etc/telegraf/telegraf.toml.j2 | 20 +++++- requirements.txt | 2 +- tests/integration/test_newrelic.py | 59 ++++++++++++++++ 15 files changed, 316 insertions(+), 92 deletions(-) create mode 100644 etc/fluentbit/output_newrelic.conf create mode 100644 etc/fluentbit/output_splunk.conf create mode 100644 tests/integration/test_newrelic.py diff --git a/README.md b/README.md index 1bae52bb..3f6c601c 100644 --- a/README.md +++ b/README.md @@ -598,8 +598,54 @@ The buildpack includes a variety of telemetry agents that can be configured to c ### New Relic +#### Set up New Relic integration + +[Fluent Bit](https://docs.fluentbit.io/manual/) is used to collect Mendix Runtime logs to [New Relic](https://newrelic.com/). + +The metrics are collected by the [New Relic Java Agent](https://docs.newrelic.com/docs/apm/agents/java-agent/features/jvms-page-java-view-app-server-metrics-jmx/) and an integration with the [Telegraf agent](https://docs.influxdata.com/telegraf/). + +To enable the integration you must provide the following variables: + +| Environment variable | Value example | Default | Description | +|-------------------------|------------------------------------------------|--------------------------|----------------------------------------------------------------------------------------------------------------------------------------| +| `NEW_RELIC_LICENSE_KEY` | `api_key` | - | License Key or API Key ([docs](https://docs.newrelic.com/docs/apis/intro-apis/new-relic-api-keys/)) | +| `NEW_RELIC_METRICS_URI` | `https://metric-api.eu.newrelic.com/metric/v1` | - | Metrics endpoint API ([docs](https://docs.newrelic.com/docs/data-apis/ingest-apis/metric-api/report-metrics-metric-api/#api-endpoint)) | +| `NEW_RELIC_LOGS_URI` | `https://log-api.eu.newrelic.com/log/v1` | - | Logs endpoint API ([docs](https://docs.newrelic.com/docs/logs/log-api/introduction-log-api/)) | +| `NEW_RELIC_APP_NAME` | `MyApp` | application domain name | Optional. Mendix App name shown on New Relic | + +:warning: For the first usage of the New Relic integration, the Mendix app should be redeployed after setting the variables up. + +#### Tags/Metadata in metrics and logs + +In addition to the runtime application logs, the following JSON-formatted metadata is automatically sent to New Relic, both for +the metrics collected by the agent and the custom ones, pushed by Telegraf: + +* `environment_id` - unique identifier of the environment; +* `instance_index` - number of the application instance; +* `hostname` - name of the application host; +* `application_name` - default application name, retrieved from domain name; +* `model_version` - model version of the Mendix runtime; +* `runtime_version` - version of the Mendix runtime. + +:information_source: `model_version` and `runtime_version` are only available to the custom metrics. + +#### Custom tags + +You can also set up custom tags in the following format `key:value`. +Below, are listed some suggested tags that you might want to use: + +* `app:{app_name}` – this enables you to identify all logs sent from your app (for example, **app:customermanagement**) +* `env:{environment_name}` – this enables you to identify logs sent from a particular environment so you can separate out production logs from test logs (for example, **env:accp**) + +#### Service-binding integration (on-prem only) - DEPRECATED + To enable New Relic, simply bind a New Relic service to this app and settings will be picked up automatically. Afterwards you have to restage your application to enable the New Relic agent. +This integration does not support logs or custom metrics. + +:warning: The default `NEW_RELIC_APP_NAME` for this integration used to be the environment ID of the application. Now the value is the domain name set to the application. +If you want to keep using the environment id, you will have to set this variable yourself to that value. + ### Splunk #### Set up Splunk integration diff --git a/buildpack/stage.py b/buildpack/stage.py index b46ab6ec..e29b0d5f 100755 --- a/buildpack/stage.py +++ b/buildpack/stage.py @@ -200,8 +200,8 @@ def cleanup_dependency_cache(cached_dir, dependency_list): appdynamics.stage(BUILDPACK_DIR, DOT_LOCAL_LOCATION, CACHE_DIR) dynatrace.stage(BUILDPACK_DIR, DOT_LOCAL_LOCATION, CACHE_DIR) splunk.stage() - fluentbit.stage(BUILDPACK_DIR, DOT_LOCAL_LOCATION, CACHE_DIR) newrelic.stage(BUILDPACK_DIR, DOT_LOCAL_LOCATION, CACHE_DIR) + fluentbit.stage(BUILDPACK_DIR, DOT_LOCAL_LOCATION, CACHE_DIR) mx_java_agent.stage(BUILDPACK_DIR, DOT_LOCAL_LOCATION, CACHE_DIR, runtime_version) telegraf.stage(BUILDPACK_DIR, DOT_LOCAL_LOCATION, CACHE_DIR, runtime_version) datadog.stage(BUILDPACK_DIR, DOT_LOCAL_LOCATION, CACHE_DIR) diff --git a/buildpack/telemetry/fluentbit.py b/buildpack/telemetry/fluentbit.py index c6a445d4..2f1d3731 100644 --- a/buildpack/telemetry/fluentbit.py +++ b/buildpack/telemetry/fluentbit.py @@ -3,11 +3,12 @@ import subprocess import shutil import socket +from typing import List, Tuple import backoff from buildpack import util -from buildpack.telemetry import splunk +from buildpack.telemetry import newrelic, splunk NAMESPACE = "fluentbit" @@ -15,6 +16,9 @@ FILTER_FILENAMES = ("redaction.lua", "metadata.lua") FLUENTBIT_ENV_VARS = { "FLUENTBIT_LOGS_PORT": os.getenv("FLUENTBIT_LOGS_PORT", default="5170"), + "FLUENTBIT_LOG_LEVEL": os.getenv( + "FLUENTBIT_LOG_LEVEL", default="info" + ).lower(), } @@ -23,8 +27,20 @@ def _set_default_env(m2ee): util.upsert_custom_environment_variable(m2ee, var_name, value) -def stage(buildpack_dir, destination_path, cache_path): +def _get_output_conf_filenames() -> List[str]: + """ + Determine the output configs to use. Only enabled integrations + will have the output file in the container. + """ + output_conf_files: List[str] = [] + if splunk.is_splunk_enabled(): + output_conf_files.append("output_splunk.conf") + if newrelic.is_enabled(): + output_conf_files.append("output_newrelic.conf") + return output_conf_files + +def stage(buildpack_dir, destination_path, cache_path): if not is_fluentbit_enabled(): return @@ -36,20 +52,19 @@ def stage(buildpack_dir, destination_path, cache_path): cache_dir=cache_path, ) - for filename in (CONF_FILENAME, *FILTER_FILENAMES): + output_conf_files = _get_output_conf_filenames() + + for filename in ( + CONF_FILENAME, *FILTER_FILENAMES, *output_conf_files + ): shutil.copy( os.path.join(buildpack_dir, "etc", NAMESPACE, filename), - os.path.join( - destination_path, - NAMESPACE, - ), + os.path.join(destination_path, NAMESPACE), ) - logging.info("Fluent Bit has been installed successfully.") def update_config(m2ee): - if not is_fluentbit_enabled(): return @@ -68,24 +83,13 @@ def update_config(m2ee): def run(model_version, runtime_version): - if not is_fluentbit_enabled(): return - fluentbit_dir = os.path.join( - os.path.abspath(".local"), - NAMESPACE, - ) - - fluentbit_bin_path = os.path.join( - fluentbit_dir, - "fluent-bit", - ) - - fluentbit_config_path = os.path.join( - fluentbit_dir, - CONF_FILENAME, - ) + fluentbit_dir = os.path.join(os.path.abspath(".local"), NAMESPACE) + fluentbit_bin_path = os.path.join(fluentbit_dir, "fluent-bit") + fluentbit_config_path = os.path.join(fluentbit_dir, CONF_FILENAME) + print_logs = _print_logs() if not os.path.exists(fluentbit_bin_path): logging.warning( @@ -93,55 +97,75 @@ def run(model_version, runtime_version): "Please redeploy your application to complete " "Fluent Bit installation." ) - splunk.print_failed_message() + splunk.integration_complete(success=False) + newrelic.integration_complete(success=False) return agent_environment = _set_up_environment(model_version, runtime_version) logging.info("Starting Fluent Bit...") - subprocess.Popen( - (fluentbit_bin_path, "-c", fluentbit_config_path), env=agent_environment + (fluentbit_bin_path, "-c", fluentbit_config_path, *print_logs), + env=agent_environment, ) # The runtime does not handle a non-open logs endpoint socket # gracefully, so wait until it's up - @backoff.on_predicate(backoff.expo, lambda x: x > 0, max_time=10) + @backoff.on_predicate(backoff.expo, lambda x: x > 0, max_time=120) def _await_logging_endpoint(): return socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect_ex( ("localhost", int(FLUENTBIT_ENV_VARS["FLUENTBIT_LOGS_PORT"])) ) logging.info("Awaiting Fluent Bit log subscriber...") - if _await_logging_endpoint() == 0: + success = True + if _await_logging_endpoint() != 0: + success = False + + _integration_complete(success) + splunk.integration_complete(success) + newrelic.integration_complete(success) + + +def _integration_complete(success: bool) -> None: + """Call when the setup is done.""" + if success: logging.info("Fluent Bit log subscriber is ready.") - splunk.print_ready_message() else: logging.error( - "Fluent Bit log subscriber was not initialized correctly." + "Fluent Bit log subscriber was not initialized correctly. " "Application logs will not be shipped to Fluent Bit." ) - splunk.print_failed_message() def _set_up_environment(model_version, runtime_version): + fluentbit_env_vars = FLUENTBIT_ENV_VARS + env_vars = dict(os.environ.copy()) - env_vars["SPLUNK_APP_HOSTNAME"] = util.get_hostname() - env_vars["SPLUNK_APP_NAME"] = util.get_app_from_domain() - env_vars["SPLUNK_APP_RUNTIME_VERSION"] = str(runtime_version) - env_vars["SPLUNK_APP_MODEL_VERSION"] = model_version + env_vars["FLUENTBIT_APP_HOSTNAME"] = util.get_hostname() + env_vars["FLUENTBIT_APP_NAME"] = util.get_app_from_domain() + env_vars["FLUENTBIT_APP_RUNTIME_VERSION"] = str(runtime_version) + env_vars["FLUENTBIT_APP_MODEL_VERSION"] = model_version - return env_vars + fluentbit_env_vars.update(env_vars) + return fluentbit_env_vars def is_fluentbit_enabled(): """ The function checks if some modules which requires Fluent Bit is configured. - """ - return any( - [splunk.is_splunk_enabled()] + [splunk.is_splunk_enabled(), newrelic.is_enabled()] ) # Add other modules, where Fluent Bit is used + + +def _print_logs() -> Tuple: + """Discard logs unless debug is active.""" + # FluentBit currently does not support log rotation, therefore + # logs don't go to a file. If debug on, send to stdout + if FLUENTBIT_ENV_VARS["FLUENTBIT_LOG_LEVEL"] == "debug": + return tuple() + return "-l", "/dev/null" diff --git a/buildpack/telemetry/metrics.py b/buildpack/telemetry/metrics.py index 9828a1a9..d8bc8268 100644 --- a/buildpack/telemetry/metrics.py +++ b/buildpack/telemetry/metrics.py @@ -18,7 +18,7 @@ from lib.m2ee.version import MXVersion from lib.m2ee.util import strtobool -from . import datadog, appdynamics, dynatrace +from . import appdynamics, datadog, dynatrace, newrelic METRICS_REGISTRIES_KEY = "Metrics.Registries" @@ -136,6 +136,7 @@ def configure_metrics_registry(m2ee): or get_appmetrics_target() or appdynamics.machine_agent_enabled() or dynatrace.is_telegraf_enabled() + or newrelic.is_enabled() ): allow_list, deny_list = get_apm_filters() paidapps_registries.append(get_statsd_registry(allow_list, deny_list)) diff --git a/buildpack/telemetry/newrelic.py b/buildpack/telemetry/newrelic.py index c7a95d05..597eca61 100644 --- a/buildpack/telemetry/newrelic.py +++ b/buildpack/telemetry/newrelic.py @@ -1,14 +1,34 @@ import logging import os +from typing import Dict, Optional from buildpack import util NAMESPACE = "newrelic" ROOT_DIR = ".local" +REQUIRED_NEW_RELIC_ENV_VARS = [ + "NEW_RELIC_LICENSE_KEY", "NEW_RELIC_LOGS_URI", "NEW_RELIC_METRICS_URI" +] +NEW_RELIC_ENV_VARS = { + "NEW_RELIC_APP_NAME": os.getenv( + "NEW_RELIC_APP_NAME", util.get_app_from_domain() + ), + "NEW_RELIC_LOG": os.path.join( + os.path.abspath(os.path.join(ROOT_DIR, NAMESPACE)), + "newrelic", + "agent.log", + ), +} + + +def _set_default_env(m2ee): + for var_name, value in NEW_RELIC_ENV_VARS.items(): + util.upsert_custom_environment_variable(m2ee, var_name, value) + def stage(buildpack_dir, install_path, cache_path): - if get_new_relic_license_key(): + if _get_new_relic_license_key(): util.resolve_dependency( f"{NAMESPACE}.agent", _get_destination_dir(install_path), @@ -22,29 +42,75 @@ def _get_destination_dir(dot_local=ROOT_DIR): def update_config(m2ee, app_name): - if get_new_relic_license_key() is None: + if _get_new_relic_license_key() is None: logging.debug("Skipping New Relic setup, no license key found in environment") return - logging.info("Adding new relic") util.upsert_custom_environment_variable( - m2ee, "NEW_RELIC_LICENSE_KEY", get_new_relic_license_key() - ) - util.upsert_custom_environment_variable(m2ee, "NEW_RELIC_APP_NAME", app_name) - util.upsert_custom_environment_variable( - m2ee, - "NEW_RELIC_LOG", - os.path.join(_get_destination_dir(), "newrelic", "agent.log"), + m2ee, "NEW_RELIC_LICENSE_KEY", _get_new_relic_license_key() ) + _set_default_env(m2ee) + util.upsert_javaopts( m2ee, - f"-javaagent:{os.path.join(_get_destination_dir(), 'newrelic', 'newrelic.jar')}", # noqa: line-too-long + [ + f"-javaagent:{os.path.join(_get_destination_dir(), 'newrelic', 'newrelic.jar')}", # noqa: line-too-long + f"-Dnewrelic.config.labels=\"{_get_labels(app_name)}\"", + ] ) -def get_new_relic_license_key(): +def _get_new_relic_license_key() -> Optional[str]: + """Get the New Relic's license key.""" + # Service-binding based integration (on-prem only) vcap_services = util.get_vcap_services_data() if vcap_services and "newrelic" in vcap_services: return vcap_services["newrelic"][0]["credentials"]["licenseKey"] - return None + + return os.getenv("NEW_RELIC_LICENSE_KEY", None) + + +def is_enabled() -> bool: + """ + The function checks if all environment variables required + for New Relic connection are set up. The service-binding + based integration (on-prem only) does not care about this. + """ + return all(map(os.getenv, REQUIRED_NEW_RELIC_ENV_VARS)) + + +def get_metrics_config() -> Dict: + """Configs to be used by telegraf.""" + return { + "api_key": os.getenv("NEW_RELIC_LICENSE_KEY", default=""), + "metrics_base_url": os.getenv("NEW_RELIC_METRICS_URI", default=""), + } + + +def _get_labels(app_name) -> str: + """Labels (tags) to be used by New Relic agent.""" + tags = get_metrics_tags(app_name) + string_tags = ";".join([f"{k}:{v}" for k, v in tags.items()]) + return string_tags + + +def get_metrics_tags(app_name) -> Dict: + """Tags to be used by telegraf.""" + return { + "application_name": util.get_app_from_domain(), + "instance_index": int(os.getenv("CF_INSTANCE_INDEX", "0")), + "environment_id": app_name, + "hostname": util.get_hostname() + } + + +def integration_complete(success: bool) -> None: + """Call when the setup is done.""" + if not is_enabled(): + return + + if success: + logging.info("New Relic has been configured successfully.") + else: + logging.error("Failed to configure New Relic.") diff --git a/buildpack/telemetry/splunk.py b/buildpack/telemetry/splunk.py index b6774fc8..9961c454 100644 --- a/buildpack/telemetry/splunk.py +++ b/buildpack/telemetry/splunk.py @@ -31,30 +31,18 @@ def update_config(m2ee): _set_default_env(m2ee) -def print_ready_message(): +def integration_complete(success: bool) -> None: """ This function can be called from external module. - For example: fluentbit.py calls this function when Fluent Bit is ready. - - """ - - if not is_splunk_enabled(): - return - - logging.info("Splunk has been configured successfully.") - - -def print_failed_message(): - """ - This function can be called from external module. - For example: fluentbit.py calls this function when Fluent Bit is failed. - + For example: fluentbit.py calls this function when Fluent Bit is done. """ - if not is_splunk_enabled(): return - logging.error("Failed to configure Splunk.") + if success: + logging.info("Splunk has been configured successfully.") + else: + logging.error("Failed to configure Splunk.") def is_splunk_enabled(): diff --git a/buildpack/telemetry/telegraf.py b/buildpack/telemetry/telegraf.py index 30860372..0d1fd564 100644 --- a/buildpack/telemetry/telegraf.py +++ b/buildpack/telemetry/telegraf.py @@ -18,7 +18,7 @@ from lib.m2ee.util import strtobool from jinja2 import Template -from . import datadog, metrics, mx_java_agent, appdynamics, dynatrace, splunk +from . import appdynamics, datadog, dynatrace, metrics, mx_java_agent, newrelic, splunk NAMESPACE = "telegraf" DEPENDENCY = f"{NAMESPACE}.agent" @@ -89,6 +89,7 @@ def include_db_metrics(): or datadog.is_enabled() or appdynamics.machine_agent_enabled() or dynatrace.is_telegraf_enabled() + or newrelic.is_enabled() ): # For customers who have Datadog or AppDynamics or APPMETRICS_TARGET enabled, # we always include the database metrics. They can opt out @@ -109,6 +110,7 @@ def is_enabled(runtime_version): or appdynamics.machine_agent_enabled() or dynatrace.is_telegraf_enabled() or metrics.micrometer_metrics_enabled(runtime_version) + or newrelic.is_enabled() ) @@ -231,6 +233,7 @@ def _get_integration_usages(): "dynatrace": dynatrace.is_telegraf_enabled, "appdynamics": appdynamics.appdynamics_used, "splunk": splunk.is_splunk_enabled, + "newrelic": newrelic.is_enabled, } for integration, is_enabled in checker_methods.items(): @@ -255,10 +258,21 @@ def update_config(m2ee, app_name): template_path = os.path.join(_get_config_file_dir(version), TEMPLATE_FILENAME) tags = util.get_tags() + if datadog.is_enabled() and "service" not in tags: # app and / or service tag not set tags["service"] = datadog.get_service_tag() + # Add application tags to the custom metrics sent to New Relic + if newrelic.is_enabled(): + newrelic_tags = newrelic.get_metrics_tags(app_name) + newrelic_tags["runtime_version"] = runtime_version + newrelic_tags["model_version"] = runtime.get_model_version() + + # Make sure the user defined values persist, if the tags overlap + newrelic_tags.update(tags) + tags = newrelic_tags + dynatrace_token, dynatrace_ingest_url = dynatrace.get_ingestion_info() with open(template_path, "r") as file_: @@ -284,6 +298,8 @@ def update_config(m2ee, app_name): appdynamics_output_script_path=APPDYNAMICS_OUTPUT_SCRIPT_PATH, dynatrace_enabled=dynatrace.is_telegraf_enabled(), dynatrace_config=_get_dynatrace_config(app_name), + newrelic_enabled=newrelic.is_enabled(), + newrelic_config=newrelic.get_metrics_config(), telegraf_debug_enabled=os.getenv("TELEGRAF_DEBUG_ENABLED", "false"), telegraf_fileout_enabled=strtobool( os.getenv("TELEGRAF_FILEOUT_ENABLED", "false") diff --git a/dependencies.yml b/dependencies.yml index 52dc5d8a..299343fc 100644 --- a/dependencies.yml +++ b/dependencies.yml @@ -83,7 +83,7 @@ dependencies: newrelic: agent: artifact: newrelic/newrelic-java-{{ version }}.zip - version: 6.5.4 + version: 8.6.0 nginx: artifact: nginx_{{ version }}_linux_x64_{{ fs }}_{{ commit }}.tgz commit: 909b06a9 diff --git a/etc/fluentbit/fluentbit.conf b/etc/fluentbit/fluentbit.conf index 467e838b..1d574064 100644 --- a/etc/fluentbit/fluentbit.conf +++ b/etc/fluentbit/fluentbit.conf @@ -3,6 +3,7 @@ Listen localhost Port ${FLUENTBIT_LOGS_PORT} Format json + Log_Level ${FLUENTBIT_LOG_LEVEL} [FILTER] Name lua @@ -16,12 +17,5 @@ script metadata.lua call add_metadata -[OUTPUT] - # SPLUNK cloud platform - Name splunk - Match * - Host ${SPLUNK_HOST} - Port ${SPLUNK_PORT} - Splunk_Token ${SPLUNK_TOKEN} - TLS On - TLS.Verify Off +# Only imports outputs from enabled integrations +@INCLUDE output_*.conf diff --git a/etc/fluentbit/metadata.lua b/etc/fluentbit/metadata.lua index e0496e58..cbbfed89 100644 --- a/etc/fluentbit/metadata.lua +++ b/etc/fluentbit/metadata.lua @@ -2,10 +2,10 @@ function add_metadata(tag, timestamp, record) record["instance_index"] = os.getenv("CF_INSTANCE_INDEX") or "" record["environment_id"] = os.getenv("ENVIRONMENT") or "" - record["hostname"] = os.getenv("SPLUNK_APP_HOSTNAME") or "" - record["application_name"] = os.getenv("SPLUNK_APP_NAME") or "" - record["runtime_version"] = os.getenv("SPLUNK_APP_RUNTIME_VERSION") or "" - record["model_version"] = os.getenv("SPLUNK_APP_MODEL_VERSION") or "" + record["hostname"] = os.getenv("FLUENTBIT_APP_HOSTNAME") or "" + record["application_name"] = os.getenv("FLUENTBIT_APP_NAME") or "" + record["runtime_version"] = os.getenv("FLUENTBIT_APP_RUNTIME_VERSION") or "" + record["model_version"] = os.getenv("FLUENTBIT_APP_MODEL_VERSION") or "" local raw_tags = os.getenv("TAGS") if raw_tags then diff --git a/etc/fluentbit/output_newrelic.conf b/etc/fluentbit/output_newrelic.conf new file mode 100644 index 00000000..c49dc94b --- /dev/null +++ b/etc/fluentbit/output_newrelic.conf @@ -0,0 +1,6 @@ +[OUTPUT] + name nrlogs + match * + base_uri ${NEW_RELIC_LOGS_URI} + api_key ${NEW_RELIC_LICENSE_KEY} + Log_Level ${FLUENTBIT_LOG_LEVEL} diff --git a/etc/fluentbit/output_splunk.conf b/etc/fluentbit/output_splunk.conf new file mode 100644 index 00000000..248ef1da --- /dev/null +++ b/etc/fluentbit/output_splunk.conf @@ -0,0 +1,10 @@ +[OUTPUT] + # SPLUNK cloud platform + Name splunk + Match * + Host ${SPLUNK_HOST} + Port ${SPLUNK_PORT} + Splunk_Token ${SPLUNK_TOKEN} + TLS On + TLS.Verify Off + Log_Level ${FLUENTBIT_LOG_LEVEL} diff --git a/etc/telegraf/telegraf.toml.j2 b/etc/telegraf/telegraf.toml.j2 index 5b4f04b7..194b0b60 100644 --- a/etc/telegraf/telegraf.toml.j2 +++ b/etc/telegraf/telegraf.toml.j2 @@ -70,7 +70,7 @@ {% endif %} {% if db_config %} -{% if not (datadog_api_key or appdynamics_enabled or dynatrace_enabled) %} +{% if not (datadog_api_key or appdynamics_enabled or dynatrace_enabled or newrelic_enabled) %} # PostgreSQL input (standard) [[inputs.postgresql]] address = "postgres://{{ db_config['DatabaseUserName'] }}:{{ db_config['DatabasePassword'] }}@{{ db_config['DatabaseHost'] }}/{{ db_config['DatabaseName'] }}" @@ -355,6 +355,20 @@ {% endfor %} {% endif %} +{% if newrelic_enabled %} +[[outputs.newrelic]] + metric_url = "{{ newrelic_config['metrics_base_url'] }}" + insights_key = "{{ newrelic_config['api_key'] }}" + + # tagexclude drops any non-relevant tags + tagexclude = ["host"] + + # Ignore any micrometer_metrics + [outputs.newrelic.tagdrop] + micrometer_metrics = ["true"] + internal_metrics = ["true"] +{% endif %} + {% if micrometer_metrics %} #################################################################################### # App metrics via micrometer # @@ -394,7 +408,7 @@ data_format = "json" json_timestamp_units = "1ns" - # tagexlude drops any non-relevant tags + # tagexclude drops any non-relevant tags tagexclude = ["host"] # Drop `mx_runtime_user_login` metrics @@ -476,7 +490,7 @@ ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_OUTPUT.md data_format = "influx" - # tagexlude drops any non-relevant tags + # tagexclude drops any non-relevant tags tagexclude = ["host"] # Drop `mx_runtime_user_login` metrics diff --git a/requirements.txt b/requirements.txt index fd624ec3..73a9487c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # pip-compile --resolver=backtracking requirements.in diff --git a/tests/integration/test_newrelic.py b/tests/integration/test_newrelic.py new file mode 100644 index 00000000..9a1a097a --- /dev/null +++ b/tests/integration/test_newrelic.py @@ -0,0 +1,59 @@ +from tests.integration import basetest + + +class TestCaseDeployWithNewRelic(basetest.BaseTest): + def _deploy_app(self, mda_file, newrelic=True): + super().setUp() + + env_vars = {} + if newrelic: + env_vars["NEW_RELIC_LICENSE_KEY"] = "dummy_token" + env_vars["NEW_RELIC_METRICS_URI"] = "https://metric-api.eu.newrelic.com/metric/v1" + env_vars["NEW_RELIC_LOGS_URI"] = "https://log-api.eu.newrelic.com/log/v1" + + self.stage_container(mda_file, env_vars=env_vars) + self.start_container() + + def _test_newrelic_running(self, mda_file): + self._deploy_app(mda_file) + self.assert_app_running() + + # Check if FluentBit is running + output = self.run_on_container("ps -ef | grep fluentbit") + assert output is not None + assert str(output).find("fluent-bit") >= 0 + + # Check if Telegraf is running + self.assert_running("telegraf") + + # Check if New Relic is running + output = self.run_on_container("ps -ef | grep newrelic") + assert str(output).find("newrelic.jar") >= 0 + + def _test_newrelic_not_running(self, mda_file): + self._deploy_app(mda_file, newrelic=False) + self.assert_app_running() + + # Check if FluentBit is not running + output = self.run_on_container("ps -ef | grep fluentbit") + assert str(output).find("fluent-bit") == -1 + + # Check if New Relic is not running + output = self.run_on_container("ps -ef | grep newrelic") + assert str(output).find("newrelic.jar") == -1 + + def _test_newrelic_is_configured(self): + self.assert_string_in_recent_logs( + "New Relic has been configured successfully." + ) + + def _test_newrelic_is_not_configured(self): + self.assert_string_in_recent_logs("Skipping New Relic setup") + + def test_newrelic_mx9(self): + self._test_newrelic_running("BuildpackTestApp-mx9-18.mda") + self._test_newrelic_is_configured() + + def test_newrelic_not_configured(self): + self._test_newrelic_not_running("BuildpackTestApp-mx9-18.mda") + self._test_newrelic_is_not_configured() From b826b65d241d2f69c8f5e0581789854be98d5a0f Mon Sep 17 00:00:00 2001 From: Matheus Svolenski Date: Fri, 27 Oct 2023 11:56:47 +0200 Subject: [PATCH 3/3] Standardize LOGS_REDACTION feature --- README.md | 29 ++++++++++++++++------------- buildpack/telemetry/datadog.py | 9 +++++++++ buildpack/telemetry/fluentbit.py | 20 +++++++++++++++++++- buildpack/telemetry/newrelic.py | 2 +- etc/fluentbit/redaction.lua | 10 +++------- 5 files changed, 48 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 3f6c601c..0643abb8 100644 --- a/README.md +++ b/README.md @@ -606,12 +606,13 @@ The metrics are collected by the [New Relic Java Agent](https://docs.newrelic.co To enable the integration you must provide the following variables: -| Environment variable | Value example | Default | Description | -|-------------------------|------------------------------------------------|--------------------------|----------------------------------------------------------------------------------------------------------------------------------------| -| `NEW_RELIC_LICENSE_KEY` | `api_key` | - | License Key or API Key ([docs](https://docs.newrelic.com/docs/apis/intro-apis/new-relic-api-keys/)) | -| `NEW_RELIC_METRICS_URI` | `https://metric-api.eu.newrelic.com/metric/v1` | - | Metrics endpoint API ([docs](https://docs.newrelic.com/docs/data-apis/ingest-apis/metric-api/report-metrics-metric-api/#api-endpoint)) | -| `NEW_RELIC_LOGS_URI` | `https://log-api.eu.newrelic.com/log/v1` | - | Logs endpoint API ([docs](https://docs.newrelic.com/docs/logs/log-api/introduction-log-api/)) | -| `NEW_RELIC_APP_NAME` | `MyApp` | application domain name | Optional. Mendix App name shown on New Relic | +| Environment variable | Value example | Default | Description | +|-------------------------|------------------------------------------------|-------------------------|----------------------------------------------------------------------------------------------------------------------------------------| +| `NEW_RELIC_LICENSE_KEY` | `api_key` | - | License Key or API Key ([docs](https://docs.newrelic.com/docs/apis/intro-apis/new-relic-api-keys/)) | +| `NEW_RELIC_METRICS_URI` | `https://metric-api.eu.newrelic.com/metric/v1` | - | Metrics endpoint API ([docs](https://docs.newrelic.com/docs/data-apis/ingest-apis/metric-api/report-metrics-metric-api/#api-endpoint)) | +| `NEW_RELIC_LOGS_URI` | `https://log-api.eu.newrelic.com/log/v1` | - | Logs endpoint API ([docs](https://docs.newrelic.com/docs/logs/log-api/introduction-log-api/)) | +| `NEW_RELIC_APP_NAME` | `MyApp` | application domain name | Optional. Mendix App name shown on New Relic | +| `LOGS_REDACTION` | `true` | `true` | Optional. Enables email address redaction from logs | :warning: For the first usage of the New Relic integration, the Mendix app should be redeployed after setting the variables up. @@ -657,12 +658,13 @@ To enable Splunk integration for a Mendix application, following environment var :warning: For the first usage of Splunk integration the Mendix app should be **redeployed** after setting the variables up. -| Environment variable | Value example | Default | Description | -|-|-|-|-| -| `SPLUNK_HOST` | `test.splunkcloud.com` | - | Host of Splunk Cloud without 'http://' | -| `SPLUNK_PORT` | `8088` | `8088` | Port of Splunk Cloud | -| `SPLUNK_TOKEN`¹ | `uuid token` | - | Token from Splunk Cloud dashboard | -| `SPLUNK_LOGS_REDACTION` | `true` | `true` | If `true` emails in log message are redacted | +| Environment variable | Value example | Default | Description | +|-------------------------|------------------------|---------|--------------------------------------------------------------------------------------------| +| `SPLUNK_HOST` | `test.splunkcloud.com` | - | Host of Splunk Cloud without 'http://' | +| `SPLUNK_PORT` | `8088` | `8088` | Port of Splunk Cloud | +| `SPLUNK_TOKEN`¹ | `uuid token` | - | Token from Splunk Cloud dashboard | +| `SPLUNK_LOGS_REDACTION` | `true` | `true` | **DEPRECATED** - If `true` emails in log message are redacted - Use LOGS_REDACTION instead | +| `LOGS_REDACTION` | `true` | `true` | Enables email address redaction from logs | 1) To create new token on Splunk Cloud dashboard go to `Settings -> Data Input -> HTTP Event Collector` and push button `New Token` in the top-right corner of the page. @@ -770,7 +772,8 @@ Additionally, the following integration-specific variables are available: | ------------------------------------- | ------------- | ------------------------------------------------------------------------------------------------------------------------------------------------- | | `DATADOG_DATABASE_DISKSTORAGE_METRIC` | `true` | Enables a metric denoting the disk storage size available to the database. This metric is set in the `DATABASE_DISKSTORAGE` environment variable. | | `DATADOG_DATABASE_RATE_COUNT_METRICS` | `false` | Enables additional rate / count database metrics currently not compatible with the Datadog PostgreSQL integration | -| `DATADOG_LOGS_REDACTION` | `true` | Enables email address redaction from logs | +| `DATADOG_LOGS_REDACTION` | `true` | **DEPRECATED** - Enables email address redaction from logs - Use LOGS_REDACTION instead | +| `LOGS_REDACTION` | `true` | Enables email address redaction from logs | To receive metrics from the runtime, the Mendix Java Agent is added to the runtime as Java agent. This agent can be configured by passing a JSON in the environment variable `METRICS_AGENT_CONFIG` as described in [Datadog for v4 Mendix Cloud](https://docs.mendix.com/developerportal/operate/datadog-metrics). diff --git a/buildpack/telemetry/datadog.py b/buildpack/telemetry/datadog.py index 8276058f..a7b3639a 100644 --- a/buildpack/telemetry/datadog.py +++ b/buildpack/telemetry/datadog.py @@ -80,6 +80,15 @@ def _is_tracing_enabled(): # Toggles logs redaction (email addresses are replaced by a generic string) def _is_logs_redaction_enabled(): + """Check if logs should be redacted.""" + + # Use this, if it is set + logs_redaction = os.getenv("LOGS_REDACTION") + if logs_redaction is not None: + return strtobool(logs_redaction) + + # Turned on by default + # DEPRECATED - Datadog-specific LOGS_REDACTION variable return strtobool(os.environ.get("DATADOG_LOGS_REDACTION", "true")) diff --git a/buildpack/telemetry/fluentbit.py b/buildpack/telemetry/fluentbit.py index 2f1d3731..0c36f4b8 100644 --- a/buildpack/telemetry/fluentbit.py +++ b/buildpack/telemetry/fluentbit.py @@ -9,7 +9,7 @@ from buildpack import util from buildpack.telemetry import newrelic, splunk - +from lib.m2ee.util import strtobool NAMESPACE = "fluentbit" CONF_FILENAME = f"{NAMESPACE}.conf" @@ -148,6 +148,8 @@ def _set_up_environment(model_version, runtime_version): env_vars["FLUENTBIT_APP_RUNTIME_VERSION"] = str(runtime_version) env_vars["FLUENTBIT_APP_MODEL_VERSION"] = model_version + env_vars["LOGS_REDACTION"] = str(_is_logs_redaction_enabled()) + fluentbit_env_vars.update(env_vars) return fluentbit_env_vars @@ -169,3 +171,19 @@ def _print_logs() -> Tuple: if FLUENTBIT_ENV_VARS["FLUENTBIT_LOG_LEVEL"] == "debug": return tuple() return "-l", "/dev/null" + + +def _is_logs_redaction_enabled() -> bool: + """Check if logs should be redacted.""" + + # Use this, if it is set + logs_redaction = os.getenv("LOGS_REDACTION") + if logs_redaction is not None: + return bool(strtobool(logs_redaction)) + + # DEPRECATED - Splunk-specific LOGS_REDACTION variable + if splunk.is_splunk_enabled(): + return bool(strtobool(os.getenv("SPLUNK_LOGS_REDACTION", "true"))) + + # Turned on by default + return True diff --git a/buildpack/telemetry/newrelic.py b/buildpack/telemetry/newrelic.py index 597eca61..e03209c3 100644 --- a/buildpack/telemetry/newrelic.py +++ b/buildpack/telemetry/newrelic.py @@ -63,7 +63,7 @@ def update_config(m2ee, app_name): def _get_new_relic_license_key() -> Optional[str]: """Get the New Relic's license key.""" - # Service-binding based integration (on-prem only) + # DEPRECATED - Service-binding integration (on-prem only) vcap_services = util.get_vcap_services_data() if vcap_services and "newrelic" in vcap_services: return vcap_services["newrelic"][0]["credentials"]["licenseKey"] diff --git a/etc/fluentbit/redaction.lua b/etc/fluentbit/redaction.lua index cd981edb..38fd2719 100644 --- a/etc/fluentbit/redaction.lua +++ b/etc/fluentbit/redaction.lua @@ -1,6 +1,6 @@ function apply_redaction(tag, timestamp, record) - local stringtoboolean={ ["true"]=true, ["false"]=false } + local stringtoboolean={ ["True"]=true, ["False"]=false } local patterns = { '\'jdbc:postgresql://(.*)\'', @@ -8,12 +8,8 @@ function apply_redaction(tag, timestamp, record) 'Endpoint set to: s3-(.*)', } - local is_logs_redaction = os.getenv("SPLUNK_LOGS_REDACTION") - if is_logs_redaction == nil then - is_logs_redaction = true - else - is_logs_redaction = stringtoboolean[is_logs_redaction] - end + local is_logs_redaction = os.getenv("LOGS_REDACTION") + is_logs_redaction = stringtoboolean[is_logs_redaction] if is_logs_redaction then table.insert(patterns, '[%w+%.%-_]+@[%w+%.%-_]+%.%a%a+') --email