New Relic Integration

Makefile

@@ -93,7 +93,7 @@ test_unit:
 
 .PHONY: test_integration
 test_integration:
-	pytest -vvv --capture=no --timeout=3600 --color=no ${TEST_FILES}
+	pytest -vvv --timeout=3600 --color=no ${TEST_FILES}
 
 .PHONY: test
 test: test_unit test_integration

README.md

@@ -598,8 +598,59 @@ The buildpack includes a variety of telemetry agents that can be configured to c
 
 ### New Relic
 
+#### Set up New Relic integration
+
+[Fluent Bit](https://docs.fluentbit.io/manual/) is used to collect Mendix Runtime logs to [New Relic](https://newrelic.com/).
+
+The metrics are collected by the [New Relic Java Agent](https://docs.newrelic.com/docs/apm/agents/java-agent/getting-started/introduction-new-relic-java/) and an integration with the [Telegraf agent](https://docs.influxdata.com/telegraf/).
+The first one collects container and database metrics, while the second collects metrics related to the Mendix Runtime.
+
+To enable the integration you must provide the following variables:
+
+| Environment variable    | Value example                                  | Default                  | Description                                                                                                                            |
+|-------------------------|------------------------------------------------|--------------------------|----------------------------------------------------------------------------------------------------------------------------------------|
+| `NEW_RELIC_LICENSE_KEY` | `api_key`                                      | -                        | License Key or API Key ([docs](https://docs.newrelic.com/docs/apis/intro-apis/new-relic-api-keys/))                                    |
+| `NEW_RELIC_METRICS_URI` | `https://metric-api.eu.newrelic.com/metric/v1` | -                        | Metrics endpoint API ([docs](https://docs.newrelic.com/docs/data-apis/ingest-apis/metric-api/report-metrics-metric-api/#api-endpoint)) |
+| `NEW_RELIC_LOGS_URI`    | `https://log-api.eu.newrelic.com/log/v1`       | -                        | Logs endpoint API ([docs](https://docs.newrelic.com/docs/logs/log-api/introduction-log-api/))                                          |
+| `NEW_RELIC_APP_NAME`    | `MyApp`                                        | application domain name  | Mendix App environment ID                                                                                                              |
+
+:warning: For the first usage of the New Relic integration, the Mendix app should be redeployed after setting the variables up.
+
+Custom tags
+You can also set up custom tags in the following format key:value. We recommend that you add the following custom tags:
+
+app:{app_name} – this enables you to identify all logs sent from your app (for example, app:customermanagement)
+env:{environment_name} – this enables you to identify logs sent from a particular environment so you can separate out production logs from test logs (for example, env:accp)
+
+#### Metadata
+
+In addition to the runtime application logs, the following JSON-formatted metadata is automatically sent to New Relic:
+
+* `environment_id` - unique identifier of the environment;
+* `instance_index` - number of the application instance;
+* `hostname` - name of the application host;
+* `application_name` - default application name, retrieved from domain name;
+* `model_version` - model version of the Mendix runtime;
+* `runtime_version` - version of the Mendix runtime.
+
+The same values are also provided with the custom metrics pushed to New Relic.
+
+#### Custom tags
+
+You can also set up custom tags in the following format `key:value`. We recommend that you add the following custom tags:
+
+* `app:{app_name}` – this enables you to identify all logs sent from your app (for example, **app:customermanagement**)
+* `env:{environment_name}` – this enables you to identify logs sent from a particular environment so you can separate out production logs from test logs (for example, **env:accp**)
+
+#### Service-base integration (on-prem only)
+
 To enable New Relic, simply bind a New Relic service to this app and settings will be picked up automatically. Afterwards you have to restage your application to enable the New Relic agent.
 
+This integration does not support logs or custom metrics.
+
+:warning: The default NEW_RELIC_APP_NAME for this integration used to be the environment ID of the application. Now the value is the domain name set to the application.
+If you want to keep using the environment id, you will have to set this variable yourself to that value.
+
 ### Splunk
 
 #### Set up Splunk integration

buildpack/stage.py

@@ -200,8 +200,8 @@ def cleanup_dependency_cache(cached_dir, dependency_list):
     appdynamics.stage(BUILDPACK_DIR, DOT_LOCAL_LOCATION, CACHE_DIR)
     dynatrace.stage(BUILDPACK_DIR, DOT_LOCAL_LOCATION, CACHE_DIR)
     splunk.stage()
-    fluentbit.stage(BUILDPACK_DIR, DOT_LOCAL_LOCATION, CACHE_DIR)
     newrelic.stage(BUILDPACK_DIR, DOT_LOCAL_LOCATION, CACHE_DIR)
+    fluentbit.stage(BUILDPACK_DIR, DOT_LOCAL_LOCATION, CACHE_DIR)
     mx_java_agent.stage(BUILDPACK_DIR, DOT_LOCAL_LOCATION, CACHE_DIR, runtime_version)
     telegraf.stage(BUILDPACK_DIR, DOT_LOCAL_LOCATION, CACHE_DIR, runtime_version)
     datadog.stage(BUILDPACK_DIR, DOT_LOCAL_LOCATION, CACHE_DIR)

buildpack/telemetry/fluentbit.py

@@ -3,18 +3,22 @@
 import subprocess
 import shutil
 import socket
+from time import sleep
+from typing import List
 
 import backoff
+from lib.m2ee.util import strtobool
 
 from buildpack import util
-from buildpack.telemetry import splunk
+from buildpack.telemetry import newrelic, splunk
 
 
 NAMESPACE = "fluentbit"
 CONF_FILENAME = f"{NAMESPACE}.conf"
 FILTER_FILENAMES = ("redaction.lua", "metadata.lua")
 FLUENTBIT_ENV_VARS = {
     "FLUENTBIT_LOGS_PORT": os.getenv("FLUENTBIT_LOGS_PORT", default="5170"),
+    "FLUENTBIT_DEBUG": os.getenv("FLUENTBIT_DEBUG", default="false"),
 }
 
 
@@ -23,11 +27,26 @@ def _set_default_env(m2ee):
         util.upsert_custom_environment_variable(m2ee, var_name, value)
 
 
-def stage(buildpack_dir, destination_path, cache_path):
+def _get_output_conf_filenames() -> List[str]:
+    """
+    Determine the output configs to use. Only enabled integrations
+    will have the output file in the container.
+    """
+    output_conf_files: List[str] = []
+    if splunk.is_splunk_enabled():
+        print("SPLUNK NOT ENABLED - WON'T PRINT")
+        output_conf_files.append("output_splunk.conf")
+    if newrelic.is_enabled():
+        print("NEW RELIC ENABLED")
+        output_conf_files.append("output_newrelic.conf")
+    return output_conf_files
 
+
+def stage(buildpack_dir, destination_path, cache_path):
+    print("BEGIN STAGE FLUENTBIT")
     if not is_fluentbit_enabled():
         return
-
+    print("STILL IN STAGE")
     util.resolve_dependency(
         "fluentbit",
         # destination_path - DOT_LOCAL_LOCATION
@@ -36,20 +55,24 @@ def stage(buildpack_dir, destination_path, cache_path):
         cache_dir=cache_path,
     )
 
-    for filename in (CONF_FILENAME, *FILTER_FILENAMES):
+    print("GET OUTPUT CONFIGS")
+    output_conf_files = _get_output_conf_filenames()
+    print(f"CONF FILES = {output_conf_files}")
+
+    for filename in (
+            CONF_FILENAME, *FILTER_FILENAMES, *output_conf_files
+    ):
         shutil.copy(
             os.path.join(buildpack_dir, "etc", NAMESPACE, filename),
             os.path.join(
                 destination_path,
                 NAMESPACE,
             ),
         )
-
     logging.info("Fluent Bit has been installed successfully.")
 
 
 def update_config(m2ee):
-
     if not is_fluentbit_enabled():
         return
 
@@ -68,7 +91,6 @@ def update_config(m2ee):
 
 
 def run(model_version, runtime_version):
-
     if not is_fluentbit_enabled():
         return
 
@@ -82,66 +104,98 @@ def run(model_version, runtime_version):
         "fluent-bit",
     )
 
-    fluentbit_config_path = os.path.join(
-        fluentbit_dir,
-        CONF_FILENAME,
-    )
+    fluentbit_config_path = os.path.join(fluentbit_dir, CONF_FILENAME)
+
+    # fluentbit_log_file = _get_log_file()
 
     if not os.path.exists(fluentbit_bin_path):
         logging.warning(
             "Fluent Bit is not installed yet. "
             "Please redeploy your application to complete "
             "Fluent Bit installation."
         )
-        splunk.print_failed_message()
+        splunk.integration_complete(success=False)
+        newrelic.integration_complete(success=False)
         return
 
     agent_environment = _set_up_environment(model_version, runtime_version)
 
     logging.info("Starting Fluent Bit...")
 
     subprocess.Popen(
-        (fluentbit_bin_path, "-c", fluentbit_config_path), env=agent_environment
+        (
+            fluentbit_bin_path,
+            "-c",
+            fluentbit_config_path,
+            # "-l",
+            # fluentbit_log_file,
+        ),
+        env=agent_environment,
     )
 
     # The runtime does not handle a non-open logs endpoint socket
     # gracefully, so wait until it's up
-    @backoff.on_predicate(backoff.expo, lambda x: x > 0, max_time=10)
+    @backoff.on_predicate(backoff.expo, lambda x: x > 0, max_time=100)
     def _await_logging_endpoint():
+        print("WAITING")
         return socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect_ex(
             ("localhost", int(FLUENTBIT_ENV_VARS["FLUENTBIT_LOGS_PORT"]))
         )
 
     logging.info("Awaiting Fluent Bit log subscriber...")
-    if _await_logging_endpoint() == 0:
+    success = True
+    if _await_logging_endpoint() != 0:
+        success = False
+
+    _integration_complete(success)
+    splunk.integration_complete(success)
+    newrelic.integration_complete(success)
+
+
+def _integration_complete(success: bool) -> None:
+    """Call when the setup is done."""
+    if success:
         logging.info("Fluent Bit log subscriber is ready.")
-        splunk.print_ready_message()
     else:
         logging.error(
             "Fluent Bit log subscriber was not initialized correctly."
             "Application logs will not be shipped to Fluent Bit."
         )
-        splunk.print_failed_message()
 
 
 def _set_up_environment(model_version, runtime_version):
+    fluentbit_env_vars = FLUENTBIT_ENV_VARS
+
     env_vars = dict(os.environ.copy())
 
-    env_vars["SPLUNK_APP_HOSTNAME"] = util.get_hostname()
-    env_vars["SPLUNK_APP_NAME"] = util.get_app_from_domain()
-    env_vars["SPLUNK_APP_RUNTIME_VERSION"] = str(runtime_version)
-    env_vars["SPLUNK_APP_MODEL_VERSION"] = model_version
+    env_vars["FLUENTBIT_APP_HOSTNAME"] = util.get_hostname()
+    env_vars["FLUENTBIT_APP_NAME"] = util.get_app_from_domain()
+    env_vars["FLUENTBIT_APP_RUNTIME_VERSION"] = str(runtime_version)
+    env_vars["FLUENTBIT_APP_MODEL_VERSION"] = model_version
 
-    return env_vars
+    fluentbit_env_vars.update(env_vars)
+    return fluentbit_env_vars
 
 
 def is_fluentbit_enabled():
     """
     The function checks if some modules which requires
     Fluent Bit is configured.
-
     """
 
+    print("IS FB ENABLED?")
+    print(f"splunk = {splunk.is_splunk_enabled()}")
+    print(f"new relic = {newrelic.is_enabled()}")
+    print(f"final = {any([splunk.is_splunk_enabled(), newrelic.is_enabled()])}")  # noqa: line-too-long
     return any(
-        [splunk.is_splunk_enabled()]
+        [splunk.is_splunk_enabled(), newrelic.is_enabled()]
     )  # Add other modules, where Fluent Bit is used
+
+
+def _get_log_file() -> str:
+    """Discard logs unless debug is active."""
+    # FluentBit currently does not support log rotation,
+    # so the file must only be used when debugging
+    if strtobool(FLUENTBIT_ENV_VARS["FLUENTBIT_DEBUG"]):
+        return "/app/log/fluentbit.log"
+    return "/dev/null"

buildpack/telemetry/metrics.py

@@ -18,7 +18,7 @@
 from lib.m2ee.version import MXVersion
 from lib.m2ee.util import strtobool
 
-from . import datadog, appdynamics, dynatrace
+from . import appdynamics, datadog, dynatrace, newrelic
 
 METRICS_REGISTRIES_KEY = "Metrics.Registries"
 
@@ -136,6 +136,7 @@ def configure_metrics_registry(m2ee):
         or get_appmetrics_target()
         or appdynamics.machine_agent_enabled()
         or dynatrace.is_telegraf_enabled()
+        or newrelic.is_enabled()
     ):
         allow_list, deny_list = get_apm_filters()
         paidapps_registries.append(get_statsd_registry(allow_list, deny_list))