New Relic Integration

README.md

@@ -598,8 +598,52 @@ The buildpack includes a variety of telemetry agents that can be configured to c
 
 ### New Relic
 
+#### Set up New Relic integration
+
+[Fluent Bit](https://docs.fluentbit.io/manual/) is used to collect Mendix Runtime logs to [New Relic](https://newrelic.com/).
+
+The metrics are collected by the New Relic Java Agent and an integration with Telegraf. The first collects some container and database metrics, while the second collects metrics related to the Mendix Runtime.
+
+To enable the integration you must provide the following variables:
+
+| Environment variable    | Value example                                  | Default | Description                                                                                                                            |
+|-------------------------|------------------------------------------------|---------|----------------------------------------------------------------------------------------------------------------------------------------|
+| `NEW_RELIC_LICENSE_KEY` | `api_key`                                      | -       | License Key or API Key ([docs](https://docs.newrelic.com/docs/apis/intro-apis/new-relic-api-keys/))           |
+| `NEW_RELIC_METRICS_API` | `https://metric-api.eu.newrelic.com/metric/v1` | -       | Metrics endpoint API ([docs](https://docs.newrelic.com/docs/data-apis/ingest-apis/metric-api/report-metrics-metric-api/#api-endpoint)) |
+| `NEW_RELIC_LOGS_API`    | `https://log-api.eu.newrelic.com/log/v1`       | -       | Logs endpoint API ([docs](https://docs.newrelic.com/docs/logs/log-api/introduction-log-api/))                                          |
+
+:warning: For the first usage of the New Relic integration, the Mendix app should be redeployed after setting the variables up.
+
+Custom tags
+You can also set up custom tags in the following format key:value. We recommend that you add the following custom tags:
+
+app:{app_name} – this enables you to identify all logs sent from your app (for example, app:customermanagement)
+env:{environment_name} – this enables you to identify logs sent from a particular environment so you can separate out production logs from test logs (for example, env:accp)
+
+#### Metadata
+
+In addition to the runtime application logs, the following JSON-formatted metadata is automatically sent to New Relic:
+
+* `environment_id` - unique identifier of the environment;
+* `instance_index` - number of the application instance;
+* `hostname` - name of the application host;
+* `application_name` - default application name, retrieved from domain name;
+* `model_version` - model version of the Mendix runtime;
+* `runtime_version` - version of the Mendix runtime.
+
+#### Custom tags (IN PROGRESS)
+
+You can also set up custom tags in the following format `key:value`. We recommend that you add the following custom tags:
+
+* `app:{app_name}` – this enables you to identify all logs sent from your app (for example, **app:customermanagement**)
+* `env:{environment_name}` – this enables you to identify logs sent from a particular environment so you can separate out production logs from test logs (for example, **env:accp**)
+
+#### Service-base integration (on-prem only)
+
 To enable New Relic, simply bind a New Relic service to this app and settings will be picked up automatically. Afterwards you have to restage your application to enable the New Relic agent.
 
+This integration does not support logs or custom metrics.
+
 ### Splunk
 
 #### Set up Splunk integration

buildpack/stage.py

@@ -200,8 +200,8 @@ def cleanup_dependency_cache(cached_dir, dependency_list):
     appdynamics.stage(BUILDPACK_DIR, DOT_LOCAL_LOCATION, CACHE_DIR)
     dynatrace.stage(BUILDPACK_DIR, DOT_LOCAL_LOCATION, CACHE_DIR)
     splunk.stage()
-    fluentbit.stage(BUILDPACK_DIR, DOT_LOCAL_LOCATION, CACHE_DIR)
     newrelic.stage(BUILDPACK_DIR, DOT_LOCAL_LOCATION, CACHE_DIR)
+    fluentbit.stage(BUILDPACK_DIR, DOT_LOCAL_LOCATION, CACHE_DIR)
     mx_java_agent.stage(BUILDPACK_DIR, DOT_LOCAL_LOCATION, CACHE_DIR, runtime_version)
     telegraf.stage(BUILDPACK_DIR, DOT_LOCAL_LOCATION, CACHE_DIR, runtime_version)
     datadog.stage(BUILDPACK_DIR, DOT_LOCAL_LOCATION, CACHE_DIR)

buildpack/telemetry/fluentbit.py

@@ -3,18 +3,20 @@
 import subprocess
 import shutil
 import socket
+from typing import List
 
 import backoff
 
 from buildpack import util
-from buildpack.telemetry import splunk
+from buildpack.telemetry import newrelic, splunk
 
 
 NAMESPACE = "fluentbit"
 CONF_FILENAME = f"{NAMESPACE}.conf"
 FILTER_FILENAMES = ("redaction.lua", "metadata.lua")
 FLUENTBIT_ENV_VARS = {
     "FLUENTBIT_LOGS_PORT": os.getenv("FLUENTBIT_LOGS_PORT", default="5170"),
+    "FLUENTBIT_DEBUG": os.getenv("FLUENTBIT_DEBUG", default="false"),
 }
 
 
@@ -23,6 +25,19 @@ def _set_default_env(m2ee):
         util.upsert_custom_environment_variable(m2ee, var_name, value)
 
 
+def _get_output_conf_filenames() -> List[str]:
+    """
+    Determine the output configs to use. Only enabled integrations
+    will have the output file in the container.
+    """
+    output_conf_files: List[str] = []
+    if splunk.is_splunk_enabled():
+        output_conf_files.append("output_splunk.conf")
+    if newrelic.is_enabled():
+        output_conf_files.append("output_newrelic.conf")
+    return output_conf_files
+
+
 def stage(buildpack_dir, destination_path, cache_path):
 
     if not is_fluentbit_enabled():
@@ -36,7 +51,11 @@ def stage(buildpack_dir, destination_path, cache_path):
         cache_dir=cache_path,
     )
 
-    for filename in (CONF_FILENAME, *FILTER_FILENAMES):
+    output_conf_files = _get_output_conf_filenames()
+
+    for filename in (
+            CONF_FILENAME, *FILTER_FILENAMES, *output_conf_files
+    ):
         shutil.copy(
             os.path.join(buildpack_dir, "etc", NAMESPACE, filename),
             os.path.join(
@@ -82,26 +101,33 @@ def run(model_version, runtime_version):
         "fluent-bit",
     )
 
-    fluentbit_config_path = os.path.join(
-        fluentbit_dir,
-        CONF_FILENAME,
-    )
+    fluentbit_config_path = os.path.join(fluentbit_dir, CONF_FILENAME)
+
+    fluentbit_log_file = _get_log_file()
 
     if not os.path.exists(fluentbit_bin_path):
         logging.warning(
             "Fluent Bit is not installed yet. "
             "Please redeploy your application to complete "
             "Fluent Bit installation."
         )
-        splunk.print_failed_message()
+        splunk.integration_setup(False)
+        newrelic.integration_setup(False)
         return
 
     agent_environment = _set_up_environment(model_version, runtime_version)
 
     logging.info("Starting Fluent Bit...")
 
     subprocess.Popen(
-        (fluentbit_bin_path, "-c", fluentbit_config_path), env=agent_environment
+        (
+            fluentbit_bin_path,
+            "-c",
+            fluentbit_config_path,
+            "-l",
+            fluentbit_log_file
+        ),
+        env=agent_environment,
     )
 
     # The runtime does not handle a non-open logs endpoint socket
@@ -113,24 +139,36 @@ def _await_logging_endpoint():
         )
 
     logging.info("Awaiting Fluent Bit log subscriber...")
-    if _await_logging_endpoint() == 0:
+    success = True
+    if _await_logging_endpoint() != 0:
+        success = False
+
+    _integration_setup(success)
+    splunk.integration_setup(success)
+    newrelic.integration_setup(success)
+
+
+def _integration_setup(success: bool) -> None:
+    """Call when the setup is done."""
+    if success:
         logging.info("Fluent Bit log subscriber is ready.")
-        splunk.print_ready_message()
     else:
         logging.error(
             "Fluent Bit log subscriber was not initialized correctly."
             "Application logs will not be shipped to Fluent Bit."
         )
-        splunk.print_failed_message()
 
 
 def _set_up_environment(model_version, runtime_version):
     env_vars = dict(os.environ.copy())
 
-    env_vars["SPLUNK_APP_HOSTNAME"] = util.get_hostname()
-    env_vars["SPLUNK_APP_NAME"] = util.get_app_from_domain()
-    env_vars["SPLUNK_APP_RUNTIME_VERSION"] = str(runtime_version)
-    env_vars["SPLUNK_APP_MODEL_VERSION"] = model_version
+    # TODO: Improve this - it should be on the envvars already
+    env_vars["FLUENTBIT_LOGS_PORT"] = "5170"
+
+    env_vars["FLUENTBIT_APP_HOSTNAME"] = util.get_hostname()
+    env_vars["FLUENTBIT_APP_NAME"] = util.get_app_from_domain()
+    env_vars["FLUENTBIT_APP_RUNTIME_VERSION"] = str(runtime_version)
+    env_vars["FLUENTBIT_APP_MODEL_VERSION"] = model_version
 
     return env_vars
 
@@ -139,9 +177,18 @@ def is_fluentbit_enabled():
     """
     The function checks if some modules which requires
     Fluent Bit is configured.
-
     """
 
     return any(
-        [splunk.is_splunk_enabled()]
+        [splunk.is_splunk_enabled(), newrelic.is_enabled()]
     )  # Add other modules, where Fluent Bit is used
+
+
+def _get_log_file() -> str:
+    """Discard logs unless debug is active."""
+    # FluentBit currently does not support log rotation,
+    # so the file must only be used when debugging
+    fluentbit_debug = os.getenv("FLUENTBIT_DEBUG", "false").lower()
+    if fluentbit_debug == "true":
+        return "/app/log/fluentbit.log"
+    return "/dev/null"

buildpack/telemetry/metrics.py

@@ -18,7 +18,7 @@
 from lib.m2ee.version import MXVersion
 from lib.m2ee.util import strtobool
 
-from . import datadog, appdynamics, dynatrace
+from . import appdynamics, datadog, dynatrace, newrelic
 
 METRICS_REGISTRIES_KEY = "Metrics.Registries"
 
@@ -136,6 +136,7 @@ def configure_metrics_registry(m2ee):
         or get_appmetrics_target()
         or appdynamics.machine_agent_enabled()
         or dynatrace.is_telegraf_enabled()
+        or newrelic.is_enabled()
     ):
         allow_list, deny_list = get_apm_filters()
         paidapps_registries.append(get_statsd_registry(allow_list, deny_list))

buildpack/telemetry/newrelic.py

@@ -1,11 +1,16 @@
 import logging
 import os
+from typing import Dict
 
 from buildpack import util
 
 NAMESPACE = "newrelic"
 ROOT_DIR = ".local"
 
+REQUIRED_NEW_RELIC_ENV_VARS = [
+    "NEW_RELIC_LICENSE_KEY", "NEW_RELIC_LOGS_API", "NEW_RELIC_METRICS_API"
+]
+
 
 def stage(buildpack_dir, install_path, cache_path):
     if get_new_relic_license_key():
@@ -44,7 +49,47 @@ def update_config(m2ee, app_name):
 
 
 def get_new_relic_license_key():
+    """Get the New Relic's license key."""
+    # Service-binding based integration (on-prem only)
     vcap_services = util.get_vcap_services_data()
     if vcap_services and "newrelic" in vcap_services:
         return vcap_services["newrelic"][0]["credentials"]["licenseKey"]
-    return None
+
+    return os.environ.get("NEW_RELIC_LICENSE_KEY", None)
+
+
+def is_enabled() -> bool:
+    """
+    The function checks if all environment variables required
+    for New Relic connection are set up. The service-binding
+    based integration (on-prem only) does not care about this.
+    """
+    return all(map(os.getenv, REQUIRED_NEW_RELIC_ENV_VARS))
+
+
+def get_metrics_config() -> Dict:
+    """Configs to be used by telegraf."""
+    return {
+        "api_key": os.getenv("NEW_RELIC_LICENSE_KEY"),
+        "metrics_base_url": os.getenv("NEW_RELIC_METRICS_API"),
+    }
+
+
+def get_metrics_tags(app_name) -> Dict:
+    """Tags to be used by telegraf."""
+    return {
+        "app": util.get_app_from_domain(),
+        "instance_index": int(os.getenv("CF_INSTANCE_INDEX", "0")),
+        "app_name": app_name
+    }
+
+
+def integration_setup(success: bool) -> None:
+    """Call when the setup is done."""
+    if not is_enabled():
+        return
+
+    if success:
+        logging.info("New Relic has been configured successfully.")
+    else:
+        logging.error("Failed to configure New Relic.")

buildpack/telemetry/splunk.py

@@ -31,30 +31,18 @@ def update_config(m2ee):
     _set_default_env(m2ee)
 
 
-def print_ready_message():
+def integration_setup(success: bool) -> None:
     """
     This function can be called from external module.
-    For example: fluentbit.py calls this function when Fluent Bit is ready.
-
-    """
-
-    if not is_splunk_enabled():
-        return
-
-    logging.info("Splunk has been configured successfully.")
-
-
-def print_failed_message():
-    """
-    This function can be called from external module.
-    For example: fluentbit.py calls this function when Fluent Bit is failed.
-
+    For example: fluentbit.py calls this function when Fluent Bit is done.
     """
-
     if not is_splunk_enabled():
         return
 
-    logging.error("Failed to configure Splunk.")
+    if success:
+        logging.info("Splunk has been configured successfully.")
+    else:
+        logging.error("Failed to configure Splunk.")
 
 
 def is_splunk_enabled():