Upgrade datadog-cloudfoundry-buildpack to 4.36.0

.github/.mdl_style.rb

@@ -1,5 +1,6 @@
 all
 exclude_rule 'MD007'
 exclude_rule 'MD013'
+exclude_rule 'MD024'
 exclude_rule 'MD029'
 exclude_rule 'MD034'

.pre-commit-config.yaml

@@ -2,12 +2,12 @@
 fail_fast: false
 repos:
   - repo: https://github.com/adrienverge/yamllint.git
-    rev: v1.30.0
+    rev: v1.33.0
     hooks:
       - id: yamllint
         args: ["--format", "parsable", "--strict"]
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v4.5.0
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
@@ -20,6 +20,6 @@ repos:
     hooks:
       - id: markdownlint_docker
   - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: 'v0.0.261'
+    rev: 'v0.1.6'
     hooks:
       - id: ruff

Makefile

@@ -5,8 +5,8 @@ TEST_FILES ?= tests/integration/test_*.py
 VERSION ?= $(shell git tag --list --sort=-version:refname "v*" | head -n 1)
 COMMIT ?= $(shell git rev-parse --short HEAD)
 
-PIP_TOOLS_VERSION ?= 6.13.0
-PIP_VERSION ?= 23.1
+PIP_TOOLS_VERSION ?= 7.3.0
+PIP_VERSION ?= 23.3.1
 PYTHON_PLATFORM ?= manylinux2014_x86_64
 PYTHON_VERSION ?= 310
 

README.md

@@ -196,6 +196,8 @@ cf set-env <YOUR_APP> DATABASE_CONNECTION_PARAMS '{"tcpKeepAlive": "true", "conn
 
 *:warning: If you set `DATABASE_URL` as JDBC connection string (prefixed with `jdbc:` and including parameters, `DATABASE_CONNECTION_PARAMS` is not required.*
 
+To allow connection to an AWS RDS database the buildpack selects the regional CA certificate stored in [`rds-certificates`](etc/rds-certificates). If the region's certificate doesn't exist, the buildpack will fail with an error `Could not find database CA certificate in map`.
+
 #### Supported VCAP Schemas
 
  Cloud Foundry database services are detected from Cloud Foundry service bindings ([VCAP](https://docs.cloudfoundry.org/devguide/deploy-apps/environment-variable.html#VCAP-SERVICES)) and translated into Mendix Runtime configuration. In case no database service is bound, the fallback is the environment variable `DATABASE_URL`.
@@ -895,7 +897,7 @@ For the third-party integrations explained above, in addition to the metrics col
 This configuration also has a filtering mechanism that allows users to specify metrics they allow or deny for the vendor they are using.
 To filter the ingestion of custom runtime metrics to third party APMs, users should provide a list of prefixes of the metrics they want to allow/deny using the environment variables listed below.
 
-Note: Custom database metrics cannot be filtered by name, to turn them off, the `APPMETRICS_INCLUDE_DB` environment variable should be set to false. 
+Note: Custom database metrics cannot be filtered by name, to turn them off, the `APPMETRICS_INCLUDE_DB` environment variable should be set to false.
 
 #### APM_METRICS_FILTER_ALLOW
 
@@ -909,7 +911,7 @@ cf set-env <YOUR_APP> APM_METRICS_FILTER_ALLOW 'mx.runtime.stats.sessions'
 
 #### APM_METRICS_FILTER_DENY
 
-Comma-separated list of prefixes for the metrics to be denied. 
+Comma-separated list of prefixes for the metrics to be denied.
 
 For example, to deny all metrics starting with jetty or mx.runtime, the environment variable should be set to `jetty,mx.runtime`:
 
@@ -925,7 +927,6 @@ If this environment variable is set to `true`, all metrics will be denied regard
 cf set-env <YOUR_APP> APM_METRICS_FILTER_DENY_ALL true
 ```
 
-
 ## Using the Buildpack without an Internet Connection
 
 If you are running Cloud Foundry without a connection to the Internet, you should specify an on-premises web server that hosts Mendix Runtime files and other buildpack dependencies. You can set the endpoint with the following environment variable:

bin/install-python

@@ -48,4 +48,4 @@ function get_python_from_manifest() {
   echo "$python_dep_info"
 }
 
-main "${@:-}"
+main "${@:-}"

buildpack/core/java.py

@@ -160,7 +160,7 @@ def stage(buildpack_path, cache_path, local_path, java_major_version):
 
 
 def _compose_jvm_target_dir(dependency):
-    return f"usr/lib/jvm/{dependency['vendor']}-{dependency['type']}-{dependency['version']}-{dependency['vendor']}-x64"  # noqa: line-too-long
+    return f"usr/lib/jvm/{dependency['vendor']}-{dependency['type']}-{dependency['version']}-{dependency['vendor']}-x64"  # noqa: C0301
 
 
 def _get_java_dependency(

buildpack/core/nginx.py

@@ -14,15 +14,15 @@
 from lib.m2ee.util import strtobool
 
 ALLOWED_HEADERS = {
-    "X-Frame-Options": r"(?i)(^allow-from https?://([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*(:\d+)?$|^deny$|^sameorigin$)",  # noqa: line-too-long
-    "Referrer-Policy": r"(?i)(^no-referrer$|^no-referrer-when-downgrade$|^origin|origin-when-cross-origin$|^same-origin|strict-origin$|^strict-origin-when-cross-origin$|^unsafe-url$)",  # noqa: line-too-long
-    "Access-Control-Allow-Origin": r"(?i)(^\*$|^null$|^https?://([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*(:\d+)?$)",  # noqa: line-too-long
+    "X-Frame-Options": r"(?i)(^allow-from https?://([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*(:\d+)?$|^deny$|^sameorigin$)",  # noqa: C0301
+    "Referrer-Policy": r"(?i)(^no-referrer$|^no-referrer-when-downgrade$|^origin|origin-when-cross-origin$|^same-origin|strict-origin$|^strict-origin-when-cross-origin$|^unsafe-url$)",  # noqa: C0301
+    "Access-Control-Allow-Origin": r"(?i)(^\*$|^null$|^https?://([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*(:\d+)?$)",  # noqa: C0301
     "X-Content-Type-Options": r"(?i)(^nosniff$)",
     "Content-Security-Policy": r"[a-zA-Z0-9:;/''\"\*_\- \.\n?=%&+]+",
-    "Strict-Transport-Security": r"(?i)(^max-age=[0-9]*$|^max-age=[0-9]*; includeSubDomains$|^max-age=[0-9]*; preload$)",  # noqa: line-too-long
-    "X-Permitted-Cross-Domain-Policies": r"(?i)(^all$|^none$|^master-only$|^by-content-type$|^by-ftp-filename$)",  # noqa: line-too-long
+    "Strict-Transport-Security": r"(?i)(^max-age=[0-9]*$|^max-age=[0-9]*; includeSubDomains$|^max-age=[0-9]*; preload$)",  # noqa: C0301
+    "X-Permitted-Cross-Domain-Policies": r"(?i)(^all$|^none$|^master-only$|^by-content-type$|^by-ftp-filename$)",  # noqa: C0301
     "Origin-Trial": r"[a-zA-Z0-9:;/''\"\*_\- \.\n?=%&+]+",
-    "X-XSS-Protection": r"(?i)(^0$|^1$|^1; mode=block$|^1; report=https?://([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*(:\d+)?$)",  # noqa: line-too-long
+    "X-XSS-Protection": r"(?i)(^0$|^1$|^1; mode=block$|^1; report=https?://([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*(:\d+)?$)",  # noqa: C0301
 }
 
 CONFIG_FILE = "nginx/conf/nginx.conf"

buildpack/databroker/config_generator/scripts/generators/debezium.py

@@ -1,12 +1,12 @@
-from buildpack.databroker.config_generator.scripts.generators.debezium_configs.debezium_interface import (  # noqa: line-too-long
+from buildpack.databroker.config_generator.scripts.generators.debezium_configs.debezium_interface import (  # noqa: C0301
     DebeziumInterface,
 )
-from buildpack.databroker.config_generator.scripts.generators.debezium_configs.debezium_default import (  # noqa: line-too-long
+from buildpack.databroker.config_generator.scripts.generators.debezium_configs.debezium_default import (  # noqa: C0301
     DebeziumDefault,
 )
 
 # Do not remove this import, it allows automatic class load
-from buildpack.databroker.config_generator.scripts.generators.debezium_configs import *  # noqa: line-too-long,F403
+from buildpack.databroker.config_generator.scripts.generators.debezium_configs import *  # noqa: C0301, F403
 
 
 def generate_config(config):

buildpack/databroker/config_generator/scripts/generators/debezium_configs/debezium_default.py

@@ -1,4 +1,4 @@
-from buildpack.databroker.config_generator.scripts.generators.debezium_configs.debezium_interface import (  # noqa: line-too-long
+from buildpack.databroker.config_generator.scripts.generators.debezium_configs.debezium_interface import (  # noqa: C0301
     DebeziumInterface,
 )
 

buildpack/databroker/config_generator/scripts/generators/debezium_configs/postgres.py

@@ -1,4 +1,4 @@
-from buildpack.databroker.config_generator.scripts.generators.debezium_configs.debezium_interface import (  # noqa: line-too-long
+from buildpack.databroker.config_generator.scripts.generators.debezium_configs.debezium_interface import (  # noqa: C0301
     DebeziumInterface,
 )
 from buildpack.databroker.config_generator.scripts.constants import (

buildpack/databroker/config_generator/templates/jmx/consumer.py

@@ -18,7 +18,7 @@
     },
     {
         "include": {
-            "bean_regex": "kafka\.consumer:type=consumer-coordinator-metrics,client-id=.*",  # noqa: line-too-long
+            "bean_regex": "kafka\.consumer:type=consumer-coordinator-metrics,client-id=.*",  # noqa: C0301
             "attribute": {
                 "commit-latency-avg": {
                     "alias": "kafka.consumer.coordinator.commit-latency-avg"
@@ -82,10 +82,10 @@
                     "alias": "kafka.consumer.coordinator.partitions-revoked-latency-max"
                 },
                 "partitions-assigned-latency-avg": {
-                    "alias": "kafka.consumer.coordinator.partitions-assigned-latency-avg"  # noqa: line-too-long
+                    "alias": "kafka.consumer.coordinator.partitions-assigned-latency-avg"  # noqa: C0301
                 },
                 "partitions-assigned-latency-max": {
-                    "alias": "kafka.consumer.coordinator.partitions-assigned-latency-max"  # noqa: line-too-long
+                    "alias": "kafka.consumer.coordinator.partitions-assigned-latency-max"  # noqa: C0301
                 },
                 "partitions-lost-latency-avg": {
                     "alias": "kafka.consumer.coordinator.partitions-lost-latency-avg"
@@ -98,7 +98,7 @@
     },
     {
         "include": {
-            "bean_regex": "kafka\.consumer:type=consumer-fetch-manager-metrics,client-id=.*",  # noqa: line-too-long
+            "bean_regex": "kafka\.consumer:type=consumer-fetch-manager-metrics,client-id=.*",  # noqa: C0301
             "attribute": {
                 "bytes-consumed-rate": {
                     "alias": "kafka.consumer.fetch.manager.bytes-consumed-rate"
@@ -146,7 +146,7 @@
     },
     {
         "include": {
-            "bean_regex": "kafka\.consumer:type=consumer-fetch-manager-metrics,client-id=.*,topic=.*",  # noqa: line-too-long
+            "bean_regex": "kafka\.consumer:type=consumer-fetch-manager-metrics,client-id=.*,topic=.*",  # noqa: C0301
             "attribute": {
                 "bytes-consumed-rate": {
                     "alias": "kafka.consumer.fetch.manager.bytes-consumed-rate"
@@ -174,7 +174,7 @@
     },
     {
         "include": {
-            "bean_regex": "kafka\.consumer:type=consumer-fetch-manager-metrics,partition=.*,topic=.*,client-id=.*",  # noqa: line-too-long
+            "bean_regex": "kafka\.consumer:type=consumer-fetch-manager-metrics,partition=.*,topic=.*,client-id=.*",  # noqa: C0301
             "attribute": {
                 "preferred-read-replica": {
                     "alias": "kafka.consumer.fetch.manager.preferred-read-replica"

buildpack/infrastructure/database.py

@@ -11,6 +11,8 @@
 from buildpack import util
 
 
+
+
 def get_config():
     # the following options are validated to get database credentials
     # 1) existence of custom runtime settings Database.... values
@@ -251,9 +253,39 @@ def __init__(self, url, env_vars=None):
         self.m2ee_config = None
 
     def init(self):
+        region_pem_map = {
+        "us-east-1": "us-east-1-bundle.pem",
+        "us-east-2": "us-east-2-bundle.pem",
+        "us-west-1": "us-west-1-bundle.pem",
+        "us-west-2": "us-west-2-bundle.pem",
+        "af-south-1": "af-south-1-bundle.pem",
+        "ap-east-1": "ap-east-1-bundle.pem",
+        "ap-south-2": "ap-south-2-bundle.pem",
+        "ap-southeast-3": "ap-southeast-3-bundle.pem",
+        "ap-southeast-4": "ap-southeast-4-bundle.pem",
+        "ap-south-1": "ap-south-1-bundle.pem",
+        "ap-northeast-3": "ap-northeast-3-bundle.pem",
+        "ap-northeast-1": "ap-northeast-1-bundle.pem",
+        "ap-northeast-2": "ap-northeast-2-bundle.pem",
+        "ap-southeast-1": "ap-southeast-1-bundle.pem",
+        "ap-southeast-2": "ap-southeast-2-bundle.pem",
+        "ca-central-1": "ca-central-1-bundle.pem",
+        "eu-central-1": "eu-central-1-bundle.pem",
+        "eu-west-1": "eu-west-1-bundle.pem",
+        "eu-west-2": "eu-west-2-bundle.pem",
+        "eu-south-1": "eu-south-1-bundle.pem",
+        "eu-west-3": "eu-west-3-bundle.pem",
+        "eu-south-2": "eu-south-2-bundle.pem",
+        "eu-north-1": "eu-north-1-bundle.pem",
+        "eu-central-2": "eu-central-2-bundle.pem",
+        "il-central-1": "il-central-1-bundle.pem",
+        "me-south-1": "me-south-1-bundle.pem",
+        "me-central-1": "me-central-1-bundle.pem",
+        "sa-east-1": "sa-east-1-bundle.pem"
+        }
         patterns = [
-            r"(?P<type>[a-zA-Z0-9]+)://(?P<user>[^:]+):(?P<password>[^@]+)@(?P<host>[^/]+)/(?P<dbname>[^?]*)(?P<extra>\?.*)?",  # noqa: line-too-long
-            r"jdbc:(?P<type>[a-zA-Z0-9]+)://(?P<host>[^;]+);database=(?P<dbname>[^;]*);user=(?P<user>[^;]+);password=(?P<password>.*)$",  # noqa: line-too-long
+            r"(?P<type>[a-zA-Z0-9]+)://(?P<user>[^:]+):(?P<password>[^@]+)@(?P<host>[^/]+)/(?P<dbname>[^?]*)(?P<extra>\?.*)?",  # noqa: C0301
+            r"jdbc:(?P<type>[a-zA-Z0-9]+)://(?P<host>[^;]+);database=(?P<dbname>[^;]*);user=(?P<user>[^;]+);password=(?P<password>.*)$",  # noqa: C0301
         ]
 
         supported_databases = {
@@ -300,18 +332,22 @@ def init(self):
         if database_type == "PostgreSQL":
             jdbc_params.update({"tcpKeepAlive": "true"})
 
-        if database_type == "PostgreSQL" and config["DatabaseHost"].split(":")[
-            0
-        ].endswith(".rds.amazonaws.com"):
-            jdbc_params.update(
-                {
-                    "sslrootcert": os.path.expandvars(
-                        "$HOME/.postgresql/amazon-rds-ca.pem"
-                    )
-                }
-            )
-            jdbc_params.update({"sslmode": "verify-full"})
-
+        try:
+            if database_type == "PostgreSQL" and config["DatabaseHost"].split(":")[
+                0
+            ].endswith(".rds.amazonaws.com"):
+                database_region = config["DatabaseHost"].split('.')[2]
+                jdbc_params.update(
+                    {
+                        "sslrootcert": os.path.expandvars(
+                            "$HOME/.postgresql/"+region_pem_map[database_region]
+                        )
+                    }
+                )
+                jdbc_params.update({"sslmode": "verify-full"})
+        except Exception:
+            raise Exception("Could not find database CA certificate in map")
+
         if database_type == "PostgreSQL" and not self.url.startswith("jdbc:"):
             self.extract_inline_cert(jdbc_params, self.SSLCERT, "postgresql.crt")
             self.extract_inline_cert(jdbc_params, self.SSLKEY, "postgresql.pk8")
@@ -465,7 +501,7 @@ def get_database_password(self):
     def get_database_jdbc_url(self):
         """Return the database jdbc url for the M2EE configuration"""
         url = self.credentials.get("url", "")
-        pattern = r"jdbc:sap://(?P<host>[^:]+):(?P<port>[0-9]+)/?(?P<q>\?(?P<params>.*))?$"  # noqa:line-too-long
+        pattern = r"jdbc:sap://(?P<host>[^:]+):(?P<port>[0-9]+)/?(?P<q>\?(?P<params>.*))?$"  # noqa:C0301
         match = re.search(pattern, url)
         if match is None:
             logging.error("Unable to parse Hana JDBC url string for parameters")
@@ -495,11 +531,42 @@ def get_additional_m2ee_config(self):
 
 def stage(buildpack_dir, build_dir):
     logging.debug("Staging database...")
+    region_pem_map = {
+        "us-east-1": "us-east-1-bundle.pem",
+        "us-east-2": "us-east-2-bundle.pem",
+        "us-west-1": "us-west-1-bundle.pem",
+        "us-west-2": "us-west-2-bundle.pem",
+        "af-south-1": "af-south-1-bundle.pem",
+        "ap-east-1": "ap-east-1-bundle.pem",
+        "ap-south-2": "ap-south-2-bundle.pem",
+        "ap-southeast-3": "ap-southeast-3-bundle.pem",
+        "ap-southeast-4": "ap-southeast-4-bundle.pem",
+        "ap-south-1": "ap-south-1-bundle.pem",
+        "ap-northeast-3": "ap-northeast-3-bundle.pem",
+        "ap-northeast-1": "ap-northeast-1-bundle.pem",
+        "ap-northeast-2": "ap-northeast-2-bundle.pem",
+        "ap-southeast-1": "ap-southeast-1-bundle.pem",
+        "ap-southeast-2": "ap-southeast-2-bundle.pem",
+        "ca-central-1": "ca-central-1-bundle.pem",
+        "eu-central-1": "eu-central-1-bundle.pem",
+        "eu-west-1": "eu-west-1-bundle.pem",
+        "eu-west-2": "eu-west-2-bundle.pem",
+        "eu-south-1": "eu-south-1-bundle.pem",
+        "eu-west-3": "eu-west-3-bundle.pem",
+        "eu-south-2": "eu-south-2-bundle.pem",
+        "eu-north-1": "eu-north-1-bundle.pem",
+        "eu-central-2": "eu-central-2-bundle.pem",
+        "il-central-1": "il-central-1-bundle.pem",
+        "me-south-1": "me-south-1-bundle.pem",
+        "me-central-1": "me-central-1-bundle.pem",
+        "sa-east-1": "sa-east-1-bundle.pem"
+    }
     util.mkdir_p(os.path.join(build_dir, ".postgresql"))
-    shutil.copy(
-        os.path.join(buildpack_dir, "etc", "amazon-rds-ca.pem"),
-        os.path.join(build_dir, ".postgresql", "amazon-rds-ca.pem"),
-    )
+    for key, value in region_pem_map.items():
+        shutil.copy(
+            os.path.join(buildpack_dir, "etc/rds-certificates", value),
+            os.path.join(build_dir, ".postgresql", value ),
+         )
 
 
 def update_config(m2ee):

buildpack/telemetry/appdynamics.py

@@ -23,7 +23,7 @@
         "APPDYNAMICS_AGENT_APPLICATION_NAME",
         default=util.get_app_from_domain(),
     ),
-    "APPDYNAMICS_AGENT_NODE_NAME": f"{os.getenv('APPDYNAMICS_AGENT_NODE_NAME', default='node')}-{CF_APPLICATION_INDEX}",  # noqa: line-too-long
+    "APPDYNAMICS_AGENT_NODE_NAME": f"{os.getenv('APPDYNAMICS_AGENT_NODE_NAME', default='node')}-{CF_APPLICATION_INDEX}",  # noqa: C0301
     "APPDYNAMICS_AGENT_TIER_NAME": os.getenv(
         "APPDYNAMICS_AGENT_TIER_NAME", default=CF_APPLICATION_NAME
     ),
@@ -33,7 +33,7 @@
     "APPDYNAMICS_CONTROLLER_SSL_ENABLED": os.getenv(
         "APPDYNAMICS_CONTROLLER_SSL_ENABLED", default="true"
     ),
-    "APPDYNAMICS_AGENT_UNIQUE_HOST_ID": f"{os.getenv('APPDYNAMICS_AGENT_UNIQUE_HOST_ID', default=CF_APPLICATION_NAME),}-{CF_APPLICATION_INDEX}",  # noqa: line-too-long
+    "APPDYNAMICS_AGENT_UNIQUE_HOST_ID": f"{os.getenv('APPDYNAMICS_AGENT_UNIQUE_HOST_ID', default=CF_APPLICATION_NAME),}-{CF_APPLICATION_INDEX}",  # noqa: C0301
 }
 
 

buildpack/telemetry/appdynamics_telegraf_output.py

@@ -151,7 +151,7 @@ def convert_and_push_payload():
     AppDynamics Machine Agent HTTP listener.
 
     """
-    # AppDynamics Docs: https://docs.appdynamics.com/22.2/en/infrastructure-visibility/machine-agent/extensions-and-custom-metrics/machine-agent-http-listener  # noqa: line-too-long
+    # AppDynamics Docs: https://docs.appdynamics.com/22.2/en/infrastructure-visibility/machine-agent/extensions-and-custom-metrics/machine-agent-http-listener  # noqa: C0301
 
     metrics_str = input()
     metrics_dict = json.loads(metrics_str)