fix(core): import time latency by lazily loading high level modules

CHANGELOG.md

@@ -8,6 +8,11 @@ This project follows [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) fo
 
 ## [Unreleased]
 
+### Fixed
+
+* **Tracer**: Lazy loads X-Ray SDK to improve import perf for those not instantiating Tracer
+* **Metrics**: Convert EMF JSON Schema as Dictionary to reduce I/O and improve import perf
+
 ## [1.10.5] - 2021-02-17
 
 No changes. Bumped version to trigger new pipeline build for layer publishing.

Makefile

@@ -17,7 +17,8 @@ lint: format
 	poetry run flake8 aws_lambda_powertools/* tests/*
 
 test:
-	poetry run pytest -vvv --cov=./ --cov-report=xml
+	poetry run pytest -m "not perf" --cov=./ --cov-report=xml
+	poetry run pytest --cache-clear tests/performance
 
 coverage-html:
 	poetry run pytest --cov-report html

aws_lambda_powertools/metrics/base.py

@@ -3,7 +3,6 @@
 import logging
 import numbers
 import os
-import pathlib
 from collections import defaultdict
 from enum import Enum
 from typing import Any, Dict, List, Union
@@ -13,13 +12,10 @@
 from ..shared import constants
 from ..shared.functions import resolve_env_var_choice
 from .exceptions import MetricUnitError, MetricValueError, SchemaValidationError
+from .schema import CLOUDWATCH_EMF_SCHEMA
 
 logger = logging.getLogger(__name__)
 
-_schema_path = pathlib.Path(__file__).parent / "./schema.json"
-with _schema_path.open() as f:
-    CLOUDWATCH_EMF_SCHEMA = json.load(f)
-
 MAX_METRICS = 100
 
 

aws_lambda_powertools/metrics/schema.json → aws_lambda_powertools/metrics/schema.py

@@ -1,114 +1,94 @@
-{
-    "type": "object",
-    "title": "Root Node",
-    "required": [
-        "_aws"
-    ],
+# flake8: noqa
+CLOUDWATCH_EMF_SCHEMA = {
     "properties": {
         "_aws": {
             "$id": "#/properties/_aws",
-            "type": "object",
-            "title": "Metadata",
-            "required": [
-                "Timestamp",
-                "CloudWatchMetrics"
-            ],
             "properties": {
-                "Timestamp": {
-                    "$id": "#/properties/_aws/properties/Timestamp",
-                    "type": "integer",
-                    "title": "The Timestamp Schema",
-                    "examples": [
-                        1565375354953
-                    ]
-                },
                 "CloudWatchMetrics": {
                     "$id": "#/properties/_aws/properties/CloudWatchMetrics",
-                    "type": "array",
-                    "title": "MetricDirectives",
                     "items": {
                         "$id": "#/properties/_aws/properties/CloudWatchMetrics/items",
-                        "type": "object",
-                        "title": "MetricDirective",
-                        "required": [
-                            "Namespace",
-                            "Dimensions",
-                            "Metrics"
-                        ],
                         "properties": {
-                            "Namespace": {
-                                "$id": "#/properties/_aws/properties/CloudWatchMetrics/items/properties/Namespace",
-                                "type": "string",
-                                "title": "CloudWatch Metrics Namespace",
-                                "examples": [
-                                    "MyApp"
-                                ],
-                                "pattern": "^(.*)$",
-                                "minLength": 1
-                            },
                             "Dimensions": {
                                 "$id": "#/properties/_aws/properties/CloudWatchMetrics/items/properties/Dimensions",
-                                "type": "array",
-                                "title": "The Dimensions Schema",
-                                "minItems": 1,
                                 "items": {
                                     "$id": "#/properties/_aws/properties/CloudWatchMetrics/items/properties/Dimensions/items",
-                                    "type": "array",
-                                    "title": "DimensionSet",
-                                    "minItems": 1,
-                                    "maxItems": 9,
                                     "items": {
                                         "$id": "#/properties/_aws/properties/CloudWatchMetrics/items/properties/Dimensions/items/items",
-                                        "type": "string",
-                                        "title": "DimensionReference",
-                                        "examples": [
-                                            "Operation"
-                                        ],
+                                        "examples": ["Operation"],
+                                        "minItems": 1,
                                         "pattern": "^(.*)$",
-                                        "minItems": 1
-                                    }
-                                }
+                                        "title": "DimensionReference",
+                                        "type": "string",
+                                    },
+                                    "maxItems": 9,
+                                    "minItems": 1,
+                                    "title": "DimensionSet",
+                                    "type": "array",
+                                },
+                                "minItems": 1,
+                                "title": "The " "Dimensions " "Schema",
+                                "type": "array",
                             },
                             "Metrics": {
                                 "$id": "#/properties/_aws/properties/CloudWatchMetrics/items/properties/Metrics",
-                                "type": "array",
-                                "title": "MetricDefinitions",
-                                "minItems": 1,
                                 "items": {
                                     "$id": "#/properties/_aws/properties/CloudWatchMetrics/items/properties/Metrics/items",
-                                    "type": "object",
-                                    "title": "MetricDefinition",
-                                    "required": [
-                                        "Name"
-                                    ],
                                     "minItems": 1,
                                     "properties": {
                                         "Name": {
                                             "$id": "#/properties/_aws/properties/CloudWatchMetrics/items/properties/Metrics/items/properties/Name",
-                                            "type": "string",
-                                            "title": "MetricName",
-                                            "examples": [
-                                                "ProcessingLatency"
-                                            ],
+                                            "examples": ["ProcessingLatency"],
+                                            "minLength": 1,
                                             "pattern": "^(.*)$",
-                                            "minLength": 1
+                                            "title": "MetricName",
+                                            "type": "string",
                                         },
                                         "Unit": {
                                             "$id": "#/properties/_aws/properties/CloudWatchMetrics/items/properties/Metrics/items/properties/Unit",
-                                            "type": "string",
+                                            "examples": ["Milliseconds"],
+                                            "pattern": "^(Seconds|Microseconds|Milliseconds|Bytes|Kilobytes|Megabytes|Gigabytes|Terabytes|Bits|Kilobits|Megabits|Gigabits|Terabits|Percent|Count|Bytes\\/Second|Kilobytes\\/Second|Megabytes\\/Second|Gigabytes\\/Second|Terabytes\\/Second|Bits\\/Second|Kilobits\\/Second|Megabits\\/Second|Gigabits\\/Second|Terabits\\/Second|Count\\/Second|None)$",
                                             "title": "MetricUnit",
-                                            "examples": [
-                                                "Milliseconds"
-                                            ],
-                                            "pattern": "^(Seconds|Microseconds|Milliseconds|Bytes|Kilobytes|Megabytes|Gigabytes|Terabytes|Bits|Kilobits|Megabits|Gigabits|Terabits|Percent|Count|Bytes\\/Second|Kilobytes\\/Second|Megabytes\\/Second|Gigabytes\\/Second|Terabytes\\/Second|Bits\\/Second|Kilobits\\/Second|Megabits\\/Second|Gigabits\\/Second|Terabits\\/Second|Count\\/Second|None)$"
-                                        }
-                                    }
-                                }
-                            }
-                        }
-                    }
-                }
-            }
+                                            "type": "string",
+                                        },
+                                    },
+                                    "required": ["Name"],
+                                    "title": "MetricDefinition",
+                                    "type": "object",
+                                },
+                                "minItems": 1,
+                                "title": "MetricDefinitions",
+                                "type": "array",
+                            },
+                            "Namespace": {
+                                "$id": "#/properties/_aws/properties/CloudWatchMetrics/items/properties/Namespace",
+                                "examples": ["MyApp"],
+                                "minLength": 1,
+                                "pattern": "^(.*)$",
+                                "title": "CloudWatch " "Metrics " "Namespace",
+                                "type": "string",
+                            },
+                        },
+                        "required": ["Namespace", "Dimensions", "Metrics"],
+                        "title": "MetricDirective",
+                        "type": "object",
+                    },
+                    "title": "MetricDirectives",
+                    "type": "array",
+                },
+                "Timestamp": {
+                    "$id": "#/properties/_aws/properties/Timestamp",
+                    "examples": [1565375354953],
+                    "title": "The Timestamp " "Schema",
+                    "type": "integer",
+                },
+            },
+            "required": ["Timestamp", "CloudWatchMetrics"],
+            "title": "Metadata",
+            "type": "object",
         }
-    }
+    },
+    "required": ["_aws"],
+    "title": "Root Node",
+    "type": "object",
 }

aws_lambda_powertools/shared/constants.py

@@ -14,3 +14,7 @@
 CHALICE_LOCAL_ENV: str = "AWS_CHALICE_CLI_MODE"
 SERVICE_NAME_ENV: str = "POWERTOOLS_SERVICE_NAME"
 XRAY_TRACE_ID_ENV: str = "_X_AMZN_TRACE_ID"
+
+
+XRAY_SDK_MODULE = "aws_xray_sdk"
+XRAY_SDK_CORE_MODULE = "aws_xray_sdk.core"

aws_lambda_powertools/shared/lazy_import.py

@@ -0,0 +1,38 @@
+import importlib
+import types
+
+
+class LazyLoader(types.ModuleType):
+    """Lazily import a module, mainly to avoid pulling in large dependencies.
+
+    `contrib`, and `ffmpeg` are examples of modules that are large and not always
+    needed, and this allows them to only be loaded when they are used.
+
+    Note: Subclassing types.ModuleType allow us to correctly adhere with sys.modules, import system
+    """
+
+    def __init__(self, local_name, parent_module_globals, name):  # pylint: disable=super-on-old-class
+        self._local_name = local_name
+        self._parent_module_globals = parent_module_globals
+
+        super(LazyLoader, self).__init__(name)
+
+    def _load(self):
+        # Import the target module and insert it into the parent's namespace
+        module = importlib.import_module(self.__name__)
+        self._parent_module_globals[self._local_name] = module
+
+        # Update this object's dict so that if someone keeps a reference to the
+        #   LazyLoader, lookups are efficient (__getattr__ is only called on lookups
+        #   that fail).
+        self.__dict__.update(module.__dict__)
+
+        return module
+
+    def __getattr__(self, item):
+        module = self._load()
+        return getattr(module, item)
+
+    def __dir__(self):
+        module = self._load()
+        return dir(module)