diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 68b3573..ad1f4e4 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -9,8 +9,10 @@ RUN apt-get update \ emacs \ exa \ fd-find \ + fzf \ git \ iproute2 \ + iputils-ping \ less \ libmagic-dev \ libsodium-dev \ @@ -22,6 +24,7 @@ RUN apt-get update \ npm \ openssh-client \ procps \ + ripgrep \ sudo \ tldr \ unzip \ @@ -35,4 +38,6 @@ RUN pip install --upgrade pip COPY .devcontainer/scripts/notify-dev-entrypoint.sh /usr/local/bin/ +ENV SHELL /bin/zsh + EXPOSE 7000 diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 9b00562..d579999 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,9 +1,9 @@ { "name": "notification-document-download-api", - "build": { - "dockerfile": "Dockerfile", - "context": ".." - }, + "dockerComposeFile": "docker-compose.yml", + "service": "notify-dd-api", + "workspaceFolder": "/workspace", + "shutdownAction": "stopCompose", "remoteEnv": { "PATH": "/home/vscode/.local/bin:${containerEnv:PATH}" // give our installed Python modules precedence }, @@ -18,17 +18,26 @@ },"python.pythonPath": "/usr/local/bin/python" }, "extensions": [ - "ms-python.python", + "charliermarsh.ruff", + "donjayamanne.python-extension-pack", "eamodio.gitlens", + "fill-labs.dependi", "GitHub.copilot", + "github.copilot-chat", + "github.vscode-pull-request-github", + "kaiwood.center-editor-window", + "matangover.mypy", + "ms-python.python", + "ms-python.vscode-pylance", "tamasfe.even-better-toml", - "charliermarsh.ruff" + "timonwong.shellcheck", + "visualstudioexptteam.vscodeintellicode", + "vsliveshare.vsliveshare", + "wenfangdu.jump", + "yzhang.markdown-all-in-one" ] } }, - "forwardPorts": [ - 7000 - ], "postCreateCommand": "notify-dev-entrypoint.sh", "remoteUser": "vscode" } \ No newline at end of file diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml new file mode 100644 index 0000000..a1db430 --- /dev/null +++ b/.devcontainer/docker-compose.yml @@ -0,0 +1,32 @@ +version: '3' +services: + # Update this to the name of the service you want to work with in your docker-compose.yml file + notify-dd-api: + build: + context: .. + dockerfile: .devcontainer/Dockerfile + # If you want add a non-root user to your Dockerfile, you can use the "remoteUser" + # property in devcontainer.json to cause VS Code its sub-processes (terminals, tasks, + # debugging) to execute as the user. Uncomment the next line if you want the entire + # container to run as this user instead. Note that, on Linux, you may need to + # ensure the UID and GID of the container user you create matches your local user. + # See https://aka.ms/vscode-remote/containers/non-root for details. + user: vscode + + volumes: + # Update this to wherever you want VS Code to mount the folder of your project + - ..:/workspace:cached + + # Uncomment the next line to use Docker from inside the container. See https://aka.ms/vscode-remote/samples/docker-from-docker-compose for details. + # - /var/run/docker.sock:/var/run/docker.sock + + # Overrides default command so things don't shut down after the process ends. + command: sleep infinity + expose: + - "7000" + networks: + - notify-network + +networks: + notify-network: + external: true diff --git a/.devcontainer/scripts/notify-dev-entrypoint.sh b/.devcontainer/scripts/notify-dev-entrypoint.sh index 285f881..8de09f0 100755 --- a/.devcontainer/scripts/notify-dev-entrypoint.sh +++ b/.devcontainer/scripts/notify-dev-entrypoint.sh @@ -17,6 +17,9 @@ echo -e "alias ll='exa -alh@ --git'" >> ~/.zshrc echo -e "alias lt='exa -al -T -L 2'" >> ~/.zshrc echo -e "alias poe='poetry run poe'" >> ~/.zshrc +echo -e "# fzf key bindings and completion" >> ~/.zshrc +echo -e "source /usr/share/doc/fzf/examples/key-bindings.zsh" >> ~/.zshrc +echo -e "source /usr/share/doc/fzf/examples/completion.zsh" >> ~/.zshrc # Poetry autocomplete echo -e "fpath+=/.zfunc" >> ~/.zshrc diff --git a/gunicorn_config.py b/gunicorn_config.py index 82979b6..647b343 100644 --- a/gunicorn_config.py +++ b/gunicorn_config.py @@ -1,7 +1,13 @@ import os import sys +import time import traceback +import newrelic.agent # See https://bit.ly/2xBVKBH + +environment = os.environ.get("NOTIFY_ENVIRONMENT") +newrelic.agent.initialize(environment=environment) # noqa: E402 + workers = 4 worker_class = "gevent" worker_connections = 256 @@ -13,10 +19,40 @@ # to be larger than the idle timeout configured for the load balancer. # > By default, Elastic Load Balancing sets the idle timeout value for your load balancer to 60 seconds. # https://docs.aws.amazon.com/elasticloadbalancing/latest/application/application-load-balancers.html#connection-idle-timeout -on_aws = os.environ.get("NOTIFY_ENVIRONMENT", "") in ["production", "staging", "scratch", "dev"] +on_aws = environment in ["production", "staging", "scratch", "dev"] if on_aws: + # To avoid load balancers reporting errors on shutdown instances, see AWS doc + # > We also recommend that you configure the idle timeout of your application + # > to be larger than the idle timeout configured for the load balancer. + # > By default, Elastic Load Balancing sets the idle timeout value for your load balancer to 60 seconds. + # https://docs.aws.amazon.com/elasticloadbalancing/latest/application/application-load-balancers.html#connection-idle-timeout keepalive = 75 + # The default graceful timeout period for Kubernetes is 30 seconds, so + # make sure that the timeouts defined here are less than the configured + # Kubernetes timeout. This ensures that the gunicorn worker will exit + # before the Kubernetes pod is terminated. This is important because + # Kubernetes will send a SIGKILL to the pod if it does not terminate + # within the grace period. If the worker is still processing requests + # when it receives the SIGKILL, it will be terminated abruptly and + # will not be able to finish processing the request. This can lead to + # 502 errors being returned to the client. + # + # Also, some libraries such as NewRelic might need some time to finish + # initialization before the worker can start processing requests. The + # timeout values should consider these factors. + # + # Gunicorn config: + # https://docs.gunicorn.org/en/stable/settings.html#graceful-timeout + # + # Kubernetes config: + # https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/ + graceful_timeout = 85 + timeout = 90 + +# Start timer for total running time +start_time = time.time() + def on_starting(server): server.log.info("Starting Document Download API") @@ -29,7 +65,9 @@ def worker_abort(worker): def on_exit(server): + elapsed_time = time.time() - start_time server.log.info("Stopping Document Download API") + server.log.info("Total gunicorn API running time: {:.2f} seconds".format(elapsed_time)) def worker_int(worker): diff --git a/mypy.ini b/mypy.ini index 43e789d..f96bf31 100644 --- a/mypy.ini +++ b/mypy.ini @@ -14,4 +14,7 @@ ignore_missing_imports = True ignore_missing_imports = True [mypy-aws_xray_sdk.*] +ignore_missing_imports = True + +[mypy-newrelic.*] ignore_missing_imports = True \ No newline at end of file diff --git a/newrelic.ini b/newrelic.ini new file mode 100644 index 0000000..f546e92 --- /dev/null +++ b/newrelic.ini @@ -0,0 +1,221 @@ +# --------------------------------------------------------------------------- + +# +# This file configures the New Relic Python Agent. +# +# The path to the configuration file should be supplied to the function +# newrelic.agent.initialize() when the agent is being initialized. +# +# The configuration file follows a structure similar to what you would +# find for Microsoft Windows INI files. For further information on the +# configuration file format see the Python ConfigParser documentation at: +# +# http://docs.python.org/library/configparser.html +# +# For further discussion on the behaviour of the Python agent that can +# be configured via this configuration file see: +# +# http://newrelic.com/docs/python/python-agent-configuration +# + +# --------------------------------------------------------------------------- + +# Here are the settings that are common to all environments. + +[newrelic] + +# You must specify the license key associated with your New +# Relic account. This key binds the Python Agent's data to your +# account in the New Relic service. +# license_key = *** REPLACE ME *** + +# The application name. Set this to be the name of your +# application as you would like it to show up in New Relic UI. +# The UI will then auto-map instances of your application into a +# entry on your home dashboard page. +# app_name = Python Application + +# When "true", the agent collects performance data about your +# application and reports this data to the New Relic UI at +# newrelic.com. This global switch is normally overridden for +# each environment below. +# monitor_mode = true + +# Sets the name of a file to log agent messages to. Useful for +# debugging any issues with the agent. This is not set by +# default as it is not known in advance what user your web +# application processes will run as and where they have +# permission to write to. Whatever you set this to you must +# ensure that the permissions for the containing directory and +# the file itself are correct, and that the user that your web +# application runs as can write to the file. If not able to +# write out a log file, it is also possible to say "stderr" and +# output to standard error output. This would normally result in +# output appearing in your web server log. +#log_file = /tmp/newrelic-python-agent.log + +# Sets the level of detail of messages sent to the log file, if +# a log file location has been provided. Possible values, in +# increasing order of detail, are: "critical", "error", "warning", +# "info" and "debug". When reporting any agent issues to New +# Relic technical support, the most useful setting for the +# support engineers is "debug". However, this can generate a lot +# of information very quickly, so it is best not to keep the +# agent at this level for longer than it takes to reproduce the +# problem you are experiencing. +# log_level = info + +# The Python Agent communicates with the New Relic service using +# SSL by default. Note that this does result in an increase in +# CPU overhead, over and above what would occur for a non SSL +# connection, to perform the encryption involved in the SSL +# communication. This work is though done in a distinct thread +# to those handling your web requests, so it should not impact +# response times. You can if you wish revert to using a non SSL +# connection, but this will result in information being sent +# over a plain socket connection and will not be as secure. +# ssl = true + +# High Security Mode enforces certain security settings, and +# prevents them from being overridden, so that no sensitive data +# is sent to New Relic. Enabling High Security Mode means that +# SSL is turned on, request parameters are not collected, and SQL +# can not be sent to New Relic in its raw form. To activate High +# Security Mode, it must be set to 'true' in this local .ini +# configuration file AND be set to 'true' in the server-side +# configuration in the New Relic user interface. For details, see +# https://docs.newrelic.com/docs/subscriptions/high-security +# high_security = false + +# The Python Agent will attempt to connect directly to the New +# Relic service. If there is an intermediate firewall between +# your host and the New Relic service that requires you to use a +# HTTP proxy, then you should set both the "proxy_host" and +# "proxy_port" settings to the required values for the HTTP +# proxy. The "proxy_user" and "proxy_pass" settings should +# additionally be set if proxy authentication is implemented by +# the HTTP proxy. The "proxy_scheme" setting dictates what +# protocol scheme is used in talking to the HTTP proxy. This +# would normally always be set as "http" which will result in the +# agent then using a SSL tunnel through the HTTP proxy for end to +# end encryption. +# proxy_scheme = http +# proxy_host = hostname +# proxy_port = 8080 +# proxy_user = +# proxy_pass = + +# Capturing request parameters is off by default. To enable the +# capturing of request parameters, first ensure that the setting +# "attributes.enabled" is set to "true" (the default value), and +# then add "request.parameters.*" to the "attributes.include" +# setting. For details about attributes configuration, please +# consult the documentation. +# attributes.include = request.parameters.* + +# The transaction tracer captures deep information about slow +# transactions and sends this to the UI on a periodic basis. The +# transaction tracer is enabled by default. Set this to "false" +# to turn it off. +# transaction_tracer.enabled = true + +# Threshold in seconds for when to collect a transaction trace. +# When the response time of a controller action exceeds this +# threshold, a transaction trace will be recorded and sent to +# the UI. Valid values are any positive float value, or (default) +# "apdex_f", which will use the threshold for a dissatisfying +# Apdex controller action - four times the Apdex T value. +# transaction_tracer.transaction_threshold = apdex_f + +# When the transaction tracer is on, SQL statements can +# optionally be recorded. The recorder has three modes, "off" +# which sends no SQL, "raw" which sends the SQL statement in its +# original form, and "obfuscated", which strips out numeric and +# string literals. +# transaction_tracer.record_sql = obfuscated + +# Threshold in seconds for when to collect stack trace for a SQL +# call. In other words, when SQL statements exceed this +# threshold, then capture and send to the UI the current stack +# trace. This is helpful for pinpointing where long SQL calls +# originate from in an application. +# transaction_tracer.stack_trace_threshold = 0.5 + +# Determines whether the agent will capture query plans for slow +# SQL queries. Only supported in MySQL and PostgreSQL. Set this +# to "false" to turn it off. +# transaction_tracer.explain_enabled = true + +# Threshold for query execution time below which query plans +# will not not be captured. Relevant only when "explain_enabled" +# is true. +# transaction_tracer.explain_threshold = 0.5 + +# Space separated list of function or method names in form +# 'module:function' or 'module:class.function' for which +# additional function timing instrumentation will be added. +# transaction_tracer.function_trace = + +# The error collector captures information about uncaught +# exceptions or logged exceptions and sends them to UI for +# viewing. The error collector is enabled by default. Set this +# to "false" to turn it off. +error_collector.enabled = true + +# To stop specific errors from reporting to the UI, set this to +# a space separated list of the Python exception type names to +# ignore. The exception name should be of the form 'module:class'. +error_collector.ignore_errors = app.v2.errors:BadRequestError jsonschema.exceptions:ValidationError + +# Browser monitoring is the Real User Monitoring feature of the UI. +# For those Python web frameworks that are supported, this +# setting enables the auto-insertion of the browser monitoring +# JavaScript fragments. +# browser_monitoring.auto_instrument = true + +# A thread profiling session can be scheduled via the UI when +# this option is enabled. The thread profiler will periodically +# capture a snapshot of the call stack for each active thread in +# the application to construct a statistically representative +# call tree. +# thread_profiler.enabled = true + +# --------------------------------------------------------------------------- + +# +# The application environments. These are specific settings which +# override the common environment settings. The settings related to a +# specific environment will be used when the environment argument to the +# newrelic.agent.initialize() function has been defined to be either +# "development", "test", "staging" or "production". + +# If this setting is enabled, it will capture package and version +# information on startup of the agent that is displayed in the APM +# environment tab. +# In applications that have a large number of packages, having this +# setting enabled may cause a CPU spike as it captures all the package +# and version information. It is recommended in those cases to disable +# this setting. +# Disabling this setting will disable the ability to detect vulnerabilities in outdated packages. +package_reporting.enabled = false + +[newrelic:development] +# monitor_mode = false +log_level = debug + +[newrelic:staging] +# app_name = Python Application (Staging) +# monitor_mode = true +log_level = debug + +[newrelic:production] +# monitor_mode = true + +[newrelic:scratch] +# monitor_mode = false + +[newrelic:dev] +# monitor_mode = false +log_level = debug + +# --------------------------------------------------------------------------- diff --git a/poetry.lock b/poetry.lock index c580c48..b24533e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1053,6 +1053,47 @@ files = [ {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, ] +[[package]] +name = "newrelic" +version = "10.3.0" +description = "New Relic Python Agent" +optional = false +python-versions = ">=3.7" +files = [ + {file = "newrelic-10.3.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cc803e30b72e2afe7b759a79fbadbed557aa5d51ef36f26d68e9d0aeb156a7f"}, + {file = "newrelic-10.3.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c2fbdbbb4de1cb305b2c0155639fe74bf6925e54cc014a07176f46fba396cb03"}, + {file = "newrelic-10.3.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4a5f11d99621495c7edd121c96d8a71f9af4c0036de8546bc3aac94b6183a3f0"}, + {file = "newrelic-10.3.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:b1caebc9854614435acc28b95efa1b8b2a30eb9ac8778a96ff65ca619ce7e832"}, + {file = "newrelic-10.3.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:667e91dc56a99358d08d93e9372f5889b3ba0650d0911718baef68ea607419d1"}, + {file = "newrelic-10.3.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5dc21702764dffb8d55349fa75ad3d37a9ee53ab4ade3cc8cb347bdf0ba36268"}, + {file = "newrelic-10.3.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:080e764b50be8522df204d5f8fb26bdfa64945da85fe2786bc3051318fa77188"}, + {file = "newrelic-10.3.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e23dfbc55ba7baa3b65d4f2739ba328e15466a4d4767a85fe732082c423b88e0"}, + {file = "newrelic-10.3.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:30e272c8aa1d812bfe4609db48aa19d3322627263906f5a1cbba267625d0b40f"}, + {file = "newrelic-10.3.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:31face7377b9083b0e889ea69d2a7d95a79dfdbfcf1bf92256e03d8863c92a68"}, + {file = "newrelic-10.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:9b459cdb67f3dc64ac16420a951f3c6c2e75b8f4bc68b2bfb3f395e83d611a61"}, + {file = "newrelic-10.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a9085b07ec7e43c079137b515c007542b8f7ecda15504fc7e863fd235481bd87"}, + {file = "newrelic-10.3.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:92c3da1480db8cd2c6f17e7b7bd09bfcb4a755caab823c9770185a97446f693d"}, + {file = "newrelic-10.3.0-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f91ad51693f98be5dfd7d115588059e4f2371628aef7412ca830bfdb001d588"}, + {file = "newrelic-10.3.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:8847709eeda25ce9e577f2d593f0ca12e3de04d3e816e6fda29fa10a4e211900"}, + {file = "newrelic-10.3.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:0f587779f03b0dc1131e2067221042ce796551bbb73a361046f25102763d1718"}, + {file = "newrelic-10.3.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb741158af8b84638187d12cce54263e4f14eb031778eea03a45cedb92b92678"}, + {file = "newrelic-10.3.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5e8e60d72a8c9d31af96ada5bc20cecb9331079061c5cd0e63ebeed514526678"}, + {file = "newrelic-10.3.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7af34b3d833f15b4deccd8a0909231ba9adb3e86106630e1a1003d8fc57e0710"}, + {file = "newrelic-10.3.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bbd6f9f67fc17337719ef4a5abd5c988824e7b824b598dcfebe54d482cca3ab8"}, + {file = "newrelic-10.3.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4315353409952f89c84f2cf99de50aadd793c18ee112701319e96f79e166feec"}, + {file = "newrelic-10.3.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1fb7c6fcce05c2b7ed00801c10f6dc22efac68debe54ca0c2ff0b1b0e1ad81f8"}, + {file = "newrelic-10.3.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:3c0a41f83a003d87a31a3e290f8c54d680b1590baa86a62ffb60f576ae4bf951"}, + {file = "newrelic-10.3.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:17cbc679ec01bdd01092c159dc76a5c3abe91a5fd713dcf9429103162792cd5b"}, + {file = "newrelic-10.3.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7a7f9566282a6c0b1d733dc014b2123577c16a65be4814ea48af46d2c4de2a57"}, + {file = "newrelic-10.3.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5aef5453250fd570af30be7a384b1c85fa5b92ad08a748f3266ea3540f1f06eb"}, + {file = "newrelic-10.3.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:ee4ff34ab06f0df4ba823dd368fb79789bcdabe4ffa3c0b880a53a65f610f852"}, + {file = "newrelic-10.3.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a9453656125daf9b2ece9e49d6d99b064dbb9e85adc1ff6fadd4bdb89ab2f4cd"}, + {file = "newrelic-10.3.0.tar.gz", hash = "sha256:26040d0b707c30dba2c93b3122e87b39d98cc4910bcbb518bf7ab7c8ab62a5b8"}, +] + +[package.extras] +infinite-tracing = ["grpcio", "protobuf"] + [[package]] name = "notifications-utils" version = "52.2.7" @@ -1885,4 +1926,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] [metadata] lock-version = "2.0" python-versions = "~3.10.9" -content-hash = "f6cc644d0ebca00b290113a33a531a1470e71ac8adfe44f1fd7ef677f07d0964" +content-hash = "042b92706fa5cda1a2a898b9c1a4ef4e3ac481c5407ab761c09bc437095d97f1" diff --git a/pyproject.toml b/pyproject.toml index 5ee81fb..d1a50a5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,7 @@ Flask-Env = "2.0.0" gevent = "24.2.1" # PaaS gunicorn = "22.0.0" - +newrelic = "10.3.0" notifications-utils = { git = "https://github.com/cds-snc/notifier-utils.git", tag = "52.2.7"} pycryptodome = "*"