Adds logging in a format that datadog can read (#46)

* Update to send logs to a file and to datadog

* Adding additional environment variable for ddog

.gitignore

@@ -11,3 +11,7 @@ __pycache__
 
 # config
 config/local.json
+
+# logs
+gobble.log
+s3_upload.log

devops/deploy.sh

@@ -10,11 +10,15 @@ export AWS_PAGER=""
 STACK_NAME="gobble"
 
 # Ensure required secrets are set
-if [[ -z "$DD_API_KEY"  ]]; then
+if [[ -z "$DD_API_KEY" ]]; then
     echo "Must provide DD_API_KEY in environment to deploy" 1>&2
     exit 1
 fi
 
+# Identify the version and commit of the current deploy
+export GIT_SHA=`git rev-parse HEAD`
+echo "Deploying version $GIT_SHA"
+
 echo "Deploying Gobble..."
 echo "View stack log here: https://$AWS_REGION.console.aws.amazon.com/cloudformation/home?region=$AWS_REGION"
 

devops/playbook.yml

@@ -65,6 +65,9 @@
     datadog_api_key: "{{ lookup('env', 'DD_API_KEY') }}"
     datadog_site: "datadoghq.com"
     datadog_config:
+      tags:
+        - "git.repository_url:github.com/transitmatters/gobble"
+        - "git.commit.sha:{{ lookup('env', 'GIT_SHA') }}"
       apm_config:
         enabled: true
       process_config:
@@ -73,4 +76,17 @@
     network_config:
       enabled: true
     system_probe_config:
-      enable_oom_kill: true
+      enable_oom_kill: true
+    datadog_checks:
+      python:
+        logs:
+          - type: file
+            path: "/home/ubuntu/gobble/gobble.log"
+            service: "gobble"
+            source: python
+            sourcecategory: sourcecode
+          - type: file
+            path: "/home/ubuntu/gobble/s3_upload.log"
+            service: "gobble"
+            source: python
+            sourcecategory: sourcecode

devops/systemd.conf

@@ -10,6 +10,8 @@ WorkingDirectory=/home/ubuntu/gobble
 Environment=DD_SERVICE="gobble"
 Environment=DD_ENV="prod"
 Environment=DD_PROFILING_ENABLED="true"
+Environment=DD_LOGS_INJECTION="true"
+Environment=DD_GIT_REPOSITORY_URL="github.com/transitmatters/gobble"
 ExecStart=/home/ubuntu/.local/bin/poetry run ddtrace-run python3 src/gobble.py
 Restart=on-failure
 RestartSec=5s

pyproject.toml

@@ -13,6 +13,7 @@ sseclient-py = "1.8.0"
 requests = "2.31.0"
 boto3 = "^1.33.11"
 ddtrace = "^2.3.1"
+python-json-logger = "^2.0.7"
 
 [tool.poetry.dev-dependencies]
 pip = ">=23.1.0"

src/gobble.py

@@ -9,11 +9,12 @@
 from constants import ROUTES_BUS, ROUTES_CR
 from config import CONFIG
 from event import process_event
+from logger import set_up_logging
 import gtfs
 import disk
 import util
 
-logging.basicConfig(level=logging.INFO)
+logging.basicConfig(level=logging.INFO, filename="gobble.log")
 tracer.enabled = CONFIG["DATADOG_TRACE_ENABLED"]
 
 API_KEY = CONFIG["mbta"]["v3_api_key"]
@@ -41,7 +42,7 @@ def main():
 
 
 if __name__ == "__main__":
-    logger = logging.getLogger(__file__)
+    logger = set_up_logging(__file__)
     main()
 else:
-    logger = logging.getLogger(__name__)
+    logger = set_up_logging(__name__)

src/logger.py

@@ -0,0 +1,14 @@
+import logging
+from pythonjsonlogger import jsonlogger
+
+
+def set_up_logging(name: str):
+    logger = logging.getLogger(name)
+
+    # sets up logger to handle stack traces and other multi-line logs in a way that Datadog can parse
+    logHandler = logging.StreamHandler()
+    formatter = jsonlogger.JsonFormatter()
+    logHandler.setFormatter(formatter)
+    logger.addHandler(logHandler)
+
+    return logger

src/s3_upload.py

@@ -10,9 +10,10 @@
 
 from config import CONFIG
 from disk import DATA_DIR
+from logger import set_up_logging
 from util import EASTERN_TIME, service_date
 
-logging.basicConfig(level=logging.INFO)
+logging.basicConfig(level=logging.INFO, filename="s3_upload.log")
 tracer.enabled = CONFIG["DATADOG_TRACE_ENABLED"]
 
 s3 = boto3.client("s3")
@@ -63,7 +64,7 @@ def upload_todays_events_to_s3():
 
 
 if __name__ == "__main__":
-    logger = logging.getLogger(__file__)
+    logger = set_up_logging(__file__)
     upload_todays_events_to_s3()
 else:
-    logger = logging.getLogger(__name__)
+    logger = set_up_logging(__name__)