Redis memory usage issues (#1440)

* Setting the default elasticache instance size in production to high_mem_large. * Created a default alert for elasticache to notify when memory utilization is over 90 percent. * Setting the desired worker count to five.
mitodl · Apr 5, 2023 · 0f504a8 · 0f504a8
1 parent f572916
commit 0f504a8
Show file tree

Hide file tree

Showing 3 changed files with 15 additions and 5 deletions.
diff --git a/src/ol_infrastructure/applications/edxapp/Pulumi.applications.edxapp.mitx.Production.yaml b/src/ol_infrastructure/applications/edxapp/Pulumi.applications.edxapp.mitx.Production.yaml
@@ -5,10 +5,10 @@ config:
   aws:region: us-east-1
   consul:address: https://consul-mitx-production.odl.mit.edu
   edxapp:business_unit: residential
+  edxapp:db_max_storage_gb: "1000"
   edxapp:db_password:
     secure: v1:vkG4n/X+C+SBaMf7:W/zi1Bxqkab0Hx1HAAiVhZuj/8WOJ7v94lxfERa5YIF5dD037uuULgPxIt+3zzHVur26pBcVIk4=
-  edxapp:db_storage_gb: 500
-  edxapp:db_max_storage_gb: 1000
+  edxapp:db_storage_gb: "500"
   edxapp:dns_zone: mitx
   edxapp:domains:
     lms: lms.mitx.mit.edu
@@ -33,7 +33,7 @@ config:
   edxapp:web_instance_type: general_purpose_xlarge
   edxapp:web_node_capacity: "5"
   edxapp:worker_disk_size: "50"
-  edxapp:worker_node_capacity: "2"
+  edxapp:worker_node_capacity: "5"
   mongodb:atlas_project_id: 61b728704656cb4747e589a0
   vault:address: https://vault-production.odl.mit.edu
   vault_server:env_namespace: operations.production
diff --git a/src/ol_infrastructure/components/aws/cache.py b/src/ol_infrastructure/components/aws/cache.py
@@ -340,7 +340,17 @@ def _get_default_redis_monitoring_profile(self, profile_name: str):
                 "metric_name": "EngineCPUUtilization",
                 "threshold": 50,  # percent
                 "unit": "Percent",
-            }
+            },
+            "DatabaseMemoryUsagePercentage": {
+                "comparison_operator": "GreaterThanThreshold",
+                "description": "ElastiCache - High memory utilization by the Redis engine.",  # noqa: E501
+                "datapoints_to_alarm": 2,
+                "level": "warning",
+                "period": 300,  # 5 minutes
+                "evaluation_periods": 2,  # 10 Minutes
+                "metric_name": "DatabaseMemoryUsagePercentage",
+                "threshold": 90,  # percent
+            },
         }
 
         monitoring_profiles: dict[str, dict] = {

diff --git a/src/ol_infrastructure/lib/stack_defaults.py b/src/ol_infrastructure/lib/stack_defaults.py
@@ -12,7 +12,7 @@
         "read_replica": OLReplicaDBConfig(),
         "monitoring_profile_name": "production",
     },
-    "redis": {"instance_type": CacheInstanceTypes.large},
+    "redis": {"instance_type": CacheInstanceTypes.high_mem_large},
 }
 
 qa_defaults = {