[a 3/3] Announce maintenance in stable deployments (#5979)

lambdas/service/app.py

@@ -73,6 +73,9 @@
 from azul.logging import (
     configure_app_logging,
 )
+from azul.maintenance import (
+    MaintenanceController,
+)
 from azul.openapi import (
     application_json,
     format_description as fd,
@@ -328,6 +331,10 @@ def manifest_controller(self) -> ManifestController:
         return self._service_controller(ManifestController,
                                         manifest_url_func=manifest_url_func)
 
+    @cached_property
+    def maintenance_controller(self) -> MaintenanceController:
+        return MaintenanceController()
+
     def _service_controller(self, controller_cls: Type[C], **kwargs) -> C:
         file_url_func: FileUrlFunc = self.file_url
         return self._controller(controller_cls,
@@ -870,6 +877,20 @@ def get_integrations():
                     body=json.dumps(body))
 
 
+@app.route(
+    '/maintenance/schedule',
+    methods=['GET'],
+    cors=True
+#     TODO: spect-out
+)
+def get_maintenance_schedule():
+    #     # Calls the controller method that returns the appropriate object;
+    # Contents of the schedule (object['maintenance']['schedule']), labeled
+    # as pending (when planned for the future) and an active event to indicate
+    # ongoing maintenance (when one is active)
+    return app.maintenance_controller.get_maintenance_schedule()
+
+
 @app.route(
     '/index/catalogs',
     methods=['GET'],

src/azul/maintenance.py

@@ -7,13 +7,14 @@
     Enum,
     auto,
 )
+import json
 from operator import (
     attrgetter,
 )
 from typing import (
     Iterator,
-    Sequence,
     Self,
+    Sequence,
 )
 
 import attrs
@@ -23,14 +24,22 @@
 )
 
 from azul import (
+    CatalogName,
     JSON,
+    cached_property,
+    config,
     reject,
     require,
-    CatalogName,
 )
 from azul.collections import (
     adict,
 )
+from azul.deployment import (
+    aws,
+)
+from azul.service.storage_service import (
+    StorageObjectNotFound,
+)
 from azul.time import (
     format_dcp2_datetime,
     parse_dcp2_datetime,
@@ -165,6 +174,13 @@ def pending_events(self) -> list[MaintenanceEvent]:
                 return self.events[i:]
         return []
 
+    def past_events(self) -> list[MaintenanceEvent]:
+        return [
+            e
+            for e in self.events
+            if e.actual_end is not None and e.actual_start is not None
+        ]
+
     def active_event(self) -> MaintenanceEvent | None:
         return only(self._active_events())
 
@@ -193,6 +209,14 @@ def add_event(self, event: MaintenanceEvent):
             self.events = events
             raise
 
+    def adjust_event(self, additional_duration: timedelta):
+        event = self.active_event()
+        reject(event is None, 'No active event')
+        event.planned_duration += additional_duration
+        self._heal(event, iter(self.pending_events()))
+        assert self.active_event() is not None
+        return event
+
     def cancel_event(self, index: int) -> MaintenanceEvent:
         event = self.pending_events()[index]
         self.events.remove(event)
@@ -224,3 +248,148 @@ def _heal(self,
                 next_event.planned_start = event.end
             event = next_event
         self.validate()
+
+
+class MaintenanceService:
+
+    @property
+    def bucket(self):
+        return aws.shared_bucket
+
+    @property
+    def object_key(self):
+        return f'azul/{config.deployment_stage}/azul.json'
+
+    @cached_property
+    def client(self):
+        return aws.s3
+
+    @property
+    def _create_empty_schedule(self):
+        schedule = {
+            "maintenance": {
+                "schedule": {
+                    "events": []
+                }
+            }
+        }
+        self.client.put_object(Bucket=self.bucket,
+                               Key=self.object_key,
+                               Body=json.dumps(schedule).encode())
+        return schedule
+
+    @property
+    def _get_schedule(self) -> JSON:
+        try:
+            response = self.client.get_object(Bucket=self.bucket,
+                                              Key=self.object_key)
+        except self.client.exceptions.NoSuchKey:
+            raise StorageObjectNotFound
+        else:
+            return json.loads(response['Body'].read())
+
+    def get_maintenance_schedule(self,
+                                 all: bool = False,
+                                 schedule: MaintenanceSchedule | None = None) -> JSON:
+        """
+        Display's the schedule, of active and pending events
+        """
+        if schedule is None:
+            schedule = self.get_schedule
+        active_event = schedule.active_event()
+        active_event = {} if active_event is None else {'active': active_event.to_json()}
+        events = schedule.pending_events()
+        if events is not None:
+            events = {'pending': list(pe.to_json() for pe in events)}
+        else:
+            events = {}
+        completed = {}
+        if all:
+            completed = {'completed': [pe.to_json() for pe in schedule.past_events()]}
+
+        events = active_event | events | completed
+        return events
+
+    @property
+    def get_schedule(self) -> MaintenanceSchedule:
+        schedule = self._get_schedule
+        schedule = MaintenanceSchedule.from_json(schedule['maintenance']['schedule'])
+        schedule.validate()
+        return schedule
+
+    def put_schedule(self, schedule: MaintenanceSchedule):
+        schedule = schedule.to_json()
+        return self.client.put_object(Bucket=self.bucket,
+                                      Key=self.object_key,
+                                      Body=json.dumps({
+                                          "maintenance": {
+                                              "schedule": schedule
+                                          }
+                                      }).encode())
+
+    def provision_event(self,
+                        planned_start: str,
+                        planned_duration: int,
+                        description: str,
+                        partial: list[str] | None = None,
+                        degraded: list[str] | None = None,
+                        unavailable: list[str] | None = None) -> MaintenanceEvent:
+        partial = [{
+            'kind': 'partial_responses',
+            'affected_catalogs': partial
+        }] if partial is not None else []
+        degraded = [{
+            'kind': 'degraded_performance',
+            'affected_catalogs': degraded
+        }] if degraded is not None else []
+        unavailable = [{
+            'kind': 'service_unavailable',
+            'affected_catalogs': unavailable
+        }] if unavailable is not None else []
+        impacts = [*partial, *degraded, *unavailable]
+        return MaintenanceEvent.from_json({
+            'planned_start': planned_start,
+            'planned_duration': planned_duration,
+            'description': description,
+            'impacts': impacts
+        })
+
+    def add_event(self, event: MaintenanceEvent) -> JSON:
+        schedule = self.get_schedule
+        schedule.add_event(event)
+        self.put_schedule(schedule)
+        return self.get_maintenance_schedule(schedule=schedule)
+
+    def cancel_event(self, index: int) -> JSON:
+        schedule = self.get_schedule
+        event = schedule.cancel_event(index)
+        self.put_schedule(schedule)
+        return event.to_json()
+
+    def start_event(self) -> JSON:
+        schedule = self.get_schedule
+        event = schedule.start_event()
+        self.put_schedule(schedule)
+        return event.to_json()
+
+    def end_event(self) -> JSON:
+        schedule = self.get_schedule
+        event = schedule.end_event()
+        self.put_schedule(schedule)
+        return event.to_json()
+
+    def adjust_event(self, additional_duration: timedelta) -> JSON:
+        schedule = self.get_schedule
+        event = schedule.adjust_event(additional_duration)
+        self.put_schedule(schedule)
+        return event.to_json()
+
+
+class MaintenanceController:
+
+    @cached_property
+    def service(self) -> MaintenanceService:
+        return MaintenanceService()
+
+    def get_maintenance_schedule(self):
+        return self.service.get_maintenance_schedule()