Skip to content

Commit

Permalink
Revert "refactor watchdog, pt.2"
Browse files Browse the repository at this point in the history
This reverts commit ace4db1.
  • Loading branch information
CollectiveUnicorn committed Sep 19, 2024
1 parent ace4db1 commit 18f4d4c
Showing 1 changed file with 12 additions and 43 deletions.
55 changes: 12 additions & 43 deletions src/leapfrogai_api/utils/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
import os
import toml
import yaml
from threading import Lock
from watchdog.observers import Observer
from watchdog.events import FileSystemEventHandler

Expand Down Expand Up @@ -34,17 +33,12 @@ def process(self, event):
return

filename = os.path.basename(event.src_path)
logger.debug(f"Processing event '{event.event_type}' for file '{filename}'")

# Check if the file matches the config filename or pattern
if fnmatch.fnmatch(filename, self.config.filename):
if event.event_type == "deleted":
logger.info(f"Detected deletion of config file '{filename}'")
self.config.remove_model_by_config(filename)
else:
logger.info(
f"Detected modification/creation of config file '{filename}'"
)
self.config.load_config_file(self.config.directory, filename)


Expand All @@ -59,7 +53,6 @@ def __init__(
self.config_sources = config_sources
self.directory = "."
self.filename = "config.yaml"
self.lock = Lock()

def __str__(self):
return f"Models: {self.models}"
Expand Down Expand Up @@ -91,8 +84,6 @@ async def watch_and_load_configs(self, directory=".", filename="config.yaml"):
try:
while True:
await asyncio.sleep(1)
# Periodically check for stale configs
await self.check_for_deleted_configs()
except (KeyboardInterrupt, asyncio.CancelledError):
# Stop the observer if the script is interrupted
observer.stop()
Expand All @@ -102,9 +93,9 @@ async def watch_and_load_configs(self, directory=".", filename="config.yaml"):
observer.join()

async def clear_all_models(self):
with self.lock:
self.models = {}
self.config_sources = {}
# Reset the model config on shutdown (so old model configs don't get cached)
self.models = {}
self.config_sources = {}
logger.info("All models have been removed")

def load_config_file(self, directory: str, config_file: str):
Expand All @@ -124,8 +115,7 @@ def load_config_file(self, directory: str, config_file: str):
return

# Parse the object into our config
with self.lock:
self.parse_models(loaded_artifact, config_file)
self.parse_models(loaded_artifact, config_file)

logger.info(f"Loaded artifact at {config_path}")
except Exception as e:
Expand All @@ -147,8 +137,7 @@ def load_all_configs(self, directory="", filename="config.yaml"):
self.load_config_file(directory=dir_path, config_file=file_path)

def get_model_backend(self, model: str) -> Model | None:
with self.lock:
return self.models.get(model)
return self.models.get(model)

def parse_models(self, loaded_artifact, config_file):
for m in loaded_artifact.get("models", []):
Expand All @@ -159,30 +148,10 @@ def parse_models(self, loaded_artifact, config_file):
logger.info(f"Added {m['name']} to model config")

def remove_model_by_config(self, config_file):
with self.lock:
model_names = self.config_sources.get(config_file, [])
for model_name in model_names:
self.models.pop(model_name, None)
logger.info(f"Removed {model_name} from model config")

# Clear config once all corresponding models are deleted
self.config_sources.pop(config_file, None)

async def check_for_deleted_configs(self):
# Get the list of config files currently present
current_config_files = set(
os.path.basename(f)
for f in glob.glob(os.path.join(self.directory, self.filename))
)
# Get the set of config files known to the system
with self.lock:
known_config_files = set(self.config_sources.keys())

# Find config files that have been deleted
deleted_config_files = known_config_files - current_config_files

for config_file in deleted_config_files:
logger.info(
f"Config file '{config_file}' no longer exists. Removing associated models."
)
self.remove_model_by_config(config_file)
model_names = self.config_sources.get(config_file, [])
for model_name in model_names:
self.models.pop(model_name, None)
logger.info(f"Removed {model_name} from model config")

# Clear config once all corresponding models are deleted
self.config_sources.pop(config_file, None)

0 comments on commit 18f4d4c

Please sign in to comment.