feat: allow configuring validators from envvar

.env.example

@@ -5,12 +5,10 @@ DBUSER              = 'postgres'
 DBPASSWORD          = 'postgres'
 DBPORT              = '5432'
 
-# The total number of validators the simulator
-# will have access to at the begining
-TOTALVALIDATORS     = 10
-# The number of validators that will be used
-# to form consensus
-NUMVALIDATORS       = 4
+# JSON array of initial validators to be created on startup.
+# Example: [{"stake": 100, "provider": "openai", "model": "gpt-4o", "amount": 2}, {"stake": 200, "provider": "anthropic", "model": "claude-3-haiku-20240307", "amount": 1}]
+VALIDATORS_CONFIG_JSON = ''
+
 LOGCONFIG          = 'dev'  # dev/prod
 FLASK_LOG_LEVEL    = 'ERROR'  # DEBUG/INFO/WARNING/ERROR/CRITICAL
 DISABLE_INFO_LOGS_ENDPOINTS = '["ping", "eth_getTransactionByHash","gen_getContractSchemaForCode","gen_getContractSchema"]'
@@ -25,7 +23,7 @@ RPCDEBUGPORT        = '4678'
 # GenVM server details
 GENVMPROTOCOL       = 'http'
 GENVMHOST           = 'genvm'
-GENVMPORT           = '6000'
+GENVMPORT          = '6000'
 # Location of file excuted inside the GenVM
 GENVMCONLOC         = '/tmp'
 # TODO: Will be removed with the new logging

backend/protocol_rpc/server.py

@@ -15,6 +15,7 @@
 from backend.protocol_rpc.configuration import GlobalConfiguration
 from backend.protocol_rpc.message_handler.base import MessageHandler
 from backend.protocol_rpc.endpoints import register_all_rpc_endpoints
+from backend.protocol_rpc.validators_init import initialize_validators
 from dotenv import load_dotenv
 
 from backend.database_handler.transactions_processor import TransactionsProcessor
@@ -58,6 +59,16 @@ def create_app():
     accounts_manager = AccountsManager(sqlalchemy_db.session)
     validators_registry = ValidatorsRegistry(sqlalchemy_db.session)
     llm_provider_registry = LLMProviderRegistry(sqlalchemy_db.session)
+
+    # Initialize validators from environment configuration in a thread
+    initialize_validators_db_session = Session(engine, expire_on_commit=False)
+    initialize_validators(
+        os.getenv("VALIDATORS_CONFIG_JSON"),
+        ValidatorsRegistry(initialize_validators_db_session),
+        AccountsManager(initialize_validators_db_session),
+    )
+    initialize_validators_db_session.commit()
+
     consensus = ConsensusAlgorithm(
         lambda: Session(engine, expire_on_commit=False), msg_handler
     )

backend/protocol_rpc/validators_init.py

@@ -0,0 +1,73 @@
+import json
+from dataclasses import dataclass
+from backend.database_handler.accounts_manager import AccountsManager
+from backend.database_handler.validators_registry import ValidatorsRegistry
+
+
+@dataclass
+class ValidatorConfig:
+    stake: int
+    provider: str
+    model: str
+    config: dict | None = None
+    plugin: str | None = None
+    plugin_config: dict | None = None
+    amount: int = 1
+
+
+def initialize_validators(
+    validators_json: str,
+    validators_registry: ValidatorsRegistry,
+    accounts_manager: AccountsManager,
+    validator_creator=None,
+):
+    """
+    Idempotently initialize validators from a JSON string by deleting all existing validators and creating new ones.
+
+    Args:
+        validators_json: JSON string containing validator configurations
+        validators_registry: Registry to store validator information
+        accounts_manager: AccountsManager to create validator accounts
+        validator_creator: Function to create validators (defaults to endpoints.create_validator)
+    """
+
+    if not validators_json:
+        print("No validators to initialize")
+        return
+
+    # If no validator_creator is provided, import the default one
+    if validator_creator is None:
+        from backend.protocol_rpc.endpoints import create_validator
+
+        validator_creator = create_validator
+
+    try:
+        validators_data = json.loads(validators_json)
+    except json.JSONDecodeError as e:
+        raise ValueError(f"Invalid JSON in validators_json: {str(e)}")
+
+    if not isinstance(validators_data, list):
+        raise ValueError("validators_json must contain a JSON array")
+
+    # Delete all existing validators
+    validators_registry.delete_all_validators()
+
+    # Create new validators
+    for validator_data in validators_data:
+        try:
+            validator = ValidatorConfig(**validator_data)
+
+            for _ in range(validator.amount):
+                validator_creator(
+                    validators_registry,
+                    accounts_manager,
+                    validator.stake,
+                    validator.provider,
+                    validator.model,
+                    validator.config,
+                    validator.plugin,
+                    validator.plugin_config,
+                )
+
+        except Exception as e:
+            raise ValueError(f"Failed to create validator `{validator_data}`: {str(e)}")

docker/Dockerfile.backend

@@ -21,7 +21,7 @@ ENV HUGGINGFACE_HUB_CACHE /home/backend-user/.cache/huggingface
 COPY ../.env .
 COPY backend $path/backend
 
-HEALTHCHECK --interval=1s --timeout=1s --retries=15 --start-period=1s CMD python backend/healthcheck.py --port ${FLASK_SERVER_PORT}
+HEALTHCHECK --interval=1s --timeout=1s --retries=30 --start-period=3s CMD python backend/healthcheck.py --port ${FLASK_SERVER_PORT}
 
 ###########START NEW IMAGE : DEBUGGER ###################
 FROM base AS debug

tests/integration/test_llm_providers_registry.py

@@ -1,16 +1,41 @@
 from tests.common.request import payload, post_request_localhost
 from tests.common.response import has_success_status
 
+# provider, model, plugin
+
 
 def test_llm_providers():
+    providers_and_models_response = post_request_localhost(
+        payload("sim_getProvidersAndModels")
+    ).json()
+    assert has_success_status(providers_and_models_response)
+    providers_and_models = providers_and_models_response["result"]
+
+    gpt4o_provider_id = next(
+        (
+            provider["id"]
+            for provider in providers_and_models
+            if provider["model"] == "gpt-4o"
+            and provider["provider"] == "openai"
+            and provider["plugin"] == "openai"
+        ),
+        None,
+    )
+
+    # Delete it
+    response = post_request_localhost(
+        payload("sim_deleteProvider", gpt4o_provider_id)
+    ).json()
+    assert has_success_status(response)
+
+    # Create it again
     provider = {
         "provider": "openai",
-        "model": "gpt-4",
+        "model": "gpt-4o",
         "config": {},
         "plugin": "openai",
         "plugin_config": {"api_key_env_var": "OPENAIKEY", "api_url": None},
     }
-    # Create a new provider
     response = post_request_localhost(payload("sim_addProvider", provider)).json()
     assert has_success_status(response)
 
@@ -23,56 +48,15 @@ def test_llm_providers():
         "plugin": "openai",
         "plugin_config": {"api_key_env_var": "OPENAIKEY", "api_url": None},
     }
-    # Uodate it
+
+    # Update it
     response = post_request_localhost(
         payload("sim_updateProvider", provider_id, updated_provider)
     ).json()
     assert has_success_status(response)
 
-    # Delete it
-    response = post_request_localhost(payload("sim_deleteProvider", provider_id)).json()
-    assert has_success_status(response)
-
-
-def test_llm_providers_behavior():
-    """
-    Test the behavior of LLM providers endpoints by performing the following steps:
-
-    1. Reset the default LLM providers.
-    2. Retrieve the list of providers and models.
-    3. Extract the first default provider and the ID of the last provider.
-    4. Add a new provider using the first default provider's data.
-    5. Update the last provider using the first default provider's data.
-    6. Delete the newly added provider.
-
-    """
+    # Reset it
     reset_result = post_request_localhost(
         payload("sim_resetDefaultsLlmProviders")
     ).json()
     assert has_success_status(reset_result)
-
-    response = post_request_localhost(payload("sim_getProvidersAndModels")).json()
-    assert has_success_status(response)
-
-    default_providers = response["result"]
-    first_default_provider: dict = default_providers[0]
-    del first_default_provider["id"]
-    last_provider_id = default_providers[-1]["id"]
-
-    # Create a new provider
-    response = post_request_localhost(
-        payload("sim_addProvider", first_default_provider)
-    ).json()
-    assert has_success_status(response)
-
-    provider_id = response["result"]
-
-    # Uodate it
-    response = post_request_localhost(
-        payload("sim_updateProvider", last_provider_id, first_default_provider)
-    ).json()
-    assert has_success_status(response)
-
-    # Delete it
-    response = post_request_localhost(payload("sim_deleteProvider", provider_id)).json()
-    assert has_success_status(response)

tests/unit/test_validators_init.py

@@ -0,0 +1,144 @@
+import pytest
+from unittest.mock import Mock
+from backend.protocol_rpc.validators_init import initialize_validators
+
+
+def test_initialize_validators_empty_json():
+    """Test that empty JSON string returns without doing anything"""
+    mock_registry = Mock()
+    mock_accounts = Mock()
+    mock_creator = Mock()
+
+    initialize_validators("", mock_registry, mock_accounts, mock_creator)
+
+    mock_registry.delete_all_validators.assert_not_called()
+    mock_creator.assert_not_called()
+
+
+def test_initialize_validators_invalid_json():
+    """Test that invalid JSON raises ValueError"""
+    mock_registry = Mock()
+    mock_accounts = Mock()
+    mock_creator = Mock()
+
+    with pytest.raises(ValueError, match="Invalid JSON"):
+        initialize_validators(
+            "{invalid json", mock_registry, mock_accounts, mock_creator
+        )
+
+
+def test_initialize_validators_non_array_json():
+    """Test that non-array JSON raises ValueError"""
+    mock_registry = Mock()
+    mock_accounts = Mock()
+    mock_creator = Mock()
+
+    with pytest.raises(ValueError, match="must contain a JSON array"):
+        initialize_validators("{}", mock_registry, mock_accounts, mock_creator)
+
+
+def test_initialize_validators_success():
+    """Test successful initialization of validators"""
+    mock_registry = Mock()
+    mock_accounts = Mock()
+    mock_creator = Mock()
+
+    validators_json = """[
+        {
+            "stake": 100,
+            "provider": "test-provider",
+            "model": "test-model",
+            "config": {"key": "value"},
+            "plugin": "test-plugin",
+            "plugin_config": {"plugin_key": "plugin_value"}
+        },
+        {
+            "stake": 200,
+            "provider": "another-provider",
+            "model": "another-model",
+            "amount": 2
+        }
+    ]"""
+
+    initialize_validators(validators_json, mock_registry, mock_accounts, mock_creator)
+
+    # Verify that existing validators were deleted
+    mock_registry.delete_all_validators.assert_called_once()
+
+    # Verify that creator was called for each validator with correct arguments
+    assert mock_creator.call_count == 3
+
+    # Check first validator creation call
+    mock_creator.assert_any_call(
+        mock_registry,
+        mock_accounts,
+        100,
+        "test-provider",
+        "test-model",
+        {"key": "value"},
+        "test-plugin",
+        {"plugin_key": "plugin_value"},
+    )
+
+    # Check second validator creation call
+    mock_creator.assert_any_call(
+        mock_registry,
+        mock_accounts,
+        200,
+        "another-provider",
+        "another-model",
+        None,
+        None,
+        None,
+    )
+
+    mock_creator.assert_any_call(
+        mock_registry,
+        mock_accounts,
+        200,
+        "another-provider",
+        "another-model",
+        None,
+        None,
+        None,
+    )
+
+
+def test_initialize_validators_invalid_config():
+    """Test that invalid validator configuration raises ValueError"""
+    mock_registry = Mock()
+    mock_accounts = Mock()
+    mock_creator = Mock()
+
+    # Missing required field 'model'
+    validators_json = """[
+        {
+            "stake": 100,
+            "provider": "test-provider"
+        }
+    ]"""
+
+    with pytest.raises(ValueError, match="Failed to create validator"):
+        initialize_validators(
+            validators_json, mock_registry, mock_accounts, mock_creator
+        )
+
+
+def test_initialize_validators_creator_error():
+    """Test that creator function errors are properly handled"""
+    mock_registry = Mock()
+    mock_accounts = Mock()
+    mock_creator = Mock(side_effect=Exception("Creator error"))
+
+    validators_json = """[
+        {
+            "stake": 100,
+            "provider": "test-provider",
+            "model": "test-model"
+        }
+    ]"""
+
+    with pytest.raises(ValueError, match="Failed to create validator.*Creator error"):
+        initialize_validators(
+            validators_json, mock_registry, mock_accounts, mock_creator
+        )