..

.github/workflows/python-api-client-tests.yaml

@@ -128,6 +128,164 @@ jobs:
           cargo build --bin worker-executor
           cargo build --bin golem-worker-service
           
+          # Create config directory
+          mkdir -p config
+          
+          # Create configuration files
+          cat << EOF > config/shard-manager.toml
+          [persistence]
+          type = "Redis"
+          [persistence.config]
+          host = "localhost"
+          port = ${{ env.REDIS_PORT }}
+          
+          [http]
+          port = ${{ env.SHARD_MANAGER_HTTP_PORT }}
+          
+          [grpc]
+          port = ${{ env.SHARD_MANAGER_GRPC_PORT }}
+          EOF
+          
+          cat << EOF > config/component-service.toml
+          [environment]
+          type = "local"
+          
+          [component_store]
+          type = "Local"
+          [component_store.config]
+          object_prefix = ""
+          root_path = "/component_store"
+          
+          [compilation]
+          type = "Enabled"
+          [compilation.config]
+          host = "localhost"
+          port = ${{ env.COMPONENT_COMPILATION_SERVICE_GRPC_PORT }}
+          
+          [db]
+          type = "Postgres"
+          [db.config]
+          host = "localhost"
+          port = ${{ env.POSTGRES_PORT }}
+          database = "${{ env.POSTGRES_DB }}"
+          username = "${{ env.POSTGRES_USER }}"
+          password = "${{ env.POSTGRES_PASSWORD }}"
+          max_connections = 10
+          
+          [grpc]
+          port = ${{ env.COMPONENT_SERVICE_GRPC_PORT }}
+          
+          [http]
+          port = ${{ env.COMPONENT_SERVICE_HTTP_PORT }}
+          EOF
+          
+          cat << EOF > config/component-compilation-service.toml
+          [rust]
+          backtrace = 1
+          log = "info,h2=warn,hyper=warn,tower=warn"
+          
+          [component_service]
+          host = "localhost"
+          port = ${{ env.COMPONENT_SERVICE_GRPC_PORT }}
+          access_token = "5C832D93-FF85-4A8F-9803-513950FDFDB1"
+          
+          [compiled_component_service]
+          type = "Enabled"
+          
+          [blob_storage]
+          type = "LocalFileSystem"
+          [blob_storage.config]
+          root = "/worker_executor_store"
+          
+          [http]
+          port = ${{ env.COMPONENT_COMPILATION_SERVICE_HTTP_PORT }}
+          
+          [grpc]
+          port = ${{ env.COMPONENT_COMPILATION_SERVICE_GRPC_PORT }}
+          EOF
+          
+          cat << EOF > config/worker-executor.toml
+          [environment]
+          type = "local"
+          
+          [key_value_storage]
+          type = "Redis"
+          [key_value_storage.config]
+          host = "localhost"
+          port = ${{ env.REDIS_PORT }}
+          
+          [indexed_storage]
+          type = "KVStoreRedis"
+          
+          [blob_storage]
+          type = "LocalFileSystem"
+          [blob_storage.config]
+          root = "/worker_executor_store"
+          
+          [component_service]
+          host = "localhost"
+          port = ${{ env.COMPONENT_SERVICE_GRPC_PORT }}
+          access_token = "2A354594-7A63-4091-A46B-CC58D379F677"
+          
+          [http]
+          port = ${{ env.WORKER_EXECUTOR_HTTP_PORT }}
+          
+          [grpc]
+          port = ${{ env.WORKER_EXECUTOR_GRPC_PORT }}
+          
+          [shard_manager_service]
+          type = "Grpc"
+          [shard_manager_service.config]
+          host = "localhost"
+          port = ${{ env.SHARD_MANAGER_GRPC_PORT }}
+          EOF
+          
+          cat << EOF > config/worker-service.toml
+          [rust]
+          backtrace = 1
+          log = "info,h2=warn,hyper=warn,tower=warn"
+          
+          [db]
+          type = "Postgres"
+          [db.config]
+          host = "localhost"
+          port = ${{ env.POSTGRES_PORT }}
+          database = "${{ env.POSTGRES_DB }}"
+          username = "${{ env.POSTGRES_USER }}"
+          password = "${{ env.POSTGRES_PASSWORD }}"
+          max_connections = 10
+          
+          [environment]
+          type = "local"
+          
+          [component_service]
+          host = "localhost"
+          port = ${{ env.COMPONENT_SERVICE_GRPC_PORT }}
+          access_token = "5C832D93-FF85-4A8F-9803-513950FDFDB1"
+          
+          [gateway_session_storage]
+          type = "Redis"
+          [gateway_session_storage.config]
+          host = "localhost"
+          port = ${{ env.REDIS_PORT }}
+          
+          [routing_table]
+          host = "localhost"
+          port = ${{ env.SHARD_MANAGER_GRPC_PORT }}
+          
+          [http]
+          port = ${{ env.WORKER_SERVICE_HTTP_PORT }}
+          
+          [grpc]
+          port = ${{ env.WORKER_SERVICE_GRPC_PORT }}
+          
+          [custom_request]
+          port = ${{ env.WORKER_SERVICE_CUSTOM_REQUEST_PORT }}
+          EOF
+          
+          # Create required directories
+          mkdir -p /component_store /worker_executor_store
+          
           # Start services in order
           echo "Starting Shard Manager..."
           cargo run --bin golem-shard-manager &

tests/api_client_test.py

@@ -9,16 +9,23 @@
 from typing import Optional, Dict, Any, Tuple
 import wasmtime
 from wasi.http import *
+import logging
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(levelname)s - %(message)s'
+)
+logger = logging.getLogger(__name__)
 
 class GolemApiClient:
     """Client for testing Golem's OpenAPI and Swagger UI functionality"""
 
-    def __init__(self, rust_server_url: str = "http://localhost:9881"):
-        self.rust_server_url = rust_server_url
+    def __init__(self, rust_server_url: str = None):
+        self.rust_server_url = rust_server_url or f"http://localhost:{os.getenv('GOLEM_ROUTER_PORT', '9881')}"
         self.output_dir = "test-output"
-        self.redis_url = os.getenv("REDIS_URL", "redis://localhost:6380")
-        self.db_url = os.getenv(
-            "DATABASE_URL", 
+        self.redis_url = f"redis://localhost:{os.getenv('REDIS_PORT', '6380')}"
+        self.db_url = (
             f"postgres://{os.getenv('POSTGRES_USER', 'golem_user')}:"
             f"{os.getenv('POSTGRES_PASSWORD', 'golem_password')}@"
             f"localhost:{os.getenv('POSTGRES_PORT', '5432')}/"
@@ -34,49 +41,56 @@ def __init__(self, rust_server_url: str = "http://localhost:9881"):
         self.wasi_config.inherit_stderr()
         self.store.set_wasi(self.wasi_config)
 
+        # Service ports from environment
+        self.service_ports = {
+            'shard_manager': int(os.getenv('SHARD_MANAGER_HTTP_PORT', '8081')),
+            'component_service': int(os.getenv('COMPONENT_SERVICE_HTTP_PORT', '8083')),
+            'component_compilation': int(os.getenv('COMPONENT_COMPILATION_SERVICE_HTTP_PORT', '8084')),
+            'worker_executor': int(os.getenv('WORKER_EXECUTOR_HTTP_PORT', '8082')),
+            'worker_service': int(os.getenv('WORKER_SERVICE_HTTP_PORT', '9005'))
+        }
+
     async def check_services(self) -> Tuple[bool, str]:
         """Check if all required services are healthy"""
         try:
             # Check Redis
+            logger.info("Checking Redis connection...")
             redis_client = redis.from_url(self.redis_url)
             redis_client.ping()
 
             # Check PostgreSQL
+            logger.info("Checking PostgreSQL connection...")
             async with await psycopg.AsyncConnection.connect(self.db_url) as conn:
                 async with conn.cursor() as cur:
                     await cur.execute("SELECT 1")
                     await cur.fetchone()
 
             # Check all Golem services
-            ports = [
-                int(os.getenv("SHARD_MANAGER_HTTP_PORT", "8081")),
-                int(os.getenv("COMPONENT_SERVICE_HTTP_PORT", "8083")),
-                int(os.getenv("COMPONENT_COMPILATION_SERVICE_HTTP_PORT", "8084")),
-                int(os.getenv("WORKER_EXECUTOR_HTTP_PORT", "8082")),
-                int(os.getenv("WORKER_SERVICE_HTTP_PORT", "9005"))
-            ]
-
-            async with httpx.AsyncClient() as client:
-                for port in ports:
-                    response = await client.get(f"http://localhost:{port}/health")
-                    if response.status_code != 200:
-                        return False, f"Service on port {port} health check failed: {response.status_code}"
+            logger.info("Checking Golem services...")
+            timeout = httpx.Timeout(30.0, connect=10.0)
+            async with httpx.AsyncClient(timeout=timeout) as client:
+                for service, port in self.service_ports.items():
+                    logger.info(f"Checking {service} on port {port}...")
+                    try:
+                        response = await client.get(f"http://localhost:{port}/health")
+                        response.raise_for_status()
+                        assert response.status_code == 200, f"Service {service} returned status {response.status_code}"
+                    except Exception as e:
+                        return False, f"Service {service} health check failed: {str(e)}"
 
             return True, "All services are healthy"
         except redis.RedisError as e:
             return False, f"Redis connection failed: {str(e)}"
         except psycopg.Error as e:
             return False, f"PostgreSQL connection failed: {str(e)}"
-        except httpx.RequestError as e:
-            return False, f"Service connection failed: {str(e)}"
         except Exception as e:
             return False, f"Unexpected error: {str(e)}"
 
     async def make_request(self, method: str, url: str, **kwargs) -> httpx.Response:
         """Make HTTP request using WASI HTTP"""
         # Create request options
         options = RequestOptions()
-        options.connect_timeout_ms = 30000  # 30 seconds
+        options.connect_timeout_ms = int(os.getenv('REQUEST_TIMEOUT_MS', '30000'))
 
         # Create request
         request = OutgoingRequest(method, url)
@@ -100,6 +114,7 @@ def get_export_path(api_id: str, version: str) -> str:
 
     async def export_openapi(self, api_id: str, version: str, format: str = "json") -> Dict[str, Any]:
         """Export OpenAPI spec from Rust server"""
+        logger.info(f"Exporting OpenAPI spec for {api_id} version {version} in {format} format")
         path = self.get_export_path(api_id, version)
 
         response = await self.make_request(
@@ -122,6 +137,7 @@ async def export_openapi(self, api_id: str, version: str, format: str = "json")
 
     async def get_swagger_ui(self, theme: str = "", api_id: str = "default", version: str = "1.0") -> str:
         """Get Swagger UI HTML from Rust server"""
+        logger.info(f"Getting Swagger UI for {api_id} version {version} with theme '{theme}'")
         params = {
             "theme": theme,
             "api_id": api_id,
@@ -144,12 +160,15 @@ async def get_swagger_ui(self, theme: str = "", api_id: str = "default", version
 
 async def test_service_health():
     """Test all service health checks"""
+    logger.info("Starting service health checks...")
     client = GolemApiClient()
     healthy, message = await client.check_services()
     assert healthy, f"Service health check failed: {message}"
+    logger.info("Service health checks passed successfully")
 
 async def test_openapi_export():
     """Test OpenAPI export functionality"""
+    logger.info("Starting OpenAPI export tests...")
     client = GolemApiClient()
 
     # Test JSON export
@@ -168,9 +187,11 @@ async def test_openapi_export():
     custom_spec = await client.export_openapi("custom-api", "2.0.0", "json")
     assert custom_spec["info"]["title"] == "custom-api API"
     assert custom_spec["info"]["version"] == "2.0.0"
+    logger.info("OpenAPI export tests passed successfully")
 
 async def test_swagger_ui():
     """Test Swagger UI functionality"""
+    logger.info("Starting Swagger UI tests...")
     client = GolemApiClient()
 
     # Test default theme
@@ -188,19 +209,27 @@ async def test_swagger_ui():
     custom_ui = await client.get_swagger_ui(api_id="custom-api", version="2.0")
     assert "custom-api" in custom_ui
     assert "2.0" in custom_ui
+    logger.info("Swagger UI tests passed successfully")
 
 async def run_all_tests():
     """Run all tests in sequence"""
-    print("Running service health checks...")
-    await test_service_health()
-
-    print("Testing OpenAPI export...")
-    await test_openapi_export()
-
-    print("Testing Swagger UI...")
-    await test_swagger_ui()
-
-    print("All tests completed successfully!")
+    try:
+        logger.info("Starting test suite...")
+
+        print("Running service health checks...")
+        await test_service_health()
+
+        print("Testing OpenAPI export...")
+        await test_openapi_export()
+
+        print("Testing Swagger UI...")
+        await test_swagger_ui()
+
+        logger.info("All tests completed successfully!")
+        print("All tests completed successfully!")
+    except Exception as e:
+        logger.error(f"Test suite failed: {str(e)}")
+        raise
 
 if __name__ == "__main__":
     asyncio.run(run_all_tests())