chore(lint): adding python lint check

.github/workflows/acceptance_tests.yaml

@@ -7,34 +7,17 @@ on:
   pull_request:
     branches:
       - 'main'
+    paths:
+      - 'server/**'
+      - 'python-sdk/**'
+      - '.github/workflows/acceptance_tests.yaml'
 
 env:
   CARGO_TERM_COLOR: always
 
 jobs:
-  changes:
-    name: Get Changes
-    runs-on: ubuntu-latest
-    outputs:
-      ui: ${{ steps.filter.outputs.ui }}
-      indexify: ${{ steps.filter.outputs.indexify }}
-    steps:
-      - uses: actions/checkout@v3
-      - uses: dorny/paths-filter@v2
-        id: filter
-        with:
-          filters: |
-            ui:
-              - 'server/ui/**'
-            indexify:
-              - 'server/**'
-              - 'python-sdk/**'
-              - '.github/**'
-
   build:
     name: Build
-    needs: changes
-    if: ${{ needs.changes.outputs.indexify == 'true' }}
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
@@ -57,8 +40,7 @@ jobs:
 
   acceptance_tests:
     name: Run Acceptance Tests
-    needs: [changes, build]
-    if: ${{ needs.changes.outputs.indexify == 'true' }}
+    needs: build
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
@@ -147,8 +129,7 @@ jobs:
 
   last_release_acceptance_tests:
     name: Run Acceptance Tests of Last Release
-    needs: [changes, build]
-    if: ${{ needs.changes.outputs.indexify == 'true' }}
+    needs: build
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
@@ -238,4 +219,4 @@ jobs:
         run: |
           cd python-sdk
           export INDEXIFY_URL=http://localhost:8900
-          poetry run python tests/test_graph_validation.py
+          poetry run python tests/test_graph_validation.py

.github/workflows/test.yaml

@@ -3,10 +3,10 @@ name: Tests
 on:
   push:
     branches:
-      - "main"
+      - 'main'
   pull_request:
     branches:
-      - "main"
+      - 'main'
 
 env:
   CARGO_TERM_COLOR: always
@@ -37,7 +37,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      
+
       - name: Setup Node.js
         if: ${{ needs.changes.outputs.ui == 'true' }}
         uses: actions/setup-node@v3
@@ -66,6 +66,20 @@ jobs:
           cd server/
           cargo +nightly fmt -- --check
 
+      - name: Install poetry
+        if: ${{ needs.changes.outputs.other == 'true' }}
+        run: pipx install poetry
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.11
+          cache: 'poetry'
+      - name: Python Format Check
+        if: ${{ needs.changes.outputs.other == 'true' }}
+        run: |
+          cd python-sdk/
+          make check
+
   test:
     name: Run Tests
     needs: changes
@@ -74,7 +88,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Update APT package list
-        run: sudo apt-get update || true  # skip errors as temp workaround for https://github.com/tensorlakeai/indexify/actions/runs/8814655533/job/24194983960#step:3:33
+        run: sudo apt-get update || true # skip errors as temp workaround for https://github.com/tensorlakeai/indexify/actions/runs/8814655533/job/24194983960#step:3:33
       - name: Install protoc
         run: sudo apt install -y protobuf-compiler npm
       - name: Set up Python 3.9
@@ -87,4 +101,4 @@ jobs:
         run: |
           docker stop --time 90 $(docker ps -aq) || true
           docker rm --force $(docker ps -aq) || true
-          docker rmi --force $(docker images -aq) || true
+          docker rmi --force $(docker images -aq) || true

python-sdk/Makefile

@@ -5,8 +5,12 @@ build:
 	@poetry build
 
 fmt:
-	black .
-	isort . --profile black
+	@poetry run black .
+	@poetry run isort . --profile black
+
+check:
+	@poetry run black --check .
+	@poetry run isort . --check-only --profile black
 
 lint:
 	@poetry run pylint ./indexify

python-sdk/indexify/common_util.py

@@ -6,8 +6,7 @@
 
 
 def get_httpx_client(
-    config_path: Optional[str] = None,
-    make_async: Optional[bool] = False
+    config_path: Optional[str] = None, make_async: Optional[bool] = False
 ) -> AsyncClient | Client:
     """
     Creates and returns an httpx.Client instance, optionally configured with TLS settings from a YAML config file.
@@ -44,30 +43,31 @@ def get_httpx_client(
         with open(config_path, "r") as file:
             config = yaml.safe_load(file)
         if config.get("use_tls", False):
-            print(f'Configuring client with TLS config: {config}')
+            print(f"Configuring client with TLS config: {config}")
             tls_config = config["tls_config"]
             return get_sync_or_async_client(make_async, **tls_config)
     return get_sync_or_async_client(make_async)
 
+
 def get_sync_or_async_client(
     make_async: Optional[bool] = False,
     cert_path: Optional[str] = None,
     key_path: Optional[str] = None,
     ca_bundle_path: Optional[str] = None,
 ) -> AsyncClient | Client:
     """
-       Creates and returns either a synchronous or asynchronous httpx client with optional TLS configuration.
+    Creates and returns either a synchronous or asynchronous httpx client with optional TLS configuration.
 
-       Args:
-           make_async (Optional[bool]): If True, returns an AsyncClient; if False, returns a synchronous Client.
-               Defaults to False.
-           cert_path (Optional[str]): Path to the client certificate file. Required for TLS configuration
-               when key_path is also provided.
-           key_path (Optional[str]): Path to the client private key file. Required for TLS configuration
-               when cert_path is also provided.
-           ca_bundle_path (Optional[str]): Path to the CA bundle file for certificate verification.
-               If not provided, defaults to system CA certificates.
-   """
+    Args:
+        make_async (Optional[bool]): If True, returns an AsyncClient; if False, returns a synchronous Client.
+            Defaults to False.
+        cert_path (Optional[str]): Path to the client certificate file. Required for TLS configuration
+            when key_path is also provided.
+        key_path (Optional[str]): Path to the client private key file. Required for TLS configuration
+            when cert_path is also provided.
+        ca_bundle_path (Optional[str]): Path to the CA bundle file for certificate verification.
+            If not provided, defaults to system CA certificates.
+    """
     if make_async:
         if cert_path and key_path:
             return httpx.AsyncClient(
@@ -82,7 +82,7 @@ def get_sync_or_async_client(
             return httpx.Client(
                 http2=True,
                 cert=(cert_path, key_path),
-                verify=ca_bundle_path if ca_bundle_path else True
+                verify=ca_bundle_path if ca_bundle_path else True,
             )
         else:
             return httpx.Client()

python-sdk/indexify/executor/agent.py

@@ -3,8 +3,8 @@
 import traceback
 from concurrent.futures.process import BrokenProcessPool
 from importlib.metadata import version
-from typing import Dict, List, Optional
 from pathlib import Path
+from typing import Dict, List, Optional
 
 from httpx_sse import aconnect_sse
 from pydantic import BaseModel
@@ -13,13 +13,13 @@
 from rich.text import Text
 from rich.theme import Theme
 
+from indexify.common_util import get_httpx_client
 from indexify.functions_sdk.data_objects import (
     FunctionWorkerOutput,
     IndexifyData,
 )
 from indexify.functions_sdk.graph_definition import ComputeGraphMetadata
 from indexify.http_client import IndexifyClient
-from indexify.common_util import get_httpx_client
 
 from ..functions_sdk.image import ImageInformation
 from . import image_dependency_installer
@@ -82,9 +82,7 @@ def __init__(
 
         self.num_workers = num_workers
         if config_path:
-            console.print(
-                "Running the extractor with TLS enabled", style="cyan bold"
-            )
+            console.print("Running the extractor with TLS enabled", style="cyan bold")
             self._protocol = "https"
         else:
             self._protocol = "http"
@@ -107,7 +105,7 @@ def __init__(
         self._task_reporter = TaskReporter(
             base_url=self._base_url,
             executor_id=self._executor_id,
-            config_path=self._config_path
+            config_path=self._config_path,
         )
 
     async def task_completion_reporter(self):
@@ -349,7 +347,7 @@ async def run(self):
         self._should_run = True
         while self._should_run:
             url = f"{self._protocol}://{self._server_addr}/internal/executors/{self._executor_id}/tasks"
-            print(f'calling url: {url}')
+            print(f"calling url: {url}")
 
             def to_sentence_case(snake_str):
                 words = snake_str.split("_")
@@ -442,8 +440,9 @@ async def _get_image_info_for_compute_graph(
     compute_fn_name: str = task.compute_fn
 
     http_client = IndexifyClient(
-        service_url=f"{protocol}://{server_addr}", namespace=namespace,
-        config_path=config_path
+        service_url=f"{protocol}://{server_addr}",
+        namespace=namespace,
+        config_path=config_path,
     )
     compute_graph: ComputeGraphMetadata = http_client.graph(graph_name)
 

python-sdk/indexify/executor/downloader.py

@@ -10,8 +10,8 @@
 from indexify.functions_sdk.data_objects import IndexifyData
 from indexify.functions_sdk.object_serializer import MsgPackSerializer
 
-from .api_objects import Task
 from ..common_util import get_httpx_client
+from .api_objects import Task
 
 custom_theme = Theme(
     {
@@ -30,7 +30,9 @@ class DownloadedInputs(BaseModel):
 
 
 class Downloader:
-    def __init__(self, code_path: str, base_url: str, config_path: Optional[str] = None):
+    def __init__(
+        self, code_path: str, base_url: str, config_path: Optional[str] = None
+    ):
         self.code_path = code_path
         self.base_url = base_url
         self._client = get_httpx_client(config_path)

python-sdk/indexify/executor/function_worker.py

@@ -15,10 +15,10 @@
 from indexify.functions_sdk.indexify_functions import (
     FunctionCallResult,
     GraphInvocationContext,
+    IndexifyFunction,
     IndexifyFunctionWrapper,
-    RouterCallResult,
     IndexifyRouter,
-    IndexifyFunction,
+    RouterCallResult,
 )
 
 function_wrapper_map: Dict[str, IndexifyFunctionWrapper] = {}

python-sdk/indexify/executor/task_reporter.py

@@ -23,7 +23,9 @@ def __bool__(self):
 
 
 class TaskReporter:
-    def __init__(self, base_url: str, executor_id: str, config_path: Optional[str] = None):
+    def __init__(
+        self, base_url: str, executor_id: str, config_path: Optional[str] = None
+    ):
         self._base_url = base_url
         self._executor_id = executor_id
         self._client = get_httpx_client(config_path)

python-sdk/tests/test_constants.py

@@ -5,8 +5,8 @@
     "tls_config": {
         "ca_bundle_path": "/path/to/ca_bundle.pem",
         "cert_path": "/path/to/cert.pem",
-        "key_path": "/path/to/key.pem"
-    }
+        "key_path": "/path/to/key.pem",
+    },
 }
 
 cert_path = tls_config["tls_config"]["cert_path"]
@@ -16,12 +16,12 @@
 config_path = "test/config/path"
 code_path = "test/code_path"
 task = Task(
-    id = "test_id",
-    namespace = "default",
-    compute_graph = "test_compute_graph",
-    compute_fn = "test_compute_fn",
-    invocation_id = "test_invocation_id",
-    input_key = "test|input|key",
-    requester_output_id = "test_output_id",
-    graph_version = 1,
+    id="test_id",
+    namespace="default",
+    compute_graph="test_compute_graph",
+    compute_fn="test_compute_fn",
+    invocation_id="test_invocation_id",
+    input_key="test|input|key",
+    requester_output_id="test_output_id",
+    graph_version=1,
 )