Merge branch 'DIRACGrid:main' into issue-257-open-telemetry-usage

.github/workflows/extensions.yml

@@ -64,7 +64,7 @@ jobs:
           export DIRACX_EXTENSIONS=gubbins,diracx
           pytest --cov-report=xml:coverage.xml --junitxml=report.xml
       - name: Upload coverage report
-        uses: codecov/codecov-action@v4.6.0
+        uses: codecov/codecov-action@v5.0.7
 
 
   build-wheels:
@@ -256,7 +256,7 @@ jobs:
 
           coverage xml -o coverage-demo.xml --data-file "${coverage_data}"
       - name: Upload coverage report
-        uses: codecov/codecov-action@v4.6.0
+        uses: codecov/codecov-action@v5.0.7
         with:
           files: ./coverage-pytest.xml,./coverage-demo.xml
 

.github/workflows/main.yml

@@ -77,7 +77,7 @@ jobs:
           pip install mypy ${{ matrix.package }}[types]
           mypy ${{ matrix.package }}/src
       - name: Upload coverage report
-        uses: codecov/codecov-action@v4.6.0
+        uses: codecov/codecov-action@v5.0.7
 
   pytest-integration:
     runs-on: ubuntu-latest
@@ -144,7 +144,7 @@ jobs:
 
           coverage xml -o coverage-demo.xml --data-file "${coverage_data}"
       - name: Upload coverage report
-        uses: codecov/codecov-action@v4.6.0
+        uses: codecov/codecov-action@v5.0.7
         with:
           files: ./coverage-pytest.xml,./coverage-demo.xml
 

.github/workflows/vulnerabilities.yml

@@ -0,0 +1,92 @@
+name: Scan Vulnerabilities
+
+on:
+  schedule:
+    - cron: '30 9 * * 1'
+
+permissions:
+  contents: read
+
+jobs:
+  scan-docker-images:
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          tags: true
+
+      - name: Run Trivy (client:dev)
+        uses: aquasecurity/[email protected]
+        with:
+          image-ref: "ghcr.io/diracgrid/diracx/client:dev"
+          format: "sarif"
+          output: "client-dev-vulnerability-report.sarif"
+
+      - name: Upload SARIF to GitHub Security (client:dev)
+        uses: github/codeql-action/upload-sarif@v3
+        with:
+          sarif_file: "client-dev-vulnerability-report.sarif"
+          category: "client-dev"
+
+      - name: Run Trivy (services:dev)
+        uses: aquasecurity/[email protected]
+        with:
+          image-ref: "ghcr.io/diracgrid/diracx/services:dev"
+          format: "sarif"
+          output: "services-dev-vulnerability-report.sarif"
+          skip-setup-trivy: true
+
+      - name: Upload SARIF to GitHub Security (services:dev)
+        uses: github/codeql-action/upload-sarif@v3
+        with:
+          sarif_file: "services-dev-vulnerability-report.sarif"
+          category: "services-dev"
+
+      - name: Get Latest Release Tag
+        id: get-latest-tag
+        run: |
+          tag=$(git rev-list --tags --max-count=1 --date-order)
+          if [ -z "$tag" ]; then
+            echo "latest_tag=" >> $GITHUB_OUTPUT
+          else
+            latest_tag=$(git describe --tags "$tag")
+            echo "latest_tag=${latest_tag}" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Run Trivy (client:release)
+        if: ${{ steps.get-latest-tag.outputs.latest_tag != '' }}
+        uses: aquasecurity/[email protected]
+        with:
+          image-ref: "ghcr.io/diracgrid/diracx/client:${{ steps.get-latest-tag.outputs.latest_tag }}"
+          format: "sarif"
+          output: "client-rel-vulnerability-report.sarif"
+          skip-setup-trivy: true
+
+      - name: Upload SARIF to GitHub Security (client:rel)
+        if: ${{ steps.get-latest-tag.outputs.latest_tag != '' }}
+        uses: github/codeql-action/upload-sarif@v3
+        with:
+          sarif_file: "client-rel-vulnerability-report.sarif"
+          category: "client-rel"
+
+      - name: Run Trivy (services:release)
+        if: ${{ steps.get-latest-tag.outputs.latest_tag != '' }}
+        uses: aquasecurity/[email protected]
+        with:
+          image-ref: "ghcr.io/diracgrid/diracx/services:${{ steps.get-latest-tag.outputs.latest_tag }}"
+          format: "sarif"
+          output: "services-rel-vulnerability-report.sarif"
+          skip-setup-trivy: true
+
+      - name: Upload SARIF to GitHub Security (services:rel)
+        if: ${{ steps.get-latest-tag.outputs.latest_tag != '' }}
+        uses: github/codeql-action/upload-sarif@v3
+        with:
+          sarif_file: "services-rel-vulnerability-report.sarif"
+          category: "services-rel"

.pre-commit-config.yaml

@@ -13,7 +13,7 @@ repos:
       - id: check-added-large-files
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: 'v0.7.1'
+    rev: 'v0.8.1'
     hooks:
       - id: ruff
         args: ["--fix"]

diracx-cli/pyproject.toml

@@ -18,7 +18,7 @@ dependencies = [
     "diracx-client",
     "diracx-core",
     "gitpython",
-    "pydantic",
+    "pydantic>=2.10",
     "rich",
     "typer",
     "pyyaml",

diracx-cli/src/diracx/cli/__init__.py

@@ -10,7 +10,7 @@
 from diracx.client.models import DeviceFlowErrorResponse
 from diracx.core.extensions import select_from_extension
 from diracx.core.preferences import get_diracx_preferences
-from diracx.core.utils import write_credentials
+from diracx.core.utils import read_credentials, write_credentials
 
 from .utils import AsyncTyper
 
@@ -116,11 +116,11 @@ async def logout():
     async with DiracClient() as api:
         credentials_path = get_diracx_preferences().credentials_path
         if credentials_path.exists():
-            credentials = json.loads(credentials_path.read_text())
+            credentials = read_credentials(credentials_path)
 
             # Revoke refresh token
             try:
-                await api.auth.revoke_refresh_token(credentials["refresh_token"])
+                await api.auth.revoke_refresh_token(credentials.refresh_token)
             except Exception as e:
                 print(f"Error revoking the refresh token {e!r}")
                 pass

diracx-cli/src/diracx/cli/jobs.py

@@ -152,7 +152,7 @@ def display_rich(data, content_range: ContentRange) -> None:
 async def submit(jdl: list[FileText]):
     async with DiracClient() as api:
         # api.valid(enforce_https=False)
-        jobs = await api.jobs.submit_bulk_jobs([x.read() for x in jdl])
+        jobs = await api.jobs.submit_bulk_jdl_jobs([x.read() for x in jdl])
     print(
         f"Inserted {len(jobs)} jobs with ids: {','.join(map(str, (job.job_id for job in jobs)))}"
     )

diracx-client/src/diracx/client/generated/__init__.py

@@ -1,14 +1,20 @@
 # coding=utf-8
 # --------------------------------------------------------------------------
-# Code generated by Microsoft (R) AutoRest Code Generator (autorest: 3.10.2, generator: @autorest/[email protected].0)
+# Code generated by Microsoft (R) AutoRest Code Generator (autorest: 3.10.3, generator: @autorest/[email protected].5)
 # Changes may cause incorrect behavior and will be lost if the code is regenerated.
 # --------------------------------------------------------------------------
+# pylint: disable=wrong-import-position
 
-from ._client import Dirac
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from ._patch import *  # pylint: disable=unused-wildcard-import
+
+from ._client import Dirac  # type: ignore
 
 try:
     from ._patch import __all__ as _patch_all
-    from ._patch import *  # pylint: disable=unused-wildcard-import
+    from ._patch import *
 except ImportError:
     _patch_all = []
 from ._patch import patch_sdk as _patch_sdk

diracx-client/src/diracx/client/generated/_client.py

@@ -1,6 +1,6 @@
 # coding=utf-8
 # --------------------------------------------------------------------------
-# Code generated by Microsoft (R) AutoRest Code Generator (autorest: 3.10.2, generator: @autorest/[email protected].0)
+# Code generated by Microsoft (R) AutoRest Code Generator (autorest: 3.10.3, generator: @autorest/[email protected].5)
 # Changes may cause incorrect behavior and will be lost if the code is regenerated.
 # --------------------------------------------------------------------------
 

diracx-client/src/diracx/client/generated/_configuration.py

@@ -1,6 +1,6 @@
 # coding=utf-8
 # --------------------------------------------------------------------------
-# Code generated by Microsoft (R) AutoRest Code Generator (autorest: 3.10.2, generator: @autorest/[email protected].0)
+# Code generated by Microsoft (R) AutoRest Code Generator (autorest: 3.10.3, generator: @autorest/[email protected].5)
 # Changes may cause incorrect behavior and will be lost if the code is regenerated.
 # --------------------------------------------------------------------------
 

diracx-client/src/diracx/client/generated/_serialization.py

@@ -1,3 +1,4 @@
+# pylint: disable=too-many-lines
 # --------------------------------------------------------------------------
 #
 # Copyright (c) Microsoft Corporation. All rights reserved.
@@ -536,7 +537,6 @@ def _flatten_subtype(cls, key, objects):
     def _classify(cls, response, objects):
         """Check the class _subtype_map for any child classes.
         We want to ignore any inherited _subtype_maps.
-        Remove the polymorphic key from the initial data.
 
         :param dict response: The initial data
         :param dict objects: The class objects
@@ -548,9 +548,9 @@ def _classify(cls, response, objects):
 
             if not isinstance(response, ET.Element):
                 rest_api_response_key = cls._get_rest_key_parts(subtype_key)[-1]
-                subtype_value = response.pop(
+                subtype_value = response.get(
                     rest_api_response_key, None
-                ) or response.pop(subtype_key, None)
+                ) or response.get(subtype_key, None)
             else:
                 subtype_value = xml_key_extractor(
                     subtype_key, cls._attribute_map[subtype_key], response
@@ -1802,13 +1802,13 @@ def _instantiate_model(self, response, attrs, additional_properties=None):
             try:
                 readonly = [
                     k
-                    for k, v in response._validation.items()
-                    if v.get("readonly")  # pylint: disable=protected-access
+                    for k, v in response._validation.items()  # pylint: disable=protected-access  # type: ignore
+                    if v.get("readonly")
                 ]
                 const = [
                     k
-                    for k, v in response._validation.items()
-                    if v.get("constant")  # pylint: disable=protected-access
+                    for k, v in response._validation.items()  # pylint: disable=protected-access  # type: ignore
+                    if v.get("constant")
                 ]
                 kwargs = {
                     k: v
@@ -1819,7 +1819,7 @@ def _instantiate_model(self, response, attrs, additional_properties=None):
                 for attr in readonly:
                     setattr(response_obj, attr, attrs.get(attr))
                 if additional_properties:
-                    response_obj.additional_properties = additional_properties
+                    response_obj.additional_properties = additional_properties  # type: ignore
                 return response_obj
             except TypeError as err:
                 msg = "Unable to deserialize {} into model {}. ".format(kwargs, response)  # type: ignore

diracx-client/src/diracx/client/generated/_vendor.py

@@ -1,5 +1,5 @@
 # --------------------------------------------------------------------------
-# Code generated by Microsoft (R) AutoRest Code Generator (autorest: 3.10.2, generator: @autorest/[email protected].0)
+# Code generated by Microsoft (R) AutoRest Code Generator (autorest: 3.10.3, generator: @autorest/[email protected].5)
 # Changes may cause incorrect behavior and will be lost if the code is regenerated.
 # --------------------------------------------------------------------------
 

diracx-client/src/diracx/client/generated/aio/__init__.py

@@ -1,14 +1,20 @@
 # coding=utf-8
 # --------------------------------------------------------------------------
-# Code generated by Microsoft (R) AutoRest Code Generator (autorest: 3.10.2, generator: @autorest/[email protected].0)
+# Code generated by Microsoft (R) AutoRest Code Generator (autorest: 3.10.3, generator: @autorest/[email protected].5)
 # Changes may cause incorrect behavior and will be lost if the code is regenerated.
 # --------------------------------------------------------------------------
+# pylint: disable=wrong-import-position
 
-from ._client import Dirac
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from ._patch import *  # pylint: disable=unused-wildcard-import
+
+from ._client import Dirac  # type: ignore
 
 try:
     from ._patch import __all__ as _patch_all
-    from ._patch import *  # pylint: disable=unused-wildcard-import
+    from ._patch import *
 except ImportError:
     _patch_all = []
 from ._patch import patch_sdk as _patch_sdk

diracx-client/src/diracx/client/generated/aio/_client.py

@@ -1,6 +1,6 @@
 # coding=utf-8
 # --------------------------------------------------------------------------
-# Code generated by Microsoft (R) AutoRest Code Generator (autorest: 3.10.2, generator: @autorest/[email protected].0)
+# Code generated by Microsoft (R) AutoRest Code Generator (autorest: 3.10.3, generator: @autorest/[email protected].5)
 # Changes may cause incorrect behavior and will be lost if the code is regenerated.
 # --------------------------------------------------------------------------
 

diracx-client/src/diracx/client/generated/aio/_configuration.py

@@ -1,6 +1,6 @@
 # coding=utf-8
 # --------------------------------------------------------------------------
-# Code generated by Microsoft (R) AutoRest Code Generator (autorest: 3.10.2, generator: @autorest/[email protected].0)
+# Code generated by Microsoft (R) AutoRest Code Generator (autorest: 3.10.3, generator: @autorest/[email protected].5)
 # Changes may cause incorrect behavior and will be lost if the code is regenerated.
 # --------------------------------------------------------------------------
 

diracx-client/src/diracx/client/generated/aio/_vendor.py

@@ -1,5 +1,5 @@
 # --------------------------------------------------------------------------
-# Code generated by Microsoft (R) AutoRest Code Generator (autorest: 3.10.2, generator: @autorest/[email protected].0)
+# Code generated by Microsoft (R) AutoRest Code Generator (autorest: 3.10.3, generator: @autorest/[email protected].5)
 # Changes may cause incorrect behavior and will be lost if the code is regenerated.
 # --------------------------------------------------------------------------
 

diracx-client/src/diracx/client/generated/aio/operations/__init__.py

@@ -1,16 +1,22 @@
 # coding=utf-8
 # --------------------------------------------------------------------------
-# Code generated by Microsoft (R) AutoRest Code Generator (autorest: 3.10.2, generator: @autorest/[email protected].0)
+# Code generated by Microsoft (R) AutoRest Code Generator (autorest: 3.10.3, generator: @autorest/[email protected].5)
 # Changes may cause incorrect behavior and will be lost if the code is regenerated.
 # --------------------------------------------------------------------------
+# pylint: disable=wrong-import-position
 
-from ._operations import WellKnownOperations
-from ._operations import AuthOperations
-from ._operations import ConfigOperations
-from ._operations import JobsOperations
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from ._patch import *  # pylint: disable=unused-wildcard-import
+
+from ._operations import WellKnownOperations  # type: ignore
+from ._operations import AuthOperations  # type: ignore
+from ._operations import ConfigOperations  # type: ignore
+from ._operations import JobsOperations  # type: ignore
 
 from ._patch import __all__ as _patch_all
-from ._patch import *  # pylint: disable=unused-wildcard-import
+from ._patch import *
 from ._patch import patch_sdk as _patch_sdk
 
 __all__ = [