dev: sync with 0.35.3 (#821)

* Merge pull request #795 from cisco-open/apigw Add API GW login. Refactor session related files. * Added validation activities when TaskValidationError is raised * Bump actions/setup-python from 5.1.1 to 5.2.0 (#809) Bumps [actions/setup-python](https://github.com/actions/setup-python) from 5.1.1 to 5.2.0. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](actions/setup-python@39cd149...f677139) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <[email protected]> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * creating new radius and tacacs data classes (#815) * updating radius and tacacs data classes for new changes * fix static * fix static * make vpn vpn id optional * sub class * match with parent class * match with parent class * match with parent class * snake case * description * src checks * src checks * src checks * Add staging * bump up version * fix: provider as tenant login (#820) * draft * bump version * update env variables usage * bump dev version 0.35.3dev0 --------- Signed-off-by: dependabot[bot] <[email protected]> Co-authored-by: Jakub Krajewski <[email protected]> Co-authored-by: Bartlomiej Radwan <[email protected]> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: nikhilkp93 <[email protected]> Co-authored-by: Jakub Krajewski <[email protected]> Co-authored-by: Nikhil <[email protected]>
cisco-open · Sep 11, 2024 · 0d537b2 · 0d537b2
1 parent 466cadb
commit 0d537b2
Show file tree

Hide file tree

Showing 15 changed files with 221 additions and 175 deletions.
diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml
@@ -18,7 +18,7 @@ jobs:
     steps:
       - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
       - name: Set up Python
-        uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1
+        uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0
         with:
           python-version: ${{ matrix.python-version }}
       - name: Set Up Poetry

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -17,7 +17,7 @@ jobs:
     steps:
       - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
       - name: Set up Python
-        uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1
+        uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0
         with:
           python-version: 3.8
       - name: Set Up Poetry

diff --git a/.github/workflows/unittests.yml b/.github/workflows/unittests.yml
@@ -18,7 +18,7 @@ jobs:
     steps:
       - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
       - name: Set up Python
-        uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1
+        uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0
         with:
           python-version: ${{ matrix.python-version }}
       - name: Set Up Poetry

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -58,7 +58,9 @@ Always write a clear log message for your commits.
 ### Environment Variables
 - `catalystwan_devel` when set: loggers will be configured according to `./logging.conf` and `urllib3.exceptions.InsecureRequestWarning` will be suppressed
 
-- `catalystwan_export_endpoints` when set `endpoints-md` pre-commit step will generate `ENDPOINTS.md` file in addition to perform definition checks. This should be set only when creating version bump commit with new release tag.
+- `catalystwan_auth_trace` when set: authentication requests will not be anonymized in logs
+
+- `catalystwan_export_endpoints` when set: `endpoints-md` pre-commit step will generate `ENDPOINTS.md` file in addition to perform definition checks. This should be set only when creating version bump commit with new release tag.
 
 ## Code guidelines
 

diff --git a/ENDPOINTS.md b/ENDPOINTS.md
diff --git a/catalystwan/abstractions.py b/catalystwan/abstractions.py
@@ -46,7 +46,7 @@ def request(self, method: str, url: str, **kwargs) -> APIEndpointClientResponse:
         ...
 
     @property
-    def api_version(self) -> Optional[Version]:
+    def api_version(self) -> Version:
         ...
 
     @property
@@ -56,3 +56,15 @@ def session_type(self) -> Optional[SessionType]:
     @property
     def validate_responses(self) -> bool:
         ...
+
+
+class AuthProtocol(Protocol):
+    """
+    Additional interface for Auth to handle login/logout for multiple auth types by common ManagerSession
+    """
+
+    def logout(self, client: APIEndpointClient) -> None:
+        ...
+
+    def clear(self) -> None:
+        ...
diff --git a/catalystwan/api/task_status_api.py b/catalystwan/api/task_status_api.py
@@ -38,7 +38,10 @@ def __check_validation_status(self, task: TaskData):
         if not task.validation:
             return None
         if task.validation.status in (OperationStatus.FAILURE, OperationStatus.VALIDATION_FAILURE):
-            raise TaskValidationError(f"Task status validation failed, validation status is: {task.validation.status}")
+            raise TaskValidationError(
+                f"Task status validation failed, validation status is: {task.validation.status}"
+                f"\n{task.validation.activity}"
+            )
 
     def wait_for_completed(
         self,

diff --git a/catalystwan/api/template_api.py b/catalystwan/api/template_api.py
@@ -194,7 +194,7 @@ def get_device_specific_variables(name: str):
         task = Task(session=self.session, task_id=response["id"]).wait_for_completed(timeout_seconds=timeout_seconds)
         if task.result:
             return True
-        logger.warning(f"Failed to attach tempate: {name} to the device: {device.hostname}.")
+        logger.warning(f"Failed to attach template: {name} to the device: {device.hostname}.")
         logger.warning(f"Task activity information: {task.sub_tasks_data[0].activity}")
         return False
 

diff --git a/catalystwan/apigw_auth.py b/catalystwan/apigw_auth.py
@@ -1,17 +1,15 @@
 import logging
-from typing import TYPE_CHECKING, Literal, Optional
+from typing import Literal, Optional
 from urllib.parse import urlparse
 
 from pydantic import BaseModel, Field, PositiveInt
 from requests import HTTPError, PreparedRequest, post
 from requests.auth import AuthBase
 from requests.exceptions import JSONDecodeError
 
+from catalystwan.abstractions import APIEndpointClient, AuthProtocol
 from catalystwan.exceptions import CatalystwanException
-from catalystwan.response import ManagerResponse
-
-if TYPE_CHECKING:
-    from catalystwan.session import ManagerSession
+from catalystwan.response import auth_response_debug
 
 LoginMode = Literal["machine", "user", "session"]
 
@@ -27,17 +25,18 @@ class ApiGwLogin(BaseModel):
     token_duration: PositiveInt = Field(default=10, description="in minutes")
 
 
-class ApiGwAuth(AuthBase):
+class ApiGwAuth(AuthBase, AuthProtocol):
     """Attaches ApiGateway Authentication to the given Requests object.
 
     1. Get a bearer token by sending a POST request to the /apigw/login endpoint.
     2. Use the token in the Authorization header for subsequent requests.
     """
 
-    def __init__(self, login: ApiGwLogin, logger: Optional[logging.Logger] = None):
+    def __init__(self, login: ApiGwLogin, logger: Optional[logging.Logger] = None, verify=False):
         self.login = login
         self.token = ""
         self.logger = logger or logging.getLogger(__name__)
+        self.verify = verify
 
     def __call__(self, request: PreparedRequest) -> PreparedRequest:
         self.handle_auth(request)
@@ -51,8 +50,8 @@ def handle_auth(self, request: PreparedRequest) -> None:
     def authenticate(self, request: PreparedRequest):
         assert request.url is not None
         url = urlparse(request.url)
-        base_url = f"{url.scheme}://{url.netloc}"
-        self.token = self.get_token(base_url, self.login)
+        base_url = f"{url.scheme}://{url.netloc}"  # noqa: E231
+        self.token = self.get_token(base_url, self.login, self.logger, self.verify)
 
     def build_digest_header(self, request: PreparedRequest) -> None:
         header = {
@@ -62,14 +61,16 @@ def build_digest_header(self, request: PreparedRequest) -> None:
         request.headers.update(header)
 
     @staticmethod
-    def get_token(base_url: str, apigw_login: ApiGwLogin) -> str:
+    def get_token(base_url: str, apigw_login: ApiGwLogin, logger: Optional[logging.Logger] = None, verify=False) -> str:
         try:
             response = post(
                 url=f"{base_url}/apigw/login",
-                verify=False,
+                verify=verify,
                 json=apigw_login.model_dump(exclude_none=True),
                 timeout=10,
             )
+            if logger is not None:
+                logger.debug(auth_response_debug(response))
             response.raise_for_status()
             token = response.json()["token"]
         except JSONDecodeError:
@@ -86,8 +87,8 @@ def get_token(base_url: str, apigw_login: ApiGwLogin) -> str:
     def __str__(self) -> str:
         return f"ApiGatewayAuth(mode={self.login.mode})"
 
-    def logout(self, session: "ManagerSession") -> Optional[ManagerResponse]:
+    def logout(self, client: APIEndpointClient) -> None:
         return None
 
-    def clear_tokens_and_cookies(self) -> None:
+    def clear(self) -> None:
         self.token = ""
diff --git a/catalystwan/dataclasses.py b/catalystwan/dataclasses.py
@@ -401,13 +401,22 @@ class RadiusServer(DataclassBase):
     address: str = field(metadata={FIELD_NAME: "address"})
     auth_port: int = field(metadata={FIELD_NAME: "authPort"})
     acct_port: int = field(metadata={FIELD_NAME: "acctPort"})
-    vpn: int = field(metadata={FIELD_NAME: "vpn"})
-    vpn_ip_subnet: str = field(metadata={FIELD_NAME: "vpnIpSubnet"})
+    vpn: Optional[int] = field(metadata={FIELD_NAME: "vpn", "description": "required field  < 20.16"})
+    vpn_ip_subnet: Optional[str] = field(metadata={FIELD_NAME: "vpnIpSubnet", "description": "required field  < 20.16"})
     key: str = field(metadata={FIELD_NAME: "key"})
     secret_key: str = field(metadata={FIELD_NAME: "secretKey"})
     priority: int = field(metadata={FIELD_NAME: "priority"})
 
 
+@define(frozen=True)
+class ExtendedRadiusServer(RadiusServer):
+    """
+    Extended RADIUS server with additional fields.
+    """
+
+    source_vpn: Optional[int] = field(default=None, metadata={FIELD_NAME: "sourceVpn"})
+
+
 @define(frozen=True)
 class TenantRadiusServer(DataclassBase):
     """
@@ -427,13 +436,22 @@ class TacacsServer(DataclassBase):
 
     address: str = field(metadata={FIELD_NAME: "address"})
     auth_port: int = field(metadata={FIELD_NAME: "authPort"})
-    vpn: int = field(metadata={FIELD_NAME: "vpn"})
-    vpn_ip_subnet: str = field(metadata={FIELD_NAME: "vpnIpSubnet"})
+    vpn: Optional[int] = field(metadata={FIELD_NAME: "vpn", "description": "required field  < 20.16"})
+    vpn_ip_subnet: Optional[str] = field(metadata={FIELD_NAME: "vpnIpSubnet", "description": "required field  < 20.16"})
     key: str = field(metadata={FIELD_NAME: "key"})
     secret_key: str = field(metadata={FIELD_NAME: "secretKey"})
     priority: int = field(metadata={FIELD_NAME: "priority"})
 
 
+@define(frozen=True)
+class ExtendedTacacsServer(TacacsServer):
+    """
+    Extended TACACS server with additional fields.
+    """
+
+    source_vpn: Optional[int] = field(default=None, metadata={FIELD_NAME: "sourceVpn"})
+
+
 @define(frozen=True)
 class TenantTacacsServer(DataclassBase):
     """

diff --git a/catalystwan/endpoints/certificate_management_device.py b/catalystwan/endpoints/certificate_management_device.py
@@ -94,6 +94,7 @@ class DeviceCsrGenerationResponse(BaseModel):
 class Validity(str, Enum):
     VALID = "valid"
     INVALID = "invalid"
+    STAGING = "staging"
 
 
 class VedgeListValidityPayload(BaseModel):

diff --git a/catalystwan/response.py b/catalystwan/response.py
@@ -4,6 +4,7 @@
 from datetime import datetime
 from email.utils import parsedate_to_datetime
 from functools import wraps
+from os import environ
 from pprint import pformat
 from typing import Any, Callable, Dict, Optional, Sequence, Type, TypeVar, Union, cast
 from urllib.parse import urlparse
@@ -104,6 +105,14 @@ def response_history_debug(response: Optional[Response], request: Union[Request,
     return "\n".join(response_debugs)
 
 
+def auth_response_debug(response: Response, title: str = "Auth") -> str:
+    if environ.get("catalystwan_auth_trace") is not None:
+        return response_history_debug(response, None)
+    return ", ".join(
+        [f"{title}: {r.request.method} {r.request.url} <{r.status_code}>" for r in response.history + [response]]
+    )
+
+
 def parse_cookies_to_dict(cookies: str) -> Dict[str, str]:
     """Utility method to parse cookie string into dict"""
     result: Dict[str, str] = {}

diff --git a/catalystwan/session.py b/catalystwan/session.py
@@ -210,8 +210,6 @@ class ManagerSession(ManagerResponseAdapter, APIEndpointClient):
         logger: override default module logger
 
     Attributes:
-        enable_relogin (bool): defaults to True, in case that session is not properly logged-in, session will try to
-            relogin and try the same request again
         api: APIContainer: container for API methods
         endpoints: APIEndpointContainter: container for API endpoints
         state: ManagerSessionState: current state of the session can be used to control session flow
@@ -237,7 +235,6 @@ def __init__(
         self._session_type = SessionType.NOT_DEFINED
         self.server_name: Optional[str] = None
         self.logger = logger or logging.getLogger(__name__)
-        self.enable_relogin: bool = True
         self.response_trace: Callable[
             [Optional[Response], Union[Request, PreparedRequest, None]], str
         ] = response_history_debug
@@ -314,7 +311,7 @@ def login(self) -> ManagerSession:
             ManagerSession: (self)
         """
         self.cookies.clear_session_cookies()
-        self._auth.clear_tokens_and_cookies()
+        self._auth.clear()
         self.auth = self._auth
         if self.subdomain:
             tenant_id = self.get_tenant_id()
@@ -346,14 +343,8 @@ def login(self) -> ManagerSession:
         self.logger.info(
             f"Logged to vManage({self.platform_version}) as {self.auth}. The session type is {self.session_type}"
         )
-        self._set_jsessionid(self.auth)
         return self
 
-    def _set_jsessionid(self, auth: Union[vManageAuth, ApiGwAuth]) -> None:
-        if isinstance(auth, vManageAuth):
-            if jsessionid := auth.jsessionid:
-                self.cookies.set("JSESSIONID", jsessionid)
-
     def wait_server_ready(self, timeout: int, poll_period: int = 10) -> None:
         """Waits until server is ready for API requests with given timeout in seconds"""
 
@@ -424,16 +415,6 @@ def request(self, method, url, *args, **kwargs) -> ManagerResponse:
             self.logger.debug(exception)
             raise ManagerRequestException(*exception.args, request=exception.request, response=exception.response)
 
-        if self.enable_relogin and response.jsessionid_expired and self.state == ManagerSessionState.OPERATIVE:
-            self.logger.warning("Logging to session. Reason: expired JSESSIONID detected in response headers")
-            self.state = ManagerSessionState.LOGIN
-            return self.request(method, url, *args, **_kwargs)
-
-        if self.enable_relogin and response.api_gw_unauthorized and self.state == ManagerSessionState.OPERATIVE:
-            self.logger.warning("Logging to API GW session. Reason: unauthorized detected in response headers")
-            self.state = ManagerSessionState.LOGIN
-            return self.request(method, url, *args, **_kwargs)
-
         if response.request.url and "passwordReset.html" in response.request.url:
             raise DefaultPasswordError("Password must be changed to use this session.")
 
@@ -511,8 +492,8 @@ def get_virtual_session_id(self, tenant_id: str) -> str:
         response = self.post(url_path)
         return response.json()["VSessionId"]
 
-    def logout(self) -> Optional[ManagerResponse]:
-        return self._auth.logout(self)
+    def logout(self) -> None:
+        self._auth.logout(self)
 
     def close(self) -> None:
         """Closes the ManagerSession.
@@ -554,7 +535,12 @@ def validate_responses(self, value: bool):
         self._validate_responses = value
 
     def __copy__(self) -> ManagerSession:
-        return ManagerSession(base_url=self.base_url, auth=self._auth, subdomain=self.subdomain, logger=self.logger)
+        return ManagerSession(
+            base_url=self.base_url,
+            auth=self._auth,
+            subdomain=self.subdomain,
+            logger=self.logger,
+        )
 
     def __str__(self) -> str:
         return f"ManagerSession(session_type={self.session_type}, auth={self._auth})"