From 883a0fc421c2b94bdfec08bde085215d4cd2d4f3 Mon Sep 17 00:00:00 2001 From: aadityasinha-dotcom Date: Mon, 26 Jun 2023 10:22:54 +0530 Subject: [PATCH 01/33] moved the validation logic into ProfileManager Signed-off-by: aadityasinha-dotcom --- .../zowe/core_for_zowe_sdk/config_file.py | 15 +++++++ .../zowe/core_for_zowe_sdk/profile_manager.py | 40 +++++++++++++++++++ 2 files changed, 55 insertions(+) diff --git a/src/core/zowe/core_for_zowe_sdk/config_file.py b/src/core/zowe/core_for_zowe_sdk/config_file.py index 67eb33bf..471da92f 100644 --- a/src/core/zowe/core_for_zowe_sdk/config_file.py +++ b/src/core/zowe/core_for_zowe_sdk/config_file.py @@ -77,6 +77,7 @@ class ConfigFile: _location: Optional[str] = None profiles: Optional[dict] = None defaults: Optional[dict] = None + schema_property: Optional[dict] = None secure_props: Optional[dict] = None _missing_secure_props: list = field(default_factory=list) @@ -120,6 +121,7 @@ def init_from_file(self) -> None: profile_jsonc = commentjson.load(fileobj) self.profiles = profile_jsonc.get("profiles", {}) + self.schema_property = profile_jsonc.get("$schema", None) self.defaults = profile_jsonc.get("defaults", {}) # loading secure props is done in load_profile_properties @@ -294,6 +296,19 @@ def load_profile_properties(self, profile_name: str) -> dict: return props + def load_schema(self) -> str: + """ + Find the schema file path from the $schema_property in the config.json + Returns + ------- + Str + + file_path to the schema property + """ + + file_path = self.schema_property + return file_path + def load_secure_props(self) -> None: """ load secure_props stored for the given config file diff --git a/src/core/zowe/core_for_zowe_sdk/profile_manager.py b/src/core/zowe/core_for_zowe_sdk/profile_manager.py index 1e6d49ca..5a2af1de 100644 --- a/src/core/zowe/core_for_zowe_sdk/profile_manager.py +++ b/src/core/zowe/core_for_zowe_sdk/profile_manager.py @@ -15,6 +15,7 @@ from typing import Optional from .config_file import ConfigFile, Profile +from .validators import validate_config_json from .custom_warnings import ( ConfigNotFoundWarning, ProfileNotFoundWarning, @@ -169,12 +170,45 @@ def get_profile( ) finally: return cfg_profile + + @staticmethod + def load_schema( + cfg: ConfigFile, + path_config_json: str, + opt_in: Optional[bool] = True, + ) -> str: + """ + Get the $schema_property from the config and load the schema + + Returns + ------- + file_path to the $schema property + """ + + path_schema_json = None + try: + path_schema_json = cfg.load_schema() + if path_schema_json is None: # check if the $schema property is not defined + warnings.warn( + f"$schema property could not found" + ) + + # validate the $schema property + if path_schema_json and opt_in: + validate_config_json(path_config_json, path_schema_json) + except Exception as exc: + warnings.warn( + f"Could not validate $schema property, {exc}" + ) + finally: + return path_schema_json def load( self, profile_name: Optional[str] = None, profile_type: Optional[str] = None, check_missing_props: bool = True, + opt_in: Optional[bool] = True, ) -> dict: """Load connection details from a team config profile. Returns @@ -209,6 +243,7 @@ def load( "Global Config": self.global_config, } profile_props: dict = {} + schema_path = None missing_secure_props = [] # track which secure props were not loaded @@ -224,6 +259,11 @@ def load( ) # Define profile name that will be merged from other layers profile_props = {**profile_loaded.data, **profile_props} + # Loading $schema property from project config + if config_type in ("Project Config"): + path_config_json = cfg._location + "/zowe.config.json" + schema_path = self.load_schema(cfg, path_config_json, opt_in) + missing_secure_props.extend(profile_loaded.missing_secure_props) if i == 1 and profile_props: From e83fb65289f19d9e1f8cb2c021f602e3cb24873f Mon Sep 17 00:00:00 2001 From: aadityasinha-dotcom Date: Mon, 26 Jun 2023 21:35:15 +0530 Subject: [PATCH 02/33] added validation error warning Signed-off-by: aadityasinha-dotcom --- src/core/zowe/core_for_zowe_sdk/profile_manager.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/core/zowe/core_for_zowe_sdk/profile_manager.py b/src/core/zowe/core_for_zowe_sdk/profile_manager.py index 5a2af1de..43d52039 100644 --- a/src/core/zowe/core_for_zowe_sdk/profile_manager.py +++ b/src/core/zowe/core_for_zowe_sdk/profile_manager.py @@ -12,6 +12,7 @@ import os.path import warnings +import jsonschema from typing import Optional from .config_file import ConfigFile, Profile @@ -196,7 +197,7 @@ def load_schema( # validate the $schema property if path_schema_json and opt_in: validate_config_json(path_config_json, path_schema_json) - except Exception as exc: + except jsonschema.ValidationError as exc: warnings.warn( f"Could not validate $schema property, {exc}" ) From ffc42e896213637ee8cc03021a982eb417f9ef0f Mon Sep 17 00:00:00 2001 From: aadityasinha-dotcom Date: Tue, 27 Jun 2023 10:55:22 +0530 Subject: [PATCH 03/33] added jsonschema exceptions Signed-off-by: aadityasinha-dotcom --- src/core/zowe/core_for_zowe_sdk/config_file.py | 1 + src/core/zowe/core_for_zowe_sdk/profile_manager.py | 14 +++++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/core/zowe/core_for_zowe_sdk/config_file.py b/src/core/zowe/core_for_zowe_sdk/config_file.py index 471da92f..66eef0a6 100644 --- a/src/core/zowe/core_for_zowe_sdk/config_file.py +++ b/src/core/zowe/core_for_zowe_sdk/config_file.py @@ -69,6 +69,7 @@ class ConfigFile: 4. Contents of the file. 4.1 Profiles 4.2 Defaults + 4.3 Schema Property 5. Secure Properties associated with the file. """ diff --git a/src/core/zowe/core_for_zowe_sdk/profile_manager.py b/src/core/zowe/core_for_zowe_sdk/profile_manager.py index 43d52039..0a9f3800 100644 --- a/src/core/zowe/core_for_zowe_sdk/profile_manager.py +++ b/src/core/zowe/core_for_zowe_sdk/profile_manager.py @@ -199,7 +199,19 @@ def load_schema( validate_config_json(path_config_json, path_schema_json) except jsonschema.ValidationError as exc: warnings.warn( - f"Could not validate $schema property, {exc}" + f"Instance was invalid under the provided $schema property, {exc}" + ) + except jsonschema.FormatError: + warnings.warn( + f"Validating a format {path_config_json} failed for {path_schema_json}" + ) + except jsonschema.UndefinedTypeCheck as exc: + warnings.warn( + f"A type checker was asked to check a type it did not have registered, {exc}" + ) + except jsonschema.UnknownType: + warnings.warn( + f"Unknown type is found in {path_schema_json}" ) finally: return path_schema_json From 267f7004e613b951b32bfde96d258b00693673f0 Mon Sep 17 00:00:00 2001 From: aadityasinha-dotcom Date: Tue, 27 Jun 2023 18:59:18 +0530 Subject: [PATCH 04/33] fixed indentation Signed-off-by: aadityasinha-dotcom --- src/core/zowe/core_for_zowe_sdk/profile_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/zowe/core_for_zowe_sdk/profile_manager.py b/src/core/zowe/core_for_zowe_sdk/profile_manager.py index 0a9f3800..7ebd775f 100644 --- a/src/core/zowe/core_for_zowe_sdk/profile_manager.py +++ b/src/core/zowe/core_for_zowe_sdk/profile_manager.py @@ -274,8 +274,8 @@ def load( # Loading $schema property from project config if config_type in ("Project Config"): - path_config_json = cfg._location + "/zowe.config.json" - schema_path = self.load_schema(cfg, path_config_json, opt_in) + path_config_json = cfg._location + "/zowe.config.json" + schema_path = self.load_schema(cfg, path_config_json, opt_in) missing_secure_props.extend(profile_loaded.missing_secure_props) From 6465b672a09d887dc075633dd875edcdf39099ce Mon Sep 17 00:00:00 2001 From: aadityasinha-dotcom Date: Wed, 28 Jun 2023 11:23:13 +0530 Subject: [PATCH 05/33] raised error Signed-off-by: aadityasinha-dotcom --- src/core/zowe/core_for_zowe_sdk/profile_manager.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/core/zowe/core_for_zowe_sdk/profile_manager.py b/src/core/zowe/core_for_zowe_sdk/profile_manager.py index 7ebd775f..68090373 100644 --- a/src/core/zowe/core_for_zowe_sdk/profile_manager.py +++ b/src/core/zowe/core_for_zowe_sdk/profile_manager.py @@ -198,23 +198,21 @@ def load_schema( if path_schema_json and opt_in: validate_config_json(path_config_json, path_schema_json) except jsonschema.ValidationError as exc: - warnings.warn( + raise Exception( f"Instance was invalid under the provided $schema property, {exc}" ) except jsonschema.FormatError: - warnings.warn( + raise Exception( f"Validating a format {path_config_json} failed for {path_schema_json}" ) except jsonschema.UndefinedTypeCheck as exc: - warnings.warn( + raise Exception( f"A type checker was asked to check a type it did not have registered, {exc}" ) except jsonschema.UnknownType: - warnings.warn( + raise Exception( f"Unknown type is found in {path_schema_json}" ) - finally: - return path_schema_json def load( self, From b93d818f6c4674c4065026887ae38ef273254dc2 Mon Sep 17 00:00:00 2001 From: aadityasinha-dotcom Date: Wed, 28 Jun 2023 12:17:39 +0530 Subject: [PATCH 06/33] fixed test Signed-off-by: aadityasinha-dotcom --- src/core/zowe/core_for_zowe_sdk/profile_manager.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/core/zowe/core_for_zowe_sdk/profile_manager.py b/src/core/zowe/core_for_zowe_sdk/profile_manager.py index 68090373..a2db9aeb 100644 --- a/src/core/zowe/core_for_zowe_sdk/profile_manager.py +++ b/src/core/zowe/core_for_zowe_sdk/profile_manager.py @@ -213,6 +213,8 @@ def load_schema( raise Exception( f"Unknown type is found in {path_schema_json}" ) + finally: + return path_schema_json def load( self, From 69d983f335488b3b618301a84bb7cf5b67482aa8 Mon Sep 17 00:00:00 2001 From: aadityasinha-dotcom Date: Thu, 29 Jun 2023 17:35:19 +0530 Subject: [PATCH 07/33] enhanced the validators.py and made some changes to validate schema Signed-off-by: aadityasinha-dotcom --- src/core/zowe/core_for_zowe_sdk/config_file.py | 17 ++++------------- .../zowe/core_for_zowe_sdk/profile_manager.py | 13 ++++++++----- src/core/zowe/core_for_zowe_sdk/validators.py | 16 +++++++++++++--- 3 files changed, 25 insertions(+), 21 deletions(-) diff --git a/src/core/zowe/core_for_zowe_sdk/config_file.py b/src/core/zowe/core_for_zowe_sdk/config_file.py index 66eef0a6..f59e82f8 100644 --- a/src/core/zowe/core_for_zowe_sdk/config_file.py +++ b/src/core/zowe/core_for_zowe_sdk/config_file.py @@ -102,6 +102,10 @@ def filepath(self) -> Optional[str]: @property def location(self) -> Optional[str]: return self._location + + @property + def schema_path(self) -> Optional[str]: + return self.schema_property @location.setter def location(self, dirname: str) -> None: @@ -297,19 +301,6 @@ def load_profile_properties(self, profile_name: str) -> dict: return props - def load_schema(self) -> str: - """ - Find the schema file path from the $schema_property in the config.json - Returns - ------- - Str - - file_path to the schema property - """ - - file_path = self.schema_property - return file_path - def load_secure_props(self) -> None: """ load secure_props stored for the given config file diff --git a/src/core/zowe/core_for_zowe_sdk/profile_manager.py b/src/core/zowe/core_for_zowe_sdk/profile_manager.py index a2db9aeb..2f9e3fdf 100644 --- a/src/core/zowe/core_for_zowe_sdk/profile_manager.py +++ b/src/core/zowe/core_for_zowe_sdk/profile_manager.py @@ -173,7 +173,7 @@ def get_profile( return cfg_profile @staticmethod - def load_schema( + def validate_schema( cfg: ConfigFile, path_config_json: str, opt_in: Optional[bool] = True, @@ -188,7 +188,7 @@ def load_schema( path_schema_json = None try: - path_schema_json = cfg.load_schema() + path_schema_json = cfg.schema_path if path_schema_json is None: # check if the $schema property is not defined warnings.warn( f"$schema property could not found" @@ -272,10 +272,13 @@ def load( ) # Define profile name that will be merged from other layers profile_props = {**profile_loaded.data, **profile_props} - # Loading $schema property from project config - if config_type in ("Project Config"): + # Validating $schema property for all the layers + if config_type in ("Project Config", "Global Config"): path_config_json = cfg._location + "/zowe.config.json" - schema_path = self.load_schema(cfg, path_config_json, opt_in) + self.validate_schema(cfg, path_config_json, opt_in) + elif config_type in ("Project User Config", "Global User Config"): + path_config_json = cfg._location + "/zowe.user.config.json" + self.validate_schema(cfg, path_config_json, opt_in) missing_secure_props.extend(profile_loaded.missing_secure_props) diff --git a/src/core/zowe/core_for_zowe_sdk/validators.py b/src/core/zowe/core_for_zowe_sdk/validators.py index c1a11719..16deeda5 100644 --- a/src/core/zowe/core_for_zowe_sdk/validators.py +++ b/src/core/zowe/core_for_zowe_sdk/validators.py @@ -12,6 +12,8 @@ import commentjson from jsonschema import validate +import os +import requests def validate_config_json(path_config_json: str, path_schema_json: str): @@ -31,10 +33,18 @@ def validate_config_json(path_config_json: str, path_schema_json: str): Provides details if config.json doesn't match schema.json, otherwise it returns None. """ + # checks if the path_schema_json point to an internet URI and download the schema using the URI + if path_schema_json[:8] in ("https://", "http://"): + response = requests.get(path_schema_json) + schema_data = response.content.decode("utf-8") + schema_json = commentjson.load(schema_data) + + # checks if the path_schema_json is a file + elif os.path.isfile(path_schema_json): + with open(path_schema_json) as file: + schema_json = commentjson.load(file) + with open(path_config_json) as file: config_json = commentjson.load(file) - with open(path_schema_json) as file: - schema_json = commentjson.load(file) - return validate(instance=config_json, schema=schema_json) From f07ebec0f319d787c599d9567db7f141a6fff6cf Mon Sep 17 00:00:00 2001 From: aadityasinha-dotcom Date: Thu, 29 Jun 2023 17:42:06 +0530 Subject: [PATCH 08/33] fixed tests Signed-off-by: aadityasinha-dotcom --- src/core/zowe/core_for_zowe_sdk/profile_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/zowe/core_for_zowe_sdk/profile_manager.py b/src/core/zowe/core_for_zowe_sdk/profile_manager.py index 2f9e3fdf..19c544f5 100644 --- a/src/core/zowe/core_for_zowe_sdk/profile_manager.py +++ b/src/core/zowe/core_for_zowe_sdk/profile_manager.py @@ -273,10 +273,10 @@ def load( profile_props = {**profile_loaded.data, **profile_props} # Validating $schema property for all the layers - if config_type in ("Project Config", "Global Config"): + if cfg._location and config_type in ("Project Config", "Global Config"): path_config_json = cfg._location + "/zowe.config.json" self.validate_schema(cfg, path_config_json, opt_in) - elif config_type in ("Project User Config", "Global User Config"): + elif cfg._location and config_type in ("Project User Config", "Global User Config"): path_config_json = cfg._location + "/zowe.user.config.json" self.validate_schema(cfg, path_config_json, opt_in) From 8f8d80c4272bd29b56fcc24882bc1e3d7d54b43a Mon Sep 17 00:00:00 2001 From: aadityasinha-dotcom Date: Fri, 30 Jun 2023 13:43:50 +0530 Subject: [PATCH 09/33] fixed validator if there is not path, fixed raising exceptions for validation errors Signed-off-by: aadityasinha-dotcom --- .../zowe/core_for_zowe_sdk/profile_manager.py | 28 ++++++++++--------- src/core/zowe/core_for_zowe_sdk/validators.py | 4 +++ 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/src/core/zowe/core_for_zowe_sdk/profile_manager.py b/src/core/zowe/core_for_zowe_sdk/profile_manager.py index 19c544f5..e853f505 100644 --- a/src/core/zowe/core_for_zowe_sdk/profile_manager.py +++ b/src/core/zowe/core_for_zowe_sdk/profile_manager.py @@ -197,24 +197,26 @@ def validate_schema( # validate the $schema property if path_schema_json and opt_in: validate_config_json(path_config_json, path_schema_json) - except jsonschema.ValidationError as exc: - raise Exception( + except jsonschema.exceptions.ValidationError as exc: + raise jsonschema.exceptions.ValidationError( f"Instance was invalid under the provided $schema property, {exc}" ) - except jsonschema.FormatError: - raise Exception( - f"Validating a format {path_config_json} failed for {path_schema_json}" + except jsonschema.exceptions.SchemaError as exc: + raise jsonschema.exception.SchemaError( + f"The provided schema is invalid, {exc}" ) - except jsonschema.UndefinedTypeCheck as exc: - raise Exception( + except jsonschema.exceptions.UndefinedTypeCheck as exc: + raise jsonschema.exceptions.UndefinedTypeCheck( f"A type checker was asked to check a type it did not have registered, {exc}" ) - except jsonschema.UnknownType: - raise Exception( - f"Unknown type is found in {path_schema_json}" + except jsonschema.exceptions.UnknownType as exc: + raise jsonschema.exceptions.UnknownType( + f"Unknown type is found in {path_schema_json}, exc" + ) + except jsonschema.exceptions.FormatError as exc: + raise jsonschema.exceptions.FormatError( + f"Validating a format {path_config_json} failed for {path_schema_json}, {exc}" ) - finally: - return path_schema_json def load( self, @@ -277,7 +279,7 @@ def load( path_config_json = cfg._location + "/zowe.config.json" self.validate_schema(cfg, path_config_json, opt_in) elif cfg._location and config_type in ("Project User Config", "Global User Config"): - path_config_json = cfg._location + "/zowe.user.config.json" + path_config_json = cfg._location + "/zowe.config.user.json" self.validate_schema(cfg, path_config_json, opt_in) missing_secure_props.extend(profile_loaded.missing_secure_props) diff --git a/src/core/zowe/core_for_zowe_sdk/validators.py b/src/core/zowe/core_for_zowe_sdk/validators.py index 16deeda5..9a29f7fe 100644 --- a/src/core/zowe/core_for_zowe_sdk/validators.py +++ b/src/core/zowe/core_for_zowe_sdk/validators.py @@ -43,6 +43,10 @@ def validate_config_json(path_config_json: str, path_schema_json: str): elif os.path.isfile(path_schema_json): with open(path_schema_json) as file: schema_json = commentjson.load(file) + + # if there is no path_schema_json it will return None + else: + return None with open(path_config_json) as file: config_json = commentjson.load(file) From c2ae1ec7080d5e31dcd6a7e663269b5c8989c5c5 Mon Sep 17 00:00:00 2001 From: aadityasinha-dotcom Date: Thu, 6 Jul 2023 18:04:54 +0530 Subject: [PATCH 10/33] moved the validate_schema method to ConfigFile class Signed-off-by: aadityasinha-dotcom --- .../zowe/core_for_zowe_sdk/config_file.py | 47 +++++++++++++++++ .../zowe/core_for_zowe_sdk/profile_manager.py | 52 +------------------ src/core/zowe/core_for_zowe_sdk/validators.py | 18 ++++--- 3 files changed, 59 insertions(+), 58 deletions(-) diff --git a/src/core/zowe/core_for_zowe_sdk/config_file.py b/src/core/zowe/core_for_zowe_sdk/config_file.py index f59e82f8..04ad098c 100644 --- a/src/core/zowe/core_for_zowe_sdk/config_file.py +++ b/src/core/zowe/core_for_zowe_sdk/config_file.py @@ -14,6 +14,7 @@ import os.path import re import sys +import jsonschema import warnings from dataclasses import dataclass, field from typing import Optional, NamedTuple @@ -21,6 +22,7 @@ import commentjson from .constants import constants +from .validators import validate_config_json from .custom_warnings import ( ProfileNotFoundWarning, ProfileParsingWarning, @@ -134,6 +136,51 @@ def init_from_file(self) -> None: # we know that the profile has saved properties # self.load_secure_props() + def validate_schema( + self, + path_config_json: str, + opt_in: Optional[bool] = True, + ) -> None: + """ + Get the $schema_property from the config and load the schema + + Returns + ------- + file_path to the $schema property + """ + + path_schema_json = None + try: + path_schema_json = self.schema_path + if path_schema_json is None: # check if the $schema property is not defined + warnings.warn( + f"$schema property could not found" + ) + + # validate the $schema property + if path_schema_json and opt_in: + validate_config_json(path_config_json, path_schema_json) + except jsonschema.exceptions.ValidationError as exc: + raise jsonschema.exceptions.ValidationError( + f"Instance was invalid under the provided $schema property, {exc}" + ) + except jsonschema.exceptions.SchemaError as exc: + raise jsonschema.exception.SchemaError( + f"The provided schema is invalid, {exc}" + ) + except jsonschema.exceptions.UndefinedTypeCheck as exc: + raise jsonschema.exceptions.UndefinedTypeCheck( + f"A type checker was asked to check a type it did not have registered, {exc}" + ) + except jsonschema.exceptions.UnknownType as exc: + raise jsonschema.exceptions.UnknownType( + f"Unknown type is found in {path_schema_json}, exc" + ) + except jsonschema.exceptions.FormatError as exc: + raise jsonschema.exceptions.FormatError( + f"Validating a format {path_config_json} failed for {path_schema_json}, {exc}" + ) + def get_profile( self, profile_name: Optional[str] = None, diff --git a/src/core/zowe/core_for_zowe_sdk/profile_manager.py b/src/core/zowe/core_for_zowe_sdk/profile_manager.py index e853f505..c2af7771 100644 --- a/src/core/zowe/core_for_zowe_sdk/profile_manager.py +++ b/src/core/zowe/core_for_zowe_sdk/profile_manager.py @@ -12,11 +12,9 @@ import os.path import warnings -import jsonschema from typing import Optional from .config_file import ConfigFile, Profile -from .validators import validate_config_json from .custom_warnings import ( ConfigNotFoundWarning, ProfileNotFoundWarning, @@ -171,52 +169,6 @@ def get_profile( ) finally: return cfg_profile - - @staticmethod - def validate_schema( - cfg: ConfigFile, - path_config_json: str, - opt_in: Optional[bool] = True, - ) -> str: - """ - Get the $schema_property from the config and load the schema - - Returns - ------- - file_path to the $schema property - """ - - path_schema_json = None - try: - path_schema_json = cfg.schema_path - if path_schema_json is None: # check if the $schema property is not defined - warnings.warn( - f"$schema property could not found" - ) - - # validate the $schema property - if path_schema_json and opt_in: - validate_config_json(path_config_json, path_schema_json) - except jsonschema.exceptions.ValidationError as exc: - raise jsonschema.exceptions.ValidationError( - f"Instance was invalid under the provided $schema property, {exc}" - ) - except jsonschema.exceptions.SchemaError as exc: - raise jsonschema.exception.SchemaError( - f"The provided schema is invalid, {exc}" - ) - except jsonschema.exceptions.UndefinedTypeCheck as exc: - raise jsonschema.exceptions.UndefinedTypeCheck( - f"A type checker was asked to check a type it did not have registered, {exc}" - ) - except jsonschema.exceptions.UnknownType as exc: - raise jsonschema.exceptions.UnknownType( - f"Unknown type is found in {path_schema_json}, exc" - ) - except jsonschema.exceptions.FormatError as exc: - raise jsonschema.exceptions.FormatError( - f"Validating a format {path_config_json} failed for {path_schema_json}, {exc}" - ) def load( self, @@ -277,10 +229,10 @@ def load( # Validating $schema property for all the layers if cfg._location and config_type in ("Project Config", "Global Config"): path_config_json = cfg._location + "/zowe.config.json" - self.validate_schema(cfg, path_config_json, opt_in) + cfg.validate_schema(path_config_json, opt_in) elif cfg._location and config_type in ("Project User Config", "Global User Config"): path_config_json = cfg._location + "/zowe.config.user.json" - self.validate_schema(cfg, path_config_json, opt_in) + cfg.validate_schema(path_config_json, opt_in) missing_secure_props.extend(profile_loaded.missing_secure_props) diff --git a/src/core/zowe/core_for_zowe_sdk/validators.py b/src/core/zowe/core_for_zowe_sdk/validators.py index 9a29f7fe..a19e8388 100644 --- a/src/core/zowe/core_for_zowe_sdk/validators.py +++ b/src/core/zowe/core_for_zowe_sdk/validators.py @@ -13,10 +13,11 @@ import commentjson from jsonschema import validate import os +from typing import Union import requests -def validate_config_json(path_config_json: str, path_schema_json: str): +def validate_config_json(path_config_json: Union[str, dict], path_schema_json: str): """ Function validating that zowe.config.json file matches zowe.schema.json. @@ -34,10 +35,8 @@ def validate_config_json(path_config_json: str, path_schema_json: str): """ # checks if the path_schema_json point to an internet URI and download the schema using the URI - if path_schema_json[:8] in ("https://", "http://"): - response = requests.get(path_schema_json) - schema_data = response.content.decode("utf-8") - schema_json = commentjson.load(schema_data) + if path_schema_json.startswith("https://") or path_schema_json.startswith("http://"): + schema_json = requests.get(path_schema_json).json() # checks if the path_schema_json is a file elif os.path.isfile(path_schema_json): @@ -47,8 +46,11 @@ def validate_config_json(path_config_json: str, path_schema_json: str): # if there is no path_schema_json it will return None else: return None - - with open(path_config_json) as file: - config_json = commentjson.load(file) + + if isinstance(path_config_json, str): + with open(path_config_json) as file: + config_json = commentjson.load(file) + else: + config_json = path_config_json return validate(instance=config_json, schema=schema_json) From 6108ebf649322f1fde5067cf50abca0081c2e548 Mon Sep 17 00:00:00 2001 From: aadityasinha-dotcom Date: Fri, 7 Jul 2023 10:04:09 +0530 Subject: [PATCH 11/33] changelog Signed-off-by: aadityasinha-dotcom --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c57aa5de..1ce731fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,3 +2,6 @@ All notable changes to the Zowe Client Python SDK will be documented in this file. +## Recent Changes + +- BugFix: Validation of zowe.config.json file matching the schema [#192](https://github.com/zowe/zowe-client-python-sdk/issues/192) \ No newline at end of file From 042a4ac2c54c8c37494600822bd41de993ca951e Mon Sep 17 00:00:00 2001 From: aadityasinha-dotcom Date: Sun, 9 Jul 2023 21:26:38 +0530 Subject: [PATCH 12/33] moved the validation logic to init_from_file so the validate_schema method will be called automatically Signed-off-by: aadityasinha-dotcom --- .../zowe/core_for_zowe_sdk/config_file.py | 61 ++++++++----------- .../zowe/core_for_zowe_sdk/profile_manager.py | 39 +++++++++--- 2 files changed, 57 insertions(+), 43 deletions(-) diff --git a/src/core/zowe/core_for_zowe_sdk/config_file.py b/src/core/zowe/core_for_zowe_sdk/config_file.py index 04ad098c..0fe850b2 100644 --- a/src/core/zowe/core_for_zowe_sdk/config_file.py +++ b/src/core/zowe/core_for_zowe_sdk/config_file.py @@ -14,7 +14,6 @@ import os.path import re import sys -import jsonschema import warnings from dataclasses import dataclass, field from typing import Optional, NamedTuple @@ -116,7 +115,11 @@ def location(self, dirname: str) -> None: else: raise FileNotFoundError(f"given path {dirname} is not valid") - def init_from_file(self) -> None: + def init_from_file( + self, + config_type: str, + opt_in: Optional[bool] = True + ) -> None: """ Initializes the class variable after setting filepath (or if not set, autodiscover the file) @@ -131,6 +134,8 @@ def init_from_file(self) -> None: self.schema_property = profile_jsonc.get("$schema", None) self.defaults = profile_jsonc.get("defaults", {}) + if self.schema_property: + self.validate_schema(config_type, opt_in) # loading secure props is done in load_profile_properties # since we want to try loading secure properties only when # we know that the profile has saved properties @@ -138,7 +143,7 @@ def init_from_file(self) -> None: def validate_schema( self, - path_config_json: str, + config_type: str, opt_in: Optional[bool] = True, ) -> None: """ @@ -149,42 +154,30 @@ def validate_schema( file_path to the $schema property """ - path_schema_json = None - try: - path_schema_json = self.schema_path - if path_schema_json is None: # check if the $schema property is not defined - warnings.warn( - f"$schema property could not found" - ) - - # validate the $schema property - if path_schema_json and opt_in: - validate_config_json(path_config_json, path_schema_json) - except jsonschema.exceptions.ValidationError as exc: - raise jsonschema.exceptions.ValidationError( - f"Instance was invalid under the provided $schema property, {exc}" - ) - except jsonschema.exceptions.SchemaError as exc: - raise jsonschema.exception.SchemaError( - f"The provided schema is invalid, {exc}" - ) - except jsonschema.exceptions.UndefinedTypeCheck as exc: - raise jsonschema.exceptions.UndefinedTypeCheck( - f"A type checker was asked to check a type it did not have registered, {exc}" - ) - except jsonschema.exceptions.UnknownType as exc: - raise jsonschema.exceptions.UnknownType( - f"Unknown type is found in {path_schema_json}, exc" - ) - except jsonschema.exceptions.FormatError as exc: - raise jsonschema.exceptions.FormatError( - f"Validating a format {path_config_json} failed for {path_schema_json}, {exc}" + path_schema_json, path_config_json = None, "" + + if self._location and config_type in ("Project Config", "Global Config"): + path_config_json = "./zowe.config.json" + elif self._location and config_type in ("Project User Config", "Global User Config"): + path_config_json = "./zowe.config.user.json" + + path_schema_json = self.schema_path + if path_schema_json is None: # check if the $schema property is not defined + warnings.warn( + f"$schema property could not found" ) + + # validate the $schema property + if path_schema_json and opt_in: + validate_config_json(path_config_json, path_schema_json) + def get_profile( self, profile_name: Optional[str] = None, profile_type: Optional[str] = None, + config_type: Optional[str] = None, + opt_in: Optional[bool] = True, ) -> Profile: """ Load given profile including secure properties and excluding values from base profile @@ -194,7 +187,7 @@ def get_profile( Returns a namedtuple called Profile """ if self.profiles is None: - self.init_from_file() + self.init_from_file(config_type, opt_in) if profile_name is None and profile_type is None: raise ProfileNotFound( diff --git a/src/core/zowe/core_for_zowe_sdk/profile_manager.py b/src/core/zowe/core_for_zowe_sdk/profile_manager.py index c2af7771..c2b32c30 100644 --- a/src/core/zowe/core_for_zowe_sdk/profile_manager.py +++ b/src/core/zowe/core_for_zowe_sdk/profile_manager.py @@ -12,6 +12,7 @@ import os.path import warnings +import jsonschema from typing import Optional from .config_file import ConfigFile, Profile @@ -129,7 +130,27 @@ def get_profile( cfg_profile = Profile() try: cfg_profile = cfg.get_profile( - profile_name=profile_name, profile_type=profile_type + profile_name=profile_name, profile_type=profile_type, config_type=config_type + ) + except jsonschema.exceptions.ValidationError as exc: + raise jsonschema.exceptions.ValidationError( + f"Instance was invalid under the provided $schema property, {exc}" + ) + except jsonschema.exceptions.SchemaError as exc: + raise jsonschema.exception.SchemaError( + f"The provided schema is invalid, {exc}" + ) + except jsonschema.exceptions.UndefinedTypeCheck as exc: + raise jsonschema.exceptions.UndefinedTypeCheck( + f"A type checker was asked to check a type it did not have registered, {exc}" + ) + except jsonschema.exceptions.UnknownType as exc: + raise jsonschema.exceptions.UnknownType( + f"Unknown type is found in {path_schema_json}, exc" + ) + except jsonschema.exceptions.FormatError as exc: + raise jsonschema.exceptions.FormatError( + f"Validating a format {path_config_json} failed for {path_schema_json}, {exc}" ) except ProfileNotFound: if profile_name: @@ -167,8 +188,8 @@ def get_profile( f"because {type(exc).__name__}'{exc}'.", ConfigNotFoundWarning, ) - finally: - return cfg_profile + + return cfg_profile def load( self, @@ -227,12 +248,12 @@ def load( profile_props = {**profile_loaded.data, **profile_props} # Validating $schema property for all the layers - if cfg._location and config_type in ("Project Config", "Global Config"): - path_config_json = cfg._location + "/zowe.config.json" - cfg.validate_schema(path_config_json, opt_in) - elif cfg._location and config_type in ("Project User Config", "Global User Config"): - path_config_json = cfg._location + "/zowe.config.user.json" - cfg.validate_schema(path_config_json, opt_in) + # if cfg._location and config_type in ("Project Config", "Global Config"): + # path_config_json = "./zowe.config.json" + # cfg.validate_schema(path_config_json, opt_in) + # elif cfg._location and config_type in ("Project User Config", "Global User Config"): + # path_config_json = "./zowe.config.user.json" + # cfg.validate_schema(path_config_json, opt_in) missing_secure_props.extend(profile_loaded.missing_secure_props) From 708da247390786acd98f64a3d2de14cc2126733a Mon Sep 17 00:00:00 2001 From: aadityasinha-dotcom Date: Mon, 10 Jul 2023 12:59:03 +0530 Subject: [PATCH 13/33] fixed test Signed-off-by: aadityasinha-dotcom --- src/core/zowe/core_for_zowe_sdk/profile_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/zowe/core_for_zowe_sdk/profile_manager.py b/src/core/zowe/core_for_zowe_sdk/profile_manager.py index c2b32c30..fe170a0a 100644 --- a/src/core/zowe/core_for_zowe_sdk/profile_manager.py +++ b/src/core/zowe/core_for_zowe_sdk/profile_manager.py @@ -146,11 +146,11 @@ def get_profile( ) except jsonschema.exceptions.UnknownType as exc: raise jsonschema.exceptions.UnknownType( - f"Unknown type is found in {path_schema_json}, exc" + f"Unknown type is found in schema_json, exc" ) except jsonschema.exceptions.FormatError as exc: raise jsonschema.exceptions.FormatError( - f"Validating a format {path_config_json} failed for {path_schema_json}, {exc}" + f"Validating a format config_json failed for schema_json, {exc}" ) except ProfileNotFound: if profile_name: From 19ede33d61803751a2e91b857701215e4e7baeca Mon Sep 17 00:00:00 2001 From: aadityasinha-dotcom Date: Mon, 10 Jul 2023 13:14:33 +0530 Subject: [PATCH 14/33] changes Signed-off-by: aadityasinha-dotcom --- src/core/zowe/core_for_zowe_sdk/profile_manager.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/core/zowe/core_for_zowe_sdk/profile_manager.py b/src/core/zowe/core_for_zowe_sdk/profile_manager.py index fe170a0a..fa99d02b 100644 --- a/src/core/zowe/core_for_zowe_sdk/profile_manager.py +++ b/src/core/zowe/core_for_zowe_sdk/profile_manager.py @@ -246,15 +246,7 @@ def load( profile_loaded.name ) # Define profile name that will be merged from other layers profile_props = {**profile_loaded.data, **profile_props} - - # Validating $schema property for all the layers - # if cfg._location and config_type in ("Project Config", "Global Config"): - # path_config_json = "./zowe.config.json" - # cfg.validate_schema(path_config_json, opt_in) - # elif cfg._location and config_type in ("Project User Config", "Global User Config"): - # path_config_json = "./zowe.config.user.json" - # cfg.validate_schema(path_config_json, opt_in) - + missing_secure_props.extend(profile_loaded.missing_secure_props) if i == 1 and profile_props: From dd0be078537669181e2a5e836e981e5cf072bc04 Mon Sep 17 00:00:00 2001 From: aadityasinha-dotcom Date: Wed, 12 Jul 2023 21:38:30 +0530 Subject: [PATCH 15/33] simplified path_config_json Signed-off-by: aadityasinha-dotcom --- src/core/zowe/core_for_zowe_sdk/config_file.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/core/zowe/core_for_zowe_sdk/config_file.py b/src/core/zowe/core_for_zowe_sdk/config_file.py index 0fe850b2..e7dcee9f 100644 --- a/src/core/zowe/core_for_zowe_sdk/config_file.py +++ b/src/core/zowe/core_for_zowe_sdk/config_file.py @@ -156,10 +156,8 @@ def validate_schema( path_schema_json, path_config_json = None, "" - if self._location and config_type in ("Project Config", "Global Config"): - path_config_json = "./zowe.config.json" - elif self._location and config_type in ("Project User Config", "Global User Config"): - path_config_json = "./zowe.config.user.json" + if self._location: + path_config_json = f"./{self.filename}" path_schema_json = self.schema_path if path_schema_json is None: # check if the $schema property is not defined From 59023815f6ed4eb09e97ee1f576a00d220c90da5 Mon Sep 17 00:00:00 2001 From: aadityasinha-dotcom Date: Mon, 17 Jul 2023 12:37:29 +0530 Subject: [PATCH 16/33] added test for valide schema more test will be added soon Signed-off-by: aadityasinha-dotcom --- tests/unit/test_zowe_core.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/unit/test_zowe_core.py b/tests/unit/test_zowe_core.py index bdee79e0..9ef3f20d 100644 --- a/tests/unit/test_zowe_core.py +++ b/tests/unit/test_zowe_core.py @@ -173,9 +173,13 @@ def setUp(self): self.original_nested_file_path = os.path.join( FIXTURES_PATH, "nested.zowe.config.json" ) + self.original_schema_file_path = os.path.join( + FIXTURES_PATH, "zowe.schema.json" + ) self.fs.add_real_file(self.original_file_path) self.fs.add_real_file(self.original_user_file_path) self.fs.add_real_file(self.original_nested_file_path) + self.fs.add_real_file(self.original_schema_file_path) self.custom_dir = os.path.dirname(FIXTURES_PATH) self.custom_appname = "zowe_abcd" @@ -380,6 +384,27 @@ def test_profile_not_found_warning(self, get_pass_func): prof_manager = ProfileManager() prof_manager.config_dir = self.custom_dir props: dict = prof_manager.load("non_existent_profile") + + @patch("keyring.get_password", side_effect=keyring_get_password) + def test_profile_loading_with_valid_schema(self, get_pass_func): + """ + Test Validation, no error should be raised for valid schema + """ + # Setup - copy profile to fake filesystem created by pyfakefs + custom_file_path = os.path.join(self.custom_dir, "zowe.config.json") + shutil.copy(self.original_nested_file_path, custom_file_path) + custom_file_path = os.path.join(self.custom_dir, "zowe.schema.json") + shutil.copy(self.original_schema_file_path, custom_file_path) + + self.setUpCreds(custom_file_path, { + "profiles.zosmf.properties.user": "user", + "profiles.zosmf.properties.password": "password", + }) + + # Test + prof_manager = ProfileManager(appname="zowe") + prof_manager.config_dir = self.custom_dir + props: dict = prof_manager.load(profile_name="zosmf") class TestValidateConfigJsonClass(unittest.TestCase): From 8d65e9f2a6f151cbd291708527c37a7278566ba7 Mon Sep 17 00:00:00 2001 From: aadityasinha-dotcom Date: Fri, 21 Jul 2023 13:49:21 +0530 Subject: [PATCH 17/33] changed the opt_in flag to opt_out Signed-off-by: aadityasinha-dotcom --- src/core/zowe/core_for_zowe_sdk/config_file.py | 12 ++++++------ src/core/zowe/core_for_zowe_sdk/profile_manager.py | 7 ++++--- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/core/zowe/core_for_zowe_sdk/config_file.py b/src/core/zowe/core_for_zowe_sdk/config_file.py index e7dcee9f..14bdbdda 100644 --- a/src/core/zowe/core_for_zowe_sdk/config_file.py +++ b/src/core/zowe/core_for_zowe_sdk/config_file.py @@ -118,7 +118,7 @@ def location(self, dirname: str) -> None: def init_from_file( self, config_type: str, - opt_in: Optional[bool] = True + opt_out: Optional[bool] = True ) -> None: """ Initializes the class variable after @@ -135,7 +135,7 @@ def init_from_file( self.defaults = profile_jsonc.get("defaults", {}) if self.schema_property: - self.validate_schema(config_type, opt_in) + self.validate_schema(config_type, opt_out) # loading secure props is done in load_profile_properties # since we want to try loading secure properties only when # we know that the profile has saved properties @@ -144,7 +144,7 @@ def init_from_file( def validate_schema( self, config_type: str, - opt_in: Optional[bool] = True, + opt_out: Optional[bool] = True, ) -> None: """ Get the $schema_property from the config and load the schema @@ -166,7 +166,7 @@ def validate_schema( ) # validate the $schema property - if path_schema_json and opt_in: + if path_schema_json and opt_out: validate_config_json(path_config_json, path_schema_json) @@ -175,7 +175,7 @@ def get_profile( profile_name: Optional[str] = None, profile_type: Optional[str] = None, config_type: Optional[str] = None, - opt_in: Optional[bool] = True, + opt_out: Optional[bool] = True, ) -> Profile: """ Load given profile including secure properties and excluding values from base profile @@ -185,7 +185,7 @@ def get_profile( Returns a namedtuple called Profile """ if self.profiles is None: - self.init_from_file(config_type, opt_in) + self.init_from_file(config_type, opt_out) if profile_name is None and profile_type is None: raise ProfileNotFound( diff --git a/src/core/zowe/core_for_zowe_sdk/profile_manager.py b/src/core/zowe/core_for_zowe_sdk/profile_manager.py index fa99d02b..98efc03c 100644 --- a/src/core/zowe/core_for_zowe_sdk/profile_manager.py +++ b/src/core/zowe/core_for_zowe_sdk/profile_manager.py @@ -116,6 +116,7 @@ def get_profile( profile_name: Optional[str], profile_type: Optional[str], config_type: str, + opt_out: Optional[bool] = True, ) -> Profile: """ Get just the profile from the config file (overriden with base props in the config file) @@ -130,7 +131,7 @@ def get_profile( cfg_profile = Profile() try: cfg_profile = cfg.get_profile( - profile_name=profile_name, profile_type=profile_type, config_type=config_type + profile_name=profile_name, profile_type=profile_type, config_type=config_type, opt_out=opt_out ) except jsonschema.exceptions.ValidationError as exc: raise jsonschema.exceptions.ValidationError( @@ -196,7 +197,7 @@ def load( profile_name: Optional[str] = None, profile_type: Optional[str] = None, check_missing_props: bool = True, - opt_in: Optional[bool] = True, + opt_out: Optional[bool] = True, ) -> dict: """Load connection details from a team config profile. Returns @@ -237,7 +238,7 @@ def load( for i, (config_type, cfg) in enumerate(config_layers.items()): profile_loaded = self.get_profile( - cfg, profile_name, profile_type, config_type + cfg, profile_name, profile_type, config_type, opt_out ) # TODO Why don't user and password show up here for Project User Config? # Probably need to update load_profile_properties method in config_file.py From 8a94fdc6386a46ed52eed666ce0638e9322da134 Mon Sep 17 00:00:00 2001 From: aadityasinha-dotcom Date: Mon, 31 Jul 2023 13:25:09 +0530 Subject: [PATCH 18/33] fix test Signed-off-by: aadityasinha-dotcom --- tests/unit/test_zowe_core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_zowe_core.py b/tests/unit/test_zowe_core.py index 9ef3f20d..dacb3786 100644 --- a/tests/unit/test_zowe_core.py +++ b/tests/unit/test_zowe_core.py @@ -393,8 +393,8 @@ def test_profile_loading_with_valid_schema(self, get_pass_func): # Setup - copy profile to fake filesystem created by pyfakefs custom_file_path = os.path.join(self.custom_dir, "zowe.config.json") shutil.copy(self.original_nested_file_path, custom_file_path) - custom_file_path = os.path.join(self.custom_dir, "zowe.schema.json") - shutil.copy(self.original_schema_file_path, custom_file_path) + shutil.copy(self.original_schema_file_path, self.custom_dir) + os.chdir(self.custom_dir) self.setUpCreds(custom_file_path, { "profiles.zosmf.properties.user": "user", From 600439abf8d81e569a41e659ce74be10f94c4cb0 Mon Sep 17 00:00:00 2001 From: aadityasinha-dotcom Date: Tue, 1 Aug 2023 21:28:21 +0530 Subject: [PATCH 19/33] added test for invalid schema Signed-off-by: aadityasinha-dotcom --- tests/unit/test_zowe_core.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/unit/test_zowe_core.py b/tests/unit/test_zowe_core.py index dacb3786..09940dce 100644 --- a/tests/unit/test_zowe_core.py +++ b/tests/unit/test_zowe_core.py @@ -170,6 +170,9 @@ def setUp(self): self.original_user_file_path = os.path.join( FIXTURES_PATH, "zowe.config.user.json" ) + self.original_invalid_file_path = os.path.join( + FIXTURES_PATH, "invalid.zowe.config.json" + ) self.original_nested_file_path = os.path.join( FIXTURES_PATH, "nested.zowe.config.json" ) @@ -180,6 +183,7 @@ def setUp(self): self.fs.add_real_file(self.original_user_file_path) self.fs.add_real_file(self.original_nested_file_path) self.fs.add_real_file(self.original_schema_file_path) + self.fs.add_real_file(self.original_invalid_file_path) self.custom_dir = os.path.dirname(FIXTURES_PATH) self.custom_appname = "zowe_abcd" @@ -405,6 +409,28 @@ def test_profile_loading_with_valid_schema(self, get_pass_func): prof_manager = ProfileManager(appname="zowe") prof_manager.config_dir = self.custom_dir props: dict = prof_manager.load(profile_name="zosmf") + + @patch("keyring.get_password", side_effect=keyring_get_password) + def test_profile_loading_with_invalid_schema(self, get_pass_func): + """ + Test Validation, no error should be raised for valid schema + """ + # Setup - copy profile to fake filesystem created by pyfakefs + with self.assertRaises(ValidationError): + custom_file_path = os.path.join(self.custom_dir, "invalid.zowe.config.json") + shutil.copy(self.original_invalid_file_path, custom_file_path) + shutil.copy(self.original_schema_file_path, self.custom_dir) + os.chdir(self.custom_dir) + + self.setUpCreds(custom_file_path, { + "profiles.zosmf.properties.user": "user", + "profiles.zosmf.properties.password": "password", + }) + + # Test + prof_manager = ProfileManager(appname="zowe") + prof_manager.config_dir = self.custom_dir + props: dict = prof_manager.load(profile_name="zosmf") class TestValidateConfigJsonClass(unittest.TestCase): From e01ffb0b0e67119731f321adbad68145139b21d8 Mon Sep 17 00:00:00 2001 From: aadityasinha-dotcom Date: Thu, 10 Aug 2023 18:24:08 +0530 Subject: [PATCH 20/33] refactored the code Signed-off-by: aadityasinha-dotcom --- src/core/zowe/core_for_zowe_sdk/config_file.py | 18 +++++++----------- .../zowe/core_for_zowe_sdk/profile_manager.py | 10 +++++----- src/core/zowe/core_for_zowe_sdk/validators.py | 12 +++++++++--- tests/unit/test_zowe_core.py | 6 ++++++ 4 files changed, 27 insertions(+), 19 deletions(-) diff --git a/src/core/zowe/core_for_zowe_sdk/config_file.py b/src/core/zowe/core_for_zowe_sdk/config_file.py index c2a6e617..35aa3846 100644 --- a/src/core/zowe/core_for_zowe_sdk/config_file.py +++ b/src/core/zowe/core_for_zowe_sdk/config_file.py @@ -111,10 +111,6 @@ def location(self) -> Optional[str]: def schema_path(self) -> Optional[str]: return self.schema_property - @property - def schema_path(self) -> Optional[str]: - self.schema_property - @location.setter def location(self, dirname: str) -> None: if os.path.isdir(dirname): @@ -125,7 +121,7 @@ def location(self, dirname: str) -> None: def init_from_file( self, config_type: str, - opt_out: Optional[bool] = True + validate_schema: Optional[bool] = True ) -> None: """ Initializes the class variable after @@ -143,7 +139,7 @@ def init_from_file( self.schema_property = profile_jsonc.get("$schema", None) if self.schema_property: - self.validate_schema(config_type, opt_out) + self.validate_schema(config_type, validate_schema) # loading secure props is done in load_profile_properties # since we want to try loading secure properties only when # we know that the profile has saved properties @@ -152,7 +148,7 @@ def init_from_file( def validate_schema( self, config_type: str, - opt_out: Optional[bool] = True, + validate_schema: Optional[bool] = True, ) -> None: """ Get the $schema_property from the config and load the schema @@ -174,8 +170,8 @@ def validate_schema( ) # validate the $schema property - if path_schema_json and opt_out: - validate_config_json(path_config_json, path_schema_json) + if path_schema_json and validate_schema: + validate_config_json(path_config_json, path_schema_json, cwd = self.location) def schema_list( self, @@ -226,7 +222,7 @@ def get_profile( profile_name: Optional[str] = None, profile_type: Optional[str] = None, config_type: Optional[str] = None, - opt_out: Optional[bool] = True, + validate_schema: Optional[bool] = True, ) -> Profile: """ Load given profile including secure properties and excluding values from base profile @@ -236,7 +232,7 @@ def get_profile( Returns a namedtuple called Profile """ if self.profiles is None: - self.init_from_file(config_type, opt_out) + self.init_from_file(config_type, validate_schema) if profile_name is None and profile_type is None: raise ProfileNotFound( diff --git a/src/core/zowe/core_for_zowe_sdk/profile_manager.py b/src/core/zowe/core_for_zowe_sdk/profile_manager.py index d44c5ac2..69fcfed8 100644 --- a/src/core/zowe/core_for_zowe_sdk/profile_manager.py +++ b/src/core/zowe/core_for_zowe_sdk/profile_manager.py @@ -161,7 +161,7 @@ def get_profile( profile_name: Optional[str], profile_type: Optional[str], config_type: str, - opt_out: Optional[bool] = True, + validate_schema: Optional[bool] = True, ) -> Profile: """ Get just the profile from the config file (overriden with base props in the config file) @@ -176,14 +176,14 @@ def get_profile( cfg_profile = Profile() try: cfg_profile = cfg.get_profile( - profile_name=profile_name, profile_type=profile_type, config_type=config_type, opt_out=opt_out + profile_name=profile_name, profile_type=profile_type, config_type=config_type, validate_schema=validate_schema ) except jsonschema.exceptions.ValidationError as exc: raise jsonschema.exceptions.ValidationError( f"Instance was invalid under the provided $schema property, {exc}" ) except jsonschema.exceptions.SchemaError as exc: - raise jsonschema.exception.SchemaError( + raise jsonschema.exceptions.SchemaError( f"The provided schema is invalid, {exc}" ) except jsonschema.exceptions.UndefinedTypeCheck as exc: @@ -242,7 +242,7 @@ def load( profile_name: Optional[str] = None, profile_type: Optional[str] = None, check_missing_props: bool = True, - opt_out: Optional[bool] = True, + validate_schema: Optional[bool] = True, override_with_env: Optional[bool] = False, ) -> dict: """Load connection details from a team config profile. @@ -285,7 +285,7 @@ def load( for i, (config_type, cfg) in enumerate(config_layers.items()): profile_loaded = self.get_profile( - cfg, profile_name, profile_type, config_type, opt_out + cfg, profile_name, profile_type, config_type, validate_schema ) # TODO Why don't user and password show up here for Project User Config? # Probably need to update load_profile_properties method in config_file.py diff --git a/src/core/zowe/core_for_zowe_sdk/validators.py b/src/core/zowe/core_for_zowe_sdk/validators.py index a19e8388..273126b0 100644 --- a/src/core/zowe/core_for_zowe_sdk/validators.py +++ b/src/core/zowe/core_for_zowe_sdk/validators.py @@ -17,7 +17,7 @@ import requests -def validate_config_json(path_config_json: Union[str, dict], path_schema_json: str): +def validate_config_json(path_config_json: Union[str, dict], path_schema_json: str, cwd: str): """ Function validating that zowe.config.json file matches zowe.schema.json. @@ -37,12 +37,18 @@ def validate_config_json(path_config_json: Union[str, dict], path_schema_json: s # checks if the path_schema_json point to an internet URI and download the schema using the URI if path_schema_json.startswith("https://") or path_schema_json.startswith("http://"): schema_json = requests.get(path_schema_json).json() - + # checks if the path_schema_json is a file elif os.path.isfile(path_schema_json): with open(path_schema_json) as file: schema_json = commentjson.load(file) - + + # checks if the path_schema_json is absolute + elif not os.path.isabs(path_schema_json): + path_schema_json = os.path.join(cwd, path_schema_json) + with open(path_schema_json) as file: + schema_json = commentjson.load(file) + # if there is no path_schema_json it will return None else: return None diff --git a/tests/unit/test_zowe_core.py b/tests/unit/test_zowe_core.py index bf3e554b..f560de71 100644 --- a/tests/unit/test_zowe_core.py +++ b/tests/unit/test_zowe_core.py @@ -179,11 +179,15 @@ def setUp(self): self.original_schema_file_path = os.path.join( FIXTURES_PATH, "zowe.schema.json" ) + self.original_invalid_schema_file_path = os.path.join( + FIXTURES_PATH, "invalid.zowe.schema.json" + ) self.fs.add_real_file(self.original_file_path) self.fs.add_real_file(self.original_user_file_path) self.fs.add_real_file(self.original_nested_file_path) self.fs.add_real_file(self.original_schema_file_path) self.fs.add_real_file(self.original_invalid_file_path) + self.fs.add_real_file(self.original_invalid_schema_file_path) self.custom_dir = os.path.dirname(FIXTURES_PATH) self.custom_appname = "zowe_abcd" self.custom_filename = f"{self.custom_appname}.config.json" @@ -419,6 +423,8 @@ def test_profile_loading_with_invalid_schema(self, get_pass_func): custom_file_path = os.path.join(self.custom_dir, "invalid.zowe.config.json") shutil.copy(self.original_invalid_file_path, custom_file_path) shutil.copy(self.original_schema_file_path, self.custom_dir) + shutil.copy(self.original_user_file_path, self.custom_dir) + shutil.copy(self.original_file_path, self.custom_dir) os.chdir(self.custom_dir) self.setUpCreds(custom_file_path, { From 0030a100c84f4fae353e6c504ddd80a76529807a Mon Sep 17 00:00:00 2001 From: aadityasinha-dotcom Date: Fri, 11 Aug 2023 11:37:54 +0530 Subject: [PATCH 21/33] added test for invalid schema and updated requirements.txt Signed-off-by: aadityasinha-dotcom --- requirements.txt | 2 +- tests/unit/fixtures/invalid.zowe.config.json | 2 +- tests/unit/test_zowe_core.py | 6 ++---- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/requirements.txt b/requirements.txt index dde23817..dd60b51e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ coverage==5.4 flake8==3.8.4 idna==2.10 importlib-metadata==3.6.0 -jsonschema==4.14.0 +jsonschema==4.19.0 keyring lxml==4.9.3 mccabe==0.6.1 diff --git a/tests/unit/fixtures/invalid.zowe.config.json b/tests/unit/fixtures/invalid.zowe.config.json index bf06a900..89be0a69 100644 --- a/tests/unit/fixtures/invalid.zowe.config.json +++ b/tests/unit/fixtures/invalid.zowe.config.json @@ -1,5 +1,5 @@ { - "$schema": "./zowe.schema.json", + "$schema": "./invalid.zowe.schema.json", "profiles": { "zosmf": { "type": "zosmf", diff --git a/tests/unit/test_zowe_core.py b/tests/unit/test_zowe_core.py index f560de71..56810a2e 100644 --- a/tests/unit/test_zowe_core.py +++ b/tests/unit/test_zowe_core.py @@ -422,9 +422,7 @@ def test_profile_loading_with_invalid_schema(self, get_pass_func): with self.assertRaises(ValidationError): custom_file_path = os.path.join(self.custom_dir, "invalid.zowe.config.json") shutil.copy(self.original_invalid_file_path, custom_file_path) - shutil.copy(self.original_schema_file_path, self.custom_dir) - shutil.copy(self.original_user_file_path, self.custom_dir) - shutil.copy(self.original_file_path, self.custom_dir) + shutil.copy(self.original_invalid_schema_file_path, self.custom_dir) os.chdir(self.custom_dir) self.setUpCreds(custom_file_path, { @@ -433,7 +431,7 @@ def test_profile_loading_with_invalid_schema(self, get_pass_func): }) # Test - prof_manager = ProfileManager(appname="zowe") + prof_manager = ProfileManager(appname="invalid.zowe") prof_manager.config_dir = self.custom_dir props: dict = prof_manager.load(profile_name="zosmf") From 8edec117cc171a24cef768a14bff28d4ba24c803 Mon Sep 17 00:00:00 2001 From: aadityasinha-dotcom Date: Thu, 17 Aug 2023 19:02:22 +0530 Subject: [PATCH 22/33] added test for internet URI Signed-off-by: aadityasinha-dotcom --- .../zowe/core_for_zowe_sdk/config_file.py | 6 +- .../zowe/core_for_zowe_sdk/profile_manager.py | 3 +- .../unit/fixtures/invalidUri.zowe.config.json | 55 +++++++++++++++++++ tests/unit/test_zowe_core.py | 27 ++++++++- 4 files changed, 86 insertions(+), 5 deletions(-) create mode 100644 tests/unit/fixtures/invalidUri.zowe.config.json diff --git a/src/core/zowe/core_for_zowe_sdk/config_file.py b/src/core/zowe/core_for_zowe_sdk/config_file.py index 35aa3846..47fa089b 100644 --- a/src/core/zowe/core_for_zowe_sdk/config_file.py +++ b/src/core/zowe/core_for_zowe_sdk/config_file.py @@ -121,7 +121,7 @@ def location(self, dirname: str) -> None: def init_from_file( self, config_type: str, - validate_schema: Optional[bool] = True + validate_schema: Optional[bool] = False ) -> None: """ Initializes the class variable after @@ -148,7 +148,7 @@ def init_from_file( def validate_schema( self, config_type: str, - validate_schema: Optional[bool] = True, + validate_schema: Optional[bool] = False, ) -> None: """ Get the $schema_property from the config and load the schema @@ -222,7 +222,7 @@ def get_profile( profile_name: Optional[str] = None, profile_type: Optional[str] = None, config_type: Optional[str] = None, - validate_schema: Optional[bool] = True, + validate_schema: Optional[bool] = False, ) -> Profile: """ Load given profile including secure properties and excluding values from base profile diff --git a/src/core/zowe/core_for_zowe_sdk/profile_manager.py b/src/core/zowe/core_for_zowe_sdk/profile_manager.py index 69fcfed8..f267a430 100644 --- a/src/core/zowe/core_for_zowe_sdk/profile_manager.py +++ b/src/core/zowe/core_for_zowe_sdk/profile_manager.py @@ -229,6 +229,7 @@ def get_profile( SecurePropsNotFoundWarning, ) except Exception as exc: + print("hi", exc) warnings.warn( f"Could not load {config_type} '{cfg.filename}' at '{cfg.filepath}'" f"because {type(exc).__name__}'{exc}'.", @@ -242,7 +243,7 @@ def load( profile_name: Optional[str] = None, profile_type: Optional[str] = None, check_missing_props: bool = True, - validate_schema: Optional[bool] = True, + validate_schema: Optional[bool] = False, override_with_env: Optional[bool] = False, ) -> dict: """Load connection details from a team config profile. diff --git a/tests/unit/fixtures/invalidUri.zowe.config.json b/tests/unit/fixtures/invalidUri.zowe.config.json new file mode 100644 index 00000000..4ec752eb --- /dev/null +++ b/tests/unit/fixtures/invalidUri.zowe.config.json @@ -0,0 +1,55 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "profiles": { + "zosmf": { + "type": "zosmf", + "properties": { + "port": 10443 + }, + "secure": [] + }, + "tso": { + "type": "tso", + "properties": { + "account": "", + "codePage": "1047", + "logonProcedure": "IZUFPROC" + }, + "secure": [] + }, + "ssh": { + "type": "ssh", + "properties": { + "port": 22 + }, + "secure": ["user"] + }, + "zftp": { + "type": "zftp", + "properties": { + "port": 21, + "secureFtp": true + }, + "secure": [] + }, + "base": { + "type": "base", + "properties": { + "host": "zowe.test.cloud", + "rejectUnauthorized": false + }, + "secure": [ + "user", + "password" + ] + } + }, + "defaults": { + "zosmf": "zosmf", + "tso": "tso", + "ssh": "ssh", + "zftp": "zftp", + "base": "base" + }, + "autoStore": true +} \ No newline at end of file diff --git a/tests/unit/test_zowe_core.py b/tests/unit/test_zowe_core.py index 56810a2e..2baf2a06 100644 --- a/tests/unit/test_zowe_core.py +++ b/tests/unit/test_zowe_core.py @@ -182,12 +182,16 @@ def setUp(self): self.original_invalid_schema_file_path = os.path.join( FIXTURES_PATH, "invalid.zowe.schema.json" ) + self.original_invalidUri_schema_file_path = os.path.join( + FIXTURES_PATH, "invalidUri.zowe.config.json" + ) self.fs.add_real_file(self.original_file_path) self.fs.add_real_file(self.original_user_file_path) self.fs.add_real_file(self.original_nested_file_path) self.fs.add_real_file(self.original_schema_file_path) self.fs.add_real_file(self.original_invalid_file_path) self.fs.add_real_file(self.original_invalid_schema_file_path) + self.fs.add_real_file(self.original_invalidUri_schema_file_path) self.custom_dir = os.path.dirname(FIXTURES_PATH) self.custom_appname = "zowe_abcd" self.custom_filename = f"{self.custom_appname}.config.json" @@ -433,7 +437,28 @@ def test_profile_loading_with_invalid_schema(self, get_pass_func): # Test prof_manager = ProfileManager(appname="invalid.zowe") prof_manager.config_dir = self.custom_dir - props: dict = prof_manager.load(profile_name="zosmf") + props: dict = prof_manager.load(profile_name="zosmf", validate_schema=True) + + @patch("keyring.get_password", side_effect=keyring_get_password) + def test_profile_loading_with_invalid_schema_internet_URI(self, get_pass_func): + """ + Test Validation, no error should be raised for valid schema + """ + # Setup - copy profile to fake filesystem created by pyfakefs + with self.assertRaises(ValidationError): + custom_file_path = os.path.join(self.custom_dir, "invalidUri.zowe.config.json") + shutil.copy(self.original_invalidUri_schema_file_path, custom_file_path) + os.chdir(self.custom_dir) + + self.setUpCreds(custom_file_path, { + "profiles.zosmf.properties.user": "user", + "profiles.zosmf.properties.password": "password", + }) + + # Test + prof_manager = ProfileManager(appname="invalidUri.zowe") + prof_manager.config_dir = self.custom_dir + props: dict = prof_manager.load(profile_name="zosmf", validate_schema=True) @patch("keyring.get_password", side_effect=keyring_get_password) def test_profile_loading_with_env_variables(self, get_pass_func): From 6f34785f6789fdf0c51f52e42b2f238efb236810 Mon Sep 17 00:00:00 2001 From: aadityasinha-dotcom Date: Mon, 21 Aug 2023 22:25:17 +0530 Subject: [PATCH 23/33] did some changes, added unit test for internet URI Signed-off-by: aadityasinha-dotcom --- .../zowe/core_for_zowe_sdk/config_file.py | 11 +- .../zowe/core_for_zowe_sdk/profile_manager.py | 4 +- src/core/zowe/core_for_zowe_sdk/validators.py | 6 +- .../unit/fixtures/invalidUri.zowe.config.json | 2 +- .../unit/fixtures/invalidUri.zowe.schema.json | 427 ++++++++++++++++++ tests/unit/test_zowe_core.py | 19 +- 6 files changed, 453 insertions(+), 16 deletions(-) create mode 100644 tests/unit/fixtures/invalidUri.zowe.schema.json diff --git a/src/core/zowe/core_for_zowe_sdk/config_file.py b/src/core/zowe/core_for_zowe_sdk/config_file.py index 47fa089b..aff4771f 100644 --- a/src/core/zowe/core_for_zowe_sdk/config_file.py +++ b/src/core/zowe/core_for_zowe_sdk/config_file.py @@ -121,7 +121,8 @@ def location(self, dirname: str) -> None: def init_from_file( self, config_type: str, - validate_schema: Optional[bool] = False + validate_schema: Optional[bool] = False, + verify: Optional[bool] = False, ) -> None: """ Initializes the class variable after @@ -139,7 +140,7 @@ def init_from_file( self.schema_property = profile_jsonc.get("$schema", None) if self.schema_property: - self.validate_schema(config_type, validate_schema) + self.validate_schema(config_type, validate_schema, verify=verify) # loading secure props is done in load_profile_properties # since we want to try loading secure properties only when # we know that the profile has saved properties @@ -149,6 +150,7 @@ def validate_schema( self, config_type: str, validate_schema: Optional[bool] = False, + verify: Optional[bool] = False, ) -> None: """ Get the $schema_property from the config and load the schema @@ -171,7 +173,7 @@ def validate_schema( # validate the $schema property if path_schema_json and validate_schema: - validate_config_json(path_config_json, path_schema_json, cwd = self.location) + validate_config_json(path_config_json, path_schema_json, cwd = self.location, verify=verify) def schema_list( self, @@ -223,6 +225,7 @@ def get_profile( profile_type: Optional[str] = None, config_type: Optional[str] = None, validate_schema: Optional[bool] = False, + verify: Optional[bool] = False, ) -> Profile: """ Load given profile including secure properties and excluding values from base profile @@ -232,7 +235,7 @@ def get_profile( Returns a namedtuple called Profile """ if self.profiles is None: - self.init_from_file(config_type, validate_schema) + self.init_from_file(config_type, validate_schema, verify) if profile_name is None and profile_type is None: raise ProfileNotFound( diff --git a/src/core/zowe/core_for_zowe_sdk/profile_manager.py b/src/core/zowe/core_for_zowe_sdk/profile_manager.py index f267a430..f8ff0b46 100644 --- a/src/core/zowe/core_for_zowe_sdk/profile_manager.py +++ b/src/core/zowe/core_for_zowe_sdk/profile_manager.py @@ -162,6 +162,7 @@ def get_profile( profile_type: Optional[str], config_type: str, validate_schema: Optional[bool] = True, + verify: Optional[bool] = False, ) -> Profile: """ Get just the profile from the config file (overriden with base props in the config file) @@ -245,6 +246,7 @@ def load( check_missing_props: bool = True, validate_schema: Optional[bool] = False, override_with_env: Optional[bool] = False, + verify: Optional[bool] = False, ) -> dict: """Load connection details from a team config profile. Returns @@ -286,7 +288,7 @@ def load( for i, (config_type, cfg) in enumerate(config_layers.items()): profile_loaded = self.get_profile( - cfg, profile_name, profile_type, config_type, validate_schema + cfg, profile_name, profile_type, config_type, validate_schema, verify ) # TODO Why don't user and password show up here for Project User Config? # Probably need to update load_profile_properties method in config_file.py diff --git a/src/core/zowe/core_for_zowe_sdk/validators.py b/src/core/zowe/core_for_zowe_sdk/validators.py index 273126b0..5ae8ff2e 100644 --- a/src/core/zowe/core_for_zowe_sdk/validators.py +++ b/src/core/zowe/core_for_zowe_sdk/validators.py @@ -13,11 +13,11 @@ import commentjson from jsonschema import validate import os -from typing import Union +from typing import Union, Optional import requests -def validate_config_json(path_config_json: Union[str, dict], path_schema_json: str, cwd: str): +def validate_config_json(path_config_json: Union[str, dict], path_schema_json: str, cwd: str, verify: Optional[bool] = False): """ Function validating that zowe.config.json file matches zowe.schema.json. @@ -36,7 +36,7 @@ def validate_config_json(path_config_json: Union[str, dict], path_schema_json: s # checks if the path_schema_json point to an internet URI and download the schema using the URI if path_schema_json.startswith("https://") or path_schema_json.startswith("http://"): - schema_json = requests.get(path_schema_json).json() + schema_json = requests.get(path_schema_json, verify = verify).json() # checks if the path_schema_json is a file elif os.path.isfile(path_schema_json): diff --git a/tests/unit/fixtures/invalidUri.zowe.config.json b/tests/unit/fixtures/invalidUri.zowe.config.json index 4ec752eb..68b06dbe 100644 --- a/tests/unit/fixtures/invalidUri.zowe.config.json +++ b/tests/unit/fixtures/invalidUri.zowe.config.json @@ -1,5 +1,5 @@ { - "$schema": "https://json-schema.org/draft/2020-12/schema", + "$schema": "./invalidUri.zowe.schema.json", "profiles": { "zosmf": { "type": "zosmf", diff --git a/tests/unit/fixtures/invalidUri.zowe.schema.json b/tests/unit/fixtures/invalidUri.zowe.schema.json new file mode 100644 index 00000000..3ef45c26 --- /dev/null +++ b/tests/unit/fixtures/invalidUri.zowe.schema.json @@ -0,0 +1,427 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$version": "1.0", + "type": "invalid", + "description": "Zowe configuration", + "properties": { + "profiles": { + "type": "object", + "description": "Mapping of profile names to profile configurations", + "patternProperties": { + "^\\S*$": { + "type": "object", + "description": "Profile configuration object", + "properties": { + "type": { + "description": "Profile type", + "type": "boolean", + "enum": [ + "zosmf", + "tso", + "ssh", + "zftp", + "base" + ] + }, + "properties": { + "description": "Profile properties object", + "type": "object" + }, + "profiles": { + "description": "Optional subprofile configurations", + "type": "object", + "$ref": "#/properties/profiles" + }, + "secure": { + "description": "Secure property names", + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true + } + }, + "allOf": [ + { + "if": { + "properties": { + "type": false + } + }, + "then": { + "properties": { + "properties": { + "title": "Missing profile type" + } + } + } + }, + { + "if": { + "properties": { + "type": { + "const": "zosmf" + } + } + }, + "then": { + "properties": { + "properties": { + "type": "object", + "title": "z/OSMF Profile", + "description": "z/OSMF Profile", + "properties": { + "host": { + "type": "string", + "description": "The z/OSMF server host name." + }, + "port": { + "type": "number", + "description": "The z/OSMF server port.", + "default": 443 + }, + "user": { + "type": "string", + "description": "Mainframe (z/OSMF) user name, which can be the same as your TSO login." + }, + "password": { + "type": "string", + "description": "Mainframe (z/OSMF) password, which can be the same as your TSO password." + }, + "rejectUnauthorized": { + "type": "boolean", + "description": "Reject self-signed certificates.", + "default": true + }, + "certFile": { + "type": "string", + "description": "The file path to a certificate file to use for authentication" + }, + "certKeyFile": { + "type": "string", + "description": "The file path to a certificate key file to use for authentication" + }, + "basePath": { + "type": "string", + "description": "The base path for your API mediation layer instance. Specify this option to prepend the base path to all z/OSMF resources when making REST requests. Do not specify this option if you are not using an API mediation layer." + }, + "protocol": { + "type": "string", + "description": "The protocol used (HTTP or HTTPS)", + "default": "https", + "enum": [ + "http", + "https" + ] + }, + "encoding": { + "type": "string", + "description": "The encoding for download and upload of z/OS data set and USS files. The default encoding if not specified is IBM-1047." + }, + "responseTimeout": { + "type": "number", + "description": "The maximum amount of time in seconds the z/OSMF Files TSO servlet should run before returning a response. Any request exceeding this amount of time will be terminated and return an error. Allowed values: 5 - 600" + } + }, + "required": [] + }, + "secure": { + "items": { + "enum": [ + "user", + "password" + ] + } + } + } + } + }, + { + "if": { + "properties": { + "type": { + "const": "tso" + } + } + }, + "then": { + "properties": { + "properties": { + "type": "object", + "title": "TSO Profile", + "description": "z/OS TSO/E User Profile", + "properties": { + "account": { + "type": "string", + "description": "Your z/OS TSO/E accounting information." + }, + "characterSet": { + "type": "string", + "description": "Character set for address space to convert messages and responses from UTF-8 to EBCDIC.", + "default": "697" + }, + "codePage": { + "type": "string", + "description": "Codepage value for TSO/E address space to convert messages and responses from UTF-8 to EBCDIC.", + "default": "1047" + }, + "columns": { + "type": "number", + "description": "The number of columns on a screen.", + "default": 80 + }, + "logonProcedure": { + "type": "string", + "description": "The logon procedure to use when creating TSO procedures on your behalf.", + "default": "IZUFPROC" + }, + "regionSize": { + "type": "number", + "description": "Region size for the TSO/E address space.", + "default": 4096 + }, + "rows": { + "type": "number", + "description": "The number of rows on a screen.", + "default": 24 + } + }, + "required": [] + } + } + } + }, + { + "if": { + "properties": { + "type": { + "const": "ssh" + } + } + }, + "then": { + "properties": { + "properties": { + "type": "object", + "title": "z/OS SSH Profile", + "description": "z/OS SSH Profile", + "properties": { + "host": { + "type": "string", + "description": "The z/OS SSH server host name." + }, + "port": { + "type": "number", + "description": "The z/OS SSH server port.", + "default": 22 + }, + "user": { + "type": "string", + "description": "Mainframe user name, which can be the same as your TSO login." + }, + "password": { + "type": "string", + "description": "Mainframe password, which can be the same as your TSO password." + }, + "privateKey": { + "type": "string", + "description": "Path to a file containing your private key, that must match a public key stored in the server for authentication" + }, + "keyPassphrase": { + "type": "string", + "description": "Private key passphrase, which unlocks the private key." + }, + "handshakeTimeout": { + "type": "number", + "description": "How long in milliseconds to wait for the SSH handshake to complete." + } + }, + "required": [] + }, + "secure": { + "items": { + "enum": [ + "user", + "password", + "keyPassphrase" + ] + } + } + } + } + }, + { + "if": { + "properties": { + "type": { + "const": "zftp" + } + } + }, + "then": { + "properties": { + "properties": { + "type": "object", + "title": "Configuration profile for z/OS FTP", + "description": "Configuration profile for z/OS FTP", + "properties": { + "host": { + "type": "string", + "description": "The hostname or IP address of the z/OS server to connect to." + }, + "port": { + "type": "number", + "description": "The port of the z/OS FTP server.", + "default": 21 + }, + "user": { + "type": "string", + "description": "Username for authentication on z/OS" + }, + "password": { + "type": "string", + "description": "Password to authenticate to FTP." + }, + "secureFtp": { + "type": [ + "boolean", + "null" + ], + "description": "Set to true for both control and data connection encryption, 'control' for control connection encryption only, or 'implicit' for implicitly encrypted control connection (this mode is deprecated in modern times, but usually uses port 990). Note: Unfortunately, this plugin's functionality only works with FTP and FTPS, not 'SFTP' which is FTP over SSH.", + "default": true + }, + "rejectUnauthorized": { + "type": [ + "boolean", + "null" + ], + "description": "Reject self-signed certificates. Only specify this if you are connecting to a secure FTP instance." + }, + "servername": { + "type": [ + "string", + "null" + ], + "description": "Server name for the SNI (Server Name Indication) TLS extension. Only specify if you are connecting securely" + }, + "connectionTimeout": { + "type": "number", + "description": "How long (in milliseconds) to wait for the control connection to be established.", + "default": 10000 + } + } + }, + "secure": { + "items": { + "enum": [ + "user", + "password" + ] + } + } + } + } + }, + { + "if": { + "properties": { + "type": { + "const": "base" + } + } + }, + "then": { + "properties": { + "properties": { + "type": "object", + "title": "Base Profile", + "description": "Base profile that stores values shared by multiple service profiles", + "properties": { + "host": { + "type": "string", + "description": "Host name of service on the mainframe." + }, + "port": { + "type": "number", + "description": "Port number of service on the mainframe." + }, + "user": { + "type": "string", + "description": "User name to authenticate to service on the mainframe." + }, + "password": { + "type": "string", + "description": "Password to authenticate to service on the mainframe." + }, + "rejectUnauthorized": { + "type": "boolean", + "description": "Reject self-signed certificates.", + "default": true + }, + "tokenType": { + "type": "string", + "description": "The type of token to get and use for the API. Omit this option to use the default token type, which is provided by 'zowe auth login'." + }, + "tokenValue": { + "type": "string", + "description": "The value of the token to pass to the API." + }, + "certFile": { + "type": "string", + "description": "The file path to a certificate file to use for authentication" + }, + "certKeyFile": { + "type": "string", + "description": "The file path to a certificate key file to use for authentication" + } + }, + "required": [] + }, + "secure": { + "items": { + "enum": [ + "user", + "password", + "tokenValue" + ] + } + } + } + } + } + ] + } + } + }, + "defaults": { + "type": "object", + "description": "Mapping of profile types to default profile names", + "properties": { + "zosmf": { + "description": "Default zosmf profile", + "type": "string" + }, + "tso": { + "description": "Default tso profile", + "type": "string" + }, + "ssh": { + "description": "Default ssh profile", + "type": "string" + }, + "zftp": { + "description": "Default zftp profile", + "type": "string" + }, + "base": { + "description": "Default base profile", + "type": "string" + } + } + }, + "autoStore": { + "type": "boolean", + "description": "If true, values you enter when prompted are stored for future use" + } + } +} \ No newline at end of file diff --git a/tests/unit/test_zowe_core.py b/tests/unit/test_zowe_core.py index 2baf2a06..50bdca45 100644 --- a/tests/unit/test_zowe_core.py +++ b/tests/unit/test_zowe_core.py @@ -9,7 +9,7 @@ import unittest from unittest import mock from unittest.mock import patch -from jsonschema import validate, ValidationError +from jsonschema import validate, ValidationError, SchemaError from zowe.core_for_zowe_sdk.validators import validate_config_json import commentjson @@ -182,15 +182,19 @@ def setUp(self): self.original_invalid_schema_file_path = os.path.join( FIXTURES_PATH, "invalid.zowe.schema.json" ) - self.original_invalidUri_schema_file_path = os.path.join( + self.original_invalidUri_file_path = os.path.join( FIXTURES_PATH, "invalidUri.zowe.config.json" ) + self.original_invalidUri_schema_file_path = os.path.join( + FIXTURES_PATH, "invalidUri.zowe.schema.json" + ) self.fs.add_real_file(self.original_file_path) self.fs.add_real_file(self.original_user_file_path) self.fs.add_real_file(self.original_nested_file_path) self.fs.add_real_file(self.original_schema_file_path) self.fs.add_real_file(self.original_invalid_file_path) self.fs.add_real_file(self.original_invalid_schema_file_path) + self.fs.add_real_file(self.original_invalidUri_file_path) self.fs.add_real_file(self.original_invalidUri_schema_file_path) self.custom_dir = os.path.dirname(FIXTURES_PATH) self.custom_appname = "zowe_abcd" @@ -445,9 +449,10 @@ def test_profile_loading_with_invalid_schema_internet_URI(self, get_pass_func): Test Validation, no error should be raised for valid schema """ # Setup - copy profile to fake filesystem created by pyfakefs - with self.assertRaises(ValidationError): + with self.assertRaises(SchemaError): custom_file_path = os.path.join(self.custom_dir, "invalidUri.zowe.config.json") - shutil.copy(self.original_invalidUri_schema_file_path, custom_file_path) + shutil.copy(self.original_invalidUri_file_path, custom_file_path) + shutil.copy(self.original_invalidUri_schema_file_path, self.custom_dir) os.chdir(self.custom_dir) self.setUpCreds(custom_file_path, { @@ -458,7 +463,7 @@ def test_profile_loading_with_invalid_schema_internet_URI(self, get_pass_func): # Test prof_manager = ProfileManager(appname="invalidUri.zowe") prof_manager.config_dir = self.custom_dir - props: dict = prof_manager.load(profile_name="zosmf", validate_schema=True) + props: dict = prof_manager.load(profile_name="zosmf", validate_schema=True, verify=False) @patch("keyring.get_password", side_effect=keyring_get_password) def test_profile_loading_with_env_variables(self, get_pass_func): @@ -504,7 +509,7 @@ def test_validate_config_json_valid(self): schema_json = commentjson.load(open(path_to_schema)) expected = validate(config_json, schema_json) - result = validate_config_json(path_to_config, path_to_schema) + result = validate_config_json(path_to_config, path_to_schema, cwd = FIXTURES_PATH) self.assertEqual(result, expected) @@ -520,6 +525,6 @@ def test_validate_config_json_invalid(self): validate(invalid_config_json, invalid_schema_json) with self.assertRaises(ValidationError) as actual_info: - validate_config_json(path_to_invalid_config, path_to_invalid_schema) + validate_config_json(path_to_invalid_config, path_to_invalid_schema, cwd = FIXTURES_PATH) self.assertEqual(str(actual_info.exception), str(expected_info.exception)) From cba960c007a1120a940d3ec05dcdb7a751efcca4 Mon Sep 17 00:00:00 2001 From: aadityasinha-dotcom Date: Mon, 28 Aug 2023 15:10:14 +0530 Subject: [PATCH 24/33] few changes added file protocol, fixed tests Signed-off-by: aadityasinha-dotcom --- src/core/zowe/core_for_zowe_sdk/config_file.py | 17 ++++++++--------- .../zowe/core_for_zowe_sdk/profile_manager.py | 3 +-- src/core/zowe/core_for_zowe_sdk/validators.py | 4 ++-- tests/unit/test_zowe_core.py | 12 ++++++------ 4 files changed, 17 insertions(+), 19 deletions(-) diff --git a/src/core/zowe/core_for_zowe_sdk/config_file.py b/src/core/zowe/core_for_zowe_sdk/config_file.py index aff4771f..5bc1f8d9 100644 --- a/src/core/zowe/core_for_zowe_sdk/config_file.py +++ b/src/core/zowe/core_for_zowe_sdk/config_file.py @@ -83,7 +83,7 @@ class ConfigFile: defaults: Optional[dict] = None schema_property: Optional[dict] = None secure_props: Optional[dict] = None - schema_property: Optional[dict] = None + jsonc: Optional[dict] = None _missing_secure_props: list = field(default_factory=list) @property @@ -121,7 +121,7 @@ def location(self, dirname: str) -> None: def init_from_file( self, config_type: str, - validate_schema: Optional[bool] = False, + validate_schema: Optional[bool] = True, verify: Optional[bool] = False, ) -> None: """ @@ -137,10 +137,10 @@ def init_from_file( self.profiles = profile_jsonc.get("profiles", {}) self.schema_property = profile_jsonc.get("$schema", None) self.defaults = profile_jsonc.get("defaults", {}) - self.schema_property = profile_jsonc.get("$schema", None) + self.jsonc = profile_jsonc - if self.schema_property: - self.validate_schema(config_type, validate_schema, verify=verify) + if self.schema_property and validate_schema: + self.validate_schema(config_type, validate_schema) # loading secure props is done in load_profile_properties # since we want to try loading secure properties only when # we know that the profile has saved properties @@ -149,7 +149,6 @@ def init_from_file( def validate_schema( self, config_type: str, - validate_schema: Optional[bool] = False, verify: Optional[bool] = False, ) -> None: """ @@ -172,8 +171,8 @@ def validate_schema( ) # validate the $schema property - if path_schema_json and validate_schema: - validate_config_json(path_config_json, path_schema_json, cwd = self.location, verify=verify) + if path_schema_json: + validate_config_json(self.jsonc, path_schema_json, cwd = self.location, verify=verify) def schema_list( self, @@ -224,7 +223,7 @@ def get_profile( profile_name: Optional[str] = None, profile_type: Optional[str] = None, config_type: Optional[str] = None, - validate_schema: Optional[bool] = False, + validate_schema: Optional[bool] = True, verify: Optional[bool] = False, ) -> Profile: """ diff --git a/src/core/zowe/core_for_zowe_sdk/profile_manager.py b/src/core/zowe/core_for_zowe_sdk/profile_manager.py index f8ff0b46..3ce26fdc 100644 --- a/src/core/zowe/core_for_zowe_sdk/profile_manager.py +++ b/src/core/zowe/core_for_zowe_sdk/profile_manager.py @@ -230,7 +230,6 @@ def get_profile( SecurePropsNotFoundWarning, ) except Exception as exc: - print("hi", exc) warnings.warn( f"Could not load {config_type} '{cfg.filename}' at '{cfg.filepath}'" f"because {type(exc).__name__}'{exc}'.", @@ -244,7 +243,7 @@ def load( profile_name: Optional[str] = None, profile_type: Optional[str] = None, check_missing_props: bool = True, - validate_schema: Optional[bool] = False, + validate_schema: Optional[bool] = True, override_with_env: Optional[bool] = False, verify: Optional[bool] = False, ) -> dict: diff --git a/src/core/zowe/core_for_zowe_sdk/validators.py b/src/core/zowe/core_for_zowe_sdk/validators.py index 5ae8ff2e..81361076 100644 --- a/src/core/zowe/core_for_zowe_sdk/validators.py +++ b/src/core/zowe/core_for_zowe_sdk/validators.py @@ -39,8 +39,8 @@ def validate_config_json(path_config_json: Union[str, dict], path_schema_json: s schema_json = requests.get(path_schema_json, verify = verify).json() # checks if the path_schema_json is a file - elif os.path.isfile(path_schema_json): - with open(path_schema_json) as file: + elif os.path.isfile(path_schema_json) or path_schema_json.startswith("file://"): + with open(path_schema_json.replace("file://", "")) as file: schema_json = commentjson.load(file) # checks if the path_schema_json is absolute diff --git a/tests/unit/test_zowe_core.py b/tests/unit/test_zowe_core.py index 50bdca45..fda364bd 100644 --- a/tests/unit/test_zowe_core.py +++ b/tests/unit/test_zowe_core.py @@ -239,7 +239,7 @@ def test_autodiscovery_and_base_profile_loading(self, get_pass_func): # Test prof_manager = ProfileManager() - props: dict = prof_manager.load(profile_type="base") + props: dict = prof_manager.load(profile_type="base", validate_schema=False) self.assertEqual(prof_manager.config_filepath, cwd_up_file_path) expected_props = { @@ -269,7 +269,7 @@ def test_custom_file_and_custom_profile_loading(self, get_pass_func): # Test prof_manager = ProfileManager(appname=self.custom_appname) prof_manager.config_dir = self.custom_dir - props: dict = prof_manager.load(profile_name="zosmf") + props: dict = prof_manager.load(profile_name="zosmf", validate_schema=False) self.assertEqual(prof_manager.config_filepath, custom_file_path) expected_props = { @@ -300,7 +300,7 @@ def test_custom_file_and_custom_profile_loading_with_nested_profile(self, get_pa # Test prof_manager = ProfileManager(appname=self.custom_appname) prof_manager.config_dir = self.custom_dir - props: dict = prof_manager.load(profile_name="lpar1.zosmf") + props: dict = prof_manager.load(profile_name="lpar1.zosmf", validate_schema=False) self.assertEqual(prof_manager.config_filepath, custom_file_path) expected_props = { @@ -331,7 +331,7 @@ def test_profile_loading_with_user_overriden_properties(self, get_pass_func): # Test prof_manager = ProfileManager() - props: dict = prof_manager.load(profile_type="zosmf") + props: dict = prof_manager.load(profile_type="zosmf", validate_schema=False) self.assertEqual(prof_manager.config_filepath, cwd_up_file_path) expected_props = { @@ -362,7 +362,7 @@ def test_profile_loading_exception(self, get_pass_func): # Test config_file = ConfigFile(name=self.custom_appname, type="team_config") - props: dict = config_file.get_profile(profile_name="non_existent_profile") + props: dict = config_file.get_profile(profile_name="non_existent_profile", validate_schema=False) @patch("keyring.get_password", side_effect=keyring_get_password_exception) def test_secure_props_loading_warning(self, get_pass_func): @@ -398,7 +398,7 @@ def test_profile_not_found_warning(self, get_pass_func): # Test prof_manager = ProfileManager() prof_manager.config_dir = self.custom_dir - props: dict = prof_manager.load("non_existent_profile") + props: dict = prof_manager.load("non_existent_profile", validate_schema=False) @patch("keyring.get_password", side_effect=keyring_get_password) def test_profile_loading_with_valid_schema(self, get_pass_func): From 6917e600b3d3d573490716792001756954cdeda7 Mon Sep 17 00:00:00 2001 From: aadityasinha-dotcom Date: Mon, 4 Sep 2023 15:03:19 +0530 Subject: [PATCH 25/33] removed verify parameter Signed-off-by: aadityasinha-dotcom --- requirements.txt | 2 +- src/core/zowe/core_for_zowe_sdk/config_file.py | 17 +++++------------ .../zowe/core_for_zowe_sdk/profile_manager.py | 4 +--- src/core/zowe/core_for_zowe_sdk/validators.py | 4 ++-- tests/unit/test_zowe_core.py | 3 ++- 5 files changed, 11 insertions(+), 19 deletions(-) diff --git a/requirements.txt b/requirements.txt index dd60b51e..dfb85eaf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ coverage==5.4 flake8==3.8.4 idna==2.10 importlib-metadata==3.6.0 -jsonschema==4.19.0 +jsonschema==4.17.3 keyring lxml==4.9.3 mccabe==0.6.1 diff --git a/src/core/zowe/core_for_zowe_sdk/config_file.py b/src/core/zowe/core_for_zowe_sdk/config_file.py index 2d9d3a11..dc4916cf 100644 --- a/src/core/zowe/core_for_zowe_sdk/config_file.py +++ b/src/core/zowe/core_for_zowe_sdk/config_file.py @@ -115,7 +115,6 @@ def init_from_file( self, config_type: str, validate_schema: Optional[bool] = True, - verify: Optional[bool] = False, ) -> None: """ Initializes the class variable after @@ -133,16 +132,14 @@ def init_from_file( self.jsonc = profile_jsonc if self.schema_property and validate_schema: - self.validate_schema(config_type, validate_schema) + self.validate_schema() # loading secure props is done in load_profile_properties # since we want to try loading secure properties only when # we know that the profile has saved properties # self.load_secure_props() def validate_schema( - self, - config_type: str, - verify: Optional[bool] = False, + self ) -> None: """ Get the $schema_property from the config and load the schema @@ -152,10 +149,7 @@ def validate_schema( file_path to the $schema property """ - path_schema_json, path_config_json = None, "" - - if self._location: - path_config_json = f"./{self.filename}" + path_schema_json = None path_schema_json = self.schema_path if path_schema_json is None: # check if the $schema property is not defined @@ -165,7 +159,7 @@ def validate_schema( # validate the $schema property if path_schema_json: - validate_config_json(self.jsonc, path_schema_json, cwd = self.location, verify=verify) + validate_config_json(self.jsonc, path_schema_json, cwd = self.location) def schema_list( self, @@ -217,7 +211,6 @@ def get_profile( profile_type: Optional[str] = None, config_type: Optional[str] = None, validate_schema: Optional[bool] = True, - verify: Optional[bool] = False, ) -> Profile: """ Load given profile including secure properties and excluding values from base profile @@ -227,7 +220,7 @@ def get_profile( Returns a namedtuple called Profile """ if self.profiles is None: - self.init_from_file(config_type, validate_schema, verify) + self.init_from_file(config_type, validate_schema) if profile_name is None and profile_type is None: raise ProfileNotFound( diff --git a/src/core/zowe/core_for_zowe_sdk/profile_manager.py b/src/core/zowe/core_for_zowe_sdk/profile_manager.py index 3ce26fdc..69fcfed8 100644 --- a/src/core/zowe/core_for_zowe_sdk/profile_manager.py +++ b/src/core/zowe/core_for_zowe_sdk/profile_manager.py @@ -162,7 +162,6 @@ def get_profile( profile_type: Optional[str], config_type: str, validate_schema: Optional[bool] = True, - verify: Optional[bool] = False, ) -> Profile: """ Get just the profile from the config file (overriden with base props in the config file) @@ -245,7 +244,6 @@ def load( check_missing_props: bool = True, validate_schema: Optional[bool] = True, override_with_env: Optional[bool] = False, - verify: Optional[bool] = False, ) -> dict: """Load connection details from a team config profile. Returns @@ -287,7 +285,7 @@ def load( for i, (config_type, cfg) in enumerate(config_layers.items()): profile_loaded = self.get_profile( - cfg, profile_name, profile_type, config_type, validate_schema, verify + cfg, profile_name, profile_type, config_type, validate_schema ) # TODO Why don't user and password show up here for Project User Config? # Probably need to update load_profile_properties method in config_file.py diff --git a/src/core/zowe/core_for_zowe_sdk/validators.py b/src/core/zowe/core_for_zowe_sdk/validators.py index 81361076..7d0a1551 100644 --- a/src/core/zowe/core_for_zowe_sdk/validators.py +++ b/src/core/zowe/core_for_zowe_sdk/validators.py @@ -17,7 +17,7 @@ import requests -def validate_config_json(path_config_json: Union[str, dict], path_schema_json: str, cwd: str, verify: Optional[bool] = False): +def validate_config_json(path_config_json: Union[str, dict], path_schema_json: str, cwd: str): """ Function validating that zowe.config.json file matches zowe.schema.json. @@ -36,7 +36,7 @@ def validate_config_json(path_config_json: Union[str, dict], path_schema_json: s # checks if the path_schema_json point to an internet URI and download the schema using the URI if path_schema_json.startswith("https://") or path_schema_json.startswith("http://"): - schema_json = requests.get(path_schema_json, verify = verify).json() + schema_json = requests.get(path_schema_json).json() # checks if the path_schema_json is a file elif os.path.isfile(path_schema_json) or path_schema_json.startswith("file://"): diff --git a/tests/unit/test_zowe_core.py b/tests/unit/test_zowe_core.py index f1f1e6c2..9686c491 100644 --- a/tests/unit/test_zowe_core.py +++ b/tests/unit/test_zowe_core.py @@ -166,6 +166,7 @@ class TestZosmfProfileManager(TestCase): def setUp(self): """Setup fixtures for ZosmfProfile class.""" # setup pyfakefs + self.session_arguments = {"verify": False} self.setUpPyfakefs() self.original_file_path = os.path.join(FIXTURES_PATH, "zowe.config.json") self.original_user_file_path = os.path.join( @@ -642,7 +643,7 @@ def test_profile_loading_with_invalid_schema_internet_URI(self, get_pass_func): # Test prof_manager = ProfileManager(appname="invalidUri.zowe") prof_manager.config_dir = self.custom_dir - props: dict = prof_manager.load(profile_name="zosmf", validate_schema=True, verify=False) + props: dict = prof_manager.load(profile_name="zosmf", validate_schema=True) @patch("keyring.get_password", side_effect=keyring_get_password) def test_profile_loading_with_env_variables(self, get_pass_func): From fc5aa3a05adf4e7cb0b52b9bb7236f7e6904d637 Mon Sep 17 00:00:00 2001 From: aadityasinha-dotcom Date: Wed, 13 Sep 2023 18:47:03 +0530 Subject: [PATCH 26/33] removed config_type parameter Signed-off-by: aadityasinha-dotcom --- src/core/zowe/core_for_zowe_sdk/config_file.py | 4 +--- src/core/zowe/core_for_zowe_sdk/profile_manager.py | 5 ++--- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/src/core/zowe/core_for_zowe_sdk/config_file.py b/src/core/zowe/core_for_zowe_sdk/config_file.py index dc4916cf..97c8c29d 100644 --- a/src/core/zowe/core_for_zowe_sdk/config_file.py +++ b/src/core/zowe/core_for_zowe_sdk/config_file.py @@ -113,7 +113,6 @@ def location(self, dirname: str) -> None: def init_from_file( self, - config_type: str, validate_schema: Optional[bool] = True, ) -> None: """ @@ -209,7 +208,6 @@ def get_profile( self, profile_name: Optional[str] = None, profile_type: Optional[str] = None, - config_type: Optional[str] = None, validate_schema: Optional[bool] = True, ) -> Profile: """ @@ -220,7 +218,7 @@ def get_profile( Returns a namedtuple called Profile """ if self.profiles is None: - self.init_from_file(config_type, validate_schema) + self.init_from_file(validate_schema) if profile_name is None and profile_type is None: raise ProfileNotFound( diff --git a/src/core/zowe/core_for_zowe_sdk/profile_manager.py b/src/core/zowe/core_for_zowe_sdk/profile_manager.py index 69fcfed8..72a27f6f 100644 --- a/src/core/zowe/core_for_zowe_sdk/profile_manager.py +++ b/src/core/zowe/core_for_zowe_sdk/profile_manager.py @@ -160,7 +160,6 @@ def get_profile( cfg: ConfigFile, profile_name: Optional[str], profile_type: Optional[str], - config_type: str, validate_schema: Optional[bool] = True, ) -> Profile: """ @@ -176,7 +175,7 @@ def get_profile( cfg_profile = Profile() try: cfg_profile = cfg.get_profile( - profile_name=profile_name, profile_type=profile_type, config_type=config_type, validate_schema=validate_schema + profile_name=profile_name, profile_type=profile_type, validate_schema=validate_schema ) except jsonschema.exceptions.ValidationError as exc: raise jsonschema.exceptions.ValidationError( @@ -285,7 +284,7 @@ def load( for i, (config_type, cfg) in enumerate(config_layers.items()): profile_loaded = self.get_profile( - cfg, profile_name, profile_type, config_type, validate_schema + cfg, profile_name, profile_type, validate_schema ) # TODO Why don't user and password show up here for Project User Config? # Probably need to update load_profile_properties method in config_file.py From 36382beb8ce2874743f76268f911c7c919758fb9 Mon Sep 17 00:00:00 2001 From: samadpls Date: Fri, 15 Sep 2023 18:22:51 +0500 Subject: [PATCH 27/33] Fixed exception handling in session.py Signed-off-by: samadpls --- src/core/zowe/core_for_zowe_sdk/session.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/zowe/core_for_zowe_sdk/session.py b/src/core/zowe/core_for_zowe_sdk/session.py index 5d5cc6cb..5b85f652 100644 --- a/src/core/zowe/core_for_zowe_sdk/session.py +++ b/src/core/zowe/core_for_zowe_sdk/session.py @@ -45,7 +45,7 @@ def __init__(self, props: dict) -> None: if props.get("host") is not None: self.session: ISession = ISession(host=props.get("host")) else: - raise "Host must be supplied" + raise Exception("Host must be supplied") # determine authentication type if props.get("user") is not None and props.get("password") is not None: @@ -61,7 +61,7 @@ def __init__(self, props: dict) -> None: self.session.tokenValue = props.get("tokenValue") self.session.type = session_constants.AUTH_TYPE_BEARER else: - raise "An authentication method must be supplied" + raise Exception("An authentication method must be supplied") # set additional parameters self.session.basePath = props.get("basePath") From 561b370a44b900a11e6c9669433b5c96e0330135 Mon Sep 17 00:00:00 2001 From: aadityasinha-dotcom Date: Tue, 19 Sep 2023 18:50:29 +0530 Subject: [PATCH 28/33] fix tests Signed-off-by: aadityasinha-dotcom --- src/core/zowe/core_for_zowe_sdk/profile_manager.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/core/zowe/core_for_zowe_sdk/profile_manager.py b/src/core/zowe/core_for_zowe_sdk/profile_manager.py index 72a27f6f..f29a7853 100644 --- a/src/core/zowe/core_for_zowe_sdk/profile_manager.py +++ b/src/core/zowe/core_for_zowe_sdk/profile_manager.py @@ -160,6 +160,7 @@ def get_profile( cfg: ConfigFile, profile_name: Optional[str], profile_type: Optional[str], + config_type: Optional[str], validate_schema: Optional[bool] = True, ) -> Profile: """ @@ -284,7 +285,7 @@ def load( for i, (config_type, cfg) in enumerate(config_layers.items()): profile_loaded = self.get_profile( - cfg, profile_name, profile_type, validate_schema + cfg, profile_name, profile_type, config_type, validate_schema ) # TODO Why don't user and password show up here for Project User Config? # Probably need to update load_profile_properties method in config_file.py From fa0dc9401dc7b66d34dfe55c1dc63e29a2f7f727 Mon Sep 17 00:00:00 2001 From: Abdul Samad Siddiqui Date: Wed, 20 Sep 2023 20:23:27 +0300 Subject: [PATCH 29/33] Added - Bugfix: exception handling Signed-off-by: Abdul Samad Siddiqui --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dc3ec78c..74361e78 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to the Zowe Client Python SDK will be documented in this fil ## Recent Changes - Bugfix: Fixed issue for datasets and jobs with special characters in URL [#211] (https://github.com/zowe/zowe-client-python-sdk/issues/211) - +- Bugfix: Fixed exception handling in session.py [#217] (https://github.com/zowe/zowe-client-python-sdk/issues/217) - Feature: Added a CredentialManager class to securely retrieve values from credentials and manage multiple credential entries on Windows [#134](https://github.com/zowe/zowe-client-python-sdk/issues/134) - Feature: Added method to load profile properties from environment variables From 79d8910fa7927c79bbb7ad5c23786d58932aa3c7 Mon Sep 17 00:00:00 2001 From: Abdul Samad Siddiqui Date: Wed, 20 Sep 2023 20:51:34 +0300 Subject: [PATCH 30/33] Update CHANGELOG.md Signed-off-by: Abdul Samad Siddiqui --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 74361e78..0c813bb2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to the Zowe Client Python SDK will be documented in this fil ## Recent Changes - Bugfix: Fixed issue for datasets and jobs with special characters in URL [#211] (https://github.com/zowe/zowe-client-python-sdk/issues/211) -- Bugfix: Fixed exception handling in session.py [#217] (https://github.com/zowe/zowe-client-python-sdk/issues/217) +- Bugfix: Fixed exception handling in session.py [#213] (https://github.com/zowe/zowe-client-python-sdk/issues/213) - Feature: Added a CredentialManager class to securely retrieve values from credentials and manage multiple credential entries on Windows [#134](https://github.com/zowe/zowe-client-python-sdk/issues/134) - Feature: Added method to load profile properties from environment variables From f749d606a74edbc3106495d26bf4a60e3f1203e0 Mon Sep 17 00:00:00 2001 From: zFernand0 <37381190+zFernand0@users.noreply.github.com> Date: Wed, 27 Sep 2023 20:29:37 +0000 Subject: [PATCH 31/33] attempt to add the entire env/lib dir Signed-off-by: zFernand0 <37381190+zFernand0@users.noreply.github.com> --- .../zowe/core_for_zowe_sdk/config_file.py | 28 ++++++------- .../core_for_zowe_sdk/credential_manager.py | 40 +++++++++---------- .../zowe/core_for_zowe_sdk/profile_manager.py | 16 ++++---- tests/unit/test_zowe_core.py | 15 +++---- 4 files changed, 50 insertions(+), 49 deletions(-) diff --git a/src/core/zowe/core_for_zowe_sdk/config_file.py b/src/core/zowe/core_for_zowe_sdk/config_file.py index 97c8c29d..b9e01b3d 100644 --- a/src/core/zowe/core_for_zowe_sdk/config_file.py +++ b/src/core/zowe/core_for_zowe_sdk/config_file.py @@ -99,7 +99,7 @@ def filepath(self) -> Optional[str]: @property def location(self) -> Optional[str]: return self._location - + @property def schema_path(self) -> Optional[str]: return self.schema_property @@ -112,7 +112,7 @@ def location(self, dirname: str) -> None: raise FileNotFoundError(f"given path {dirname} is not valid") def init_from_file( - self, + self, validate_schema: Optional[bool] = True, ) -> None: """ @@ -155,22 +155,22 @@ def validate_schema( warnings.warn( f"$schema property could not found" ) - - # validate the $schema property + + # validate the $schema property if path_schema_json: validate_config_json(self.jsonc, path_schema_json, cwd = self.location) - + def schema_list( self, ) -> list: """ Loads the schema properties in a sorted order according to the priority - + Returns ------- Dictionary - + Returns the profile properties from schema (prop: value) """ @@ -181,23 +181,23 @@ def schema_list( if schema.startswith("https://") or schema.startswith("http://"): schema_json = requests.get(schema).json() + elif os.path.isfile(schema) or schema.startswith("file://"): + with open(schema.replace("file://", "")) as f: + schema_json = json.load(f) + elif not os.path.isabs(schema): schema = os.path.join(self.location, schema) with open(schema) as f: schema_json = json.load(f) - - elif os.path.isfile(schema): - with open(schema) as f: - schema_json = json.load(f) else: return [] profile_props:dict = {} schema_json = dict(schema_json) - + for props in schema_json['properties']['profiles']['patternProperties']["^\\S*$"]["allOf"]: props = props["then"] - + while "properties" in props: props = props.pop("properties") profile_props = props @@ -303,7 +303,7 @@ def get_profilename_from_profiletype(self, profile_type: str) -> str: profile_name=profile_type, error_msg=f"No profile with matching profile_type '{profile_type}' found", ) - + def find_profile(self, path: str, profiles: dict): """ Find a profile at a specified location from within a set of nested profiles diff --git a/src/core/zowe/core_for_zowe_sdk/credential_manager.py b/src/core/zowe/core_for_zowe_sdk/credential_manager.py index 656878c3..de9de637 100644 --- a/src/core/zowe/core_for_zowe_sdk/credential_manager.py +++ b/src/core/zowe/core_for_zowe_sdk/credential_manager.py @@ -10,8 +10,8 @@ Copyright Contributors to the Zowe Project. """ import sys -import warnings -import base64 +import warnings +import base64 import logging from typing import Optional import commentjson @@ -30,7 +30,7 @@ class CredentialManager: secure_props = {} - + @staticmethod def load_secure_props() -> None: @@ -51,7 +51,7 @@ def load_secure_props() -> None: secret_value = CredentialManager._retrieve_credential(service_name) # Handle the case when secret_value is None if secret_value is None: - return + return except Exception as exc: raise SecureProfileLoadFailed( @@ -63,9 +63,9 @@ def load_secure_props() -> None: secure_config_json = commentjson.loads(base64.b64decode(secure_config).decode()) # update the secure props CredentialManager.secure_props = secure_config_json - - - @staticmethod + + + @staticmethod def _retrieve_credential(service_name: str) -> Optional[str]: """ Retrieve the credential from the keyring or storage. @@ -96,7 +96,7 @@ def _retrieve_credential(service_name: str) -> Optional[str]: encoded_credential += temp_value index += 1 temp_value = keyring.get_password(f"{service_name}-{index}", f"{constants['ZoweAccountName']}-{index}") - + if is_win32: try: encoded_credential = encoded_credential.encode('utf-16le').decode() @@ -106,10 +106,10 @@ def _retrieve_credential(service_name: str) -> Optional[str]: if encoded_credential is not None and encoded_credential.endswith("\0"): encoded_credential = encoded_credential[:-1] - - return encoded_credential - - + + return encoded_credential + + @staticmethod def delete_credential(service_name: str, account_name: str) -> None: """ @@ -125,7 +125,7 @@ def delete_credential(service_name: str, account_name: str) -> None: ------- None """ - + try: keyring.delete_password(service_name, account_name) except keyring.errors.PasswordDeleteError: @@ -143,7 +143,7 @@ def delete_credential(service_name: str, account_name: str) -> None: break index += 1 - + @staticmethod def save_secure_props()-> None: """ @@ -154,16 +154,16 @@ def save_secure_props()-> None: """ if not HAS_KEYRING: return - + service_name = constants["ZoweServiceName"] credential = CredentialManager.secure_props # Check if credential is a non-empty string if credential: is_win32 = sys.platform == "win32" - - encoded_credential = base64.b64encode(commentjson.dumps(credential).encode()).decode() + + encoded_credential = base64.b64encode(commentjson.dumps(credential).encode()).decode() if is_win32: - service_name += "/" + constants["ZoweAccountName"] + service_name += "/" + constants["ZoweAccountName"] # Delete the existing credential CredentialManager.delete_credential(service_name , constants["ZoweAccountName"]) # Check if the encoded credential exceeds the maximum length for win32 @@ -177,9 +177,9 @@ def save_secure_props()-> None: password=(chunk + '\0' *(len(chunk)%2)).encode().decode('utf-16le') field_name = f"{constants['ZoweAccountName']}-{index}" keyring.set_password(f"{service_name}-{index}", field_name, password) - + else: # Credential length is within the maximum limit or not on win32, set it as a single keyring entry keyring.set_password( - service_name, constants["ZoweAccountName"], + service_name, constants["ZoweAccountName"], encoded_credential) \ No newline at end of file diff --git a/src/core/zowe/core_for_zowe_sdk/profile_manager.py b/src/core/zowe/core_for_zowe_sdk/profile_manager.py index f29a7853..aa49b954 100644 --- a/src/core/zowe/core_for_zowe_sdk/profile_manager.py +++ b/src/core/zowe/core_for_zowe_sdk/profile_manager.py @@ -117,24 +117,24 @@ def get_env( ) -> dict: """ Maps the env variables to the profile properties - + Returns ------- Dictionary Containing profile properties from env variables (prop: value) """ - + props = cfg.schema_list() if props == []: return {} - + env, env_var = {}, {} - + for var in list(os.environ.keys()): if var.startswith("ZOWE_OPT"): env[var[len("ZOWE_OPT_"):].lower()] = os.environ.get(var) - + for k, v in env.items(): word = k.split("_") @@ -154,7 +154,7 @@ def get_env( env_var[k] = bool(v) return env_var - + @staticmethod def get_profile( cfg: ConfigFile, @@ -234,7 +234,7 @@ def get_profile( f"because {type(exc).__name__}'{exc}'.", ConfigNotFoundWarning, ) - + return cfg_profile def load( @@ -294,7 +294,7 @@ def load( profile_loaded.name ) # Define profile name that will be merged from other layers profile_props = {**profile_loaded.data, **profile_props} - + missing_secure_props.extend(profile_loaded.missing_secure_props) if override_with_env: diff --git a/tests/unit/test_zowe_core.py b/tests/unit/test_zowe_core.py index 3ff5456e..39e093ad 100644 --- a/tests/unit/test_zowe_core.py +++ b/tests/unit/test_zowe_core.py @@ -129,7 +129,7 @@ def test_encode_uri_component(self): """Test string is being adjusted to the correct URL parameter""" sdk_api = SdkApi(self.basic_props, self.default_url) - + actual_not_empty = sdk_api._encode_uri_component('MY.STRING@.TEST#.$HERE(MBR#NAME)') expected_not_empty = 'MY.STRING%40.TEST%23.%24HERE(MBR%23NAME)' self.assertEqual(actual_not_empty, expected_not_empty) @@ -183,7 +183,7 @@ class TestZosmfProfileManager(TestCase): def setUp(self): """Setup fixtures for ZosmfProfile class.""" # setup pyfakefs - self.session_arguments = {"verify": False} + self.session_arguments = {"verify": False} self.setUpPyfakefs() self.original_file_path = os.path.join(FIXTURES_PATH, "zowe.config.json") self.original_user_file_path = os.path.join( @@ -207,6 +207,7 @@ def setUp(self): self.original_invalidUri_schema_file_path = os.path.join( FIXTURES_PATH, "invalidUri.zowe.schema.json" ) + self.fs.add_real_directory(os.path.join(FIXTURES_PATH, "../../../env/lib")) self.fs.add_real_file(self.original_file_path) self.fs.add_real_file(self.original_user_file_path) self.fs.add_real_file(self.original_nested_file_path) @@ -398,7 +399,7 @@ def test_secure_props_loading_warning(self, get_pass_func): prof_manager = ProfileManager() prof_manager.config_dir = self.custom_dir props: dict = prof_manager.load("base") - + @patch("keyring.get_password", side_effect=keyring_get_password) def test_profile_not_found_warning(self, get_pass_func): """ @@ -416,7 +417,7 @@ def test_profile_not_found_warning(self, get_pass_func): prof_manager = ProfileManager() prof_manager.config_dir = self.custom_dir props: dict = prof_manager.load("non_existent_profile", validate_schema=False) - + @patch("sys.platform", "win32") @patch("zowe.core_for_zowe_sdk.CredentialManager._retrieve_credential") def test_load_secure_props(self, retrieve_cred_func): @@ -521,7 +522,7 @@ def test_save_secure_props_normal_credential(self, delete_pass_func, retrieve_cr """ Test the save_secure_props method for saving credentials to keyring. """ - + # Set up mock values and expected results service_name = constants["ZoweServiceName"] + "/" + constants["ZoweAccountName"] # Setup - copy profile to fake filesystem created by pyfakefs @@ -556,7 +557,7 @@ def test_save_secure_props_normal_credential(self, delete_pass_func, retrieve_cr @patch("keyring.set_password") @patch("zowe.core_for_zowe_sdk.CredentialManager.delete_credential") def test_save_secure_props_exceed_limit(self, delete_pass_func, set_pass_func, retrieve_cred_func): - + # Set up mock values and expected results service_name = constants["ZoweServiceName"] + "/" + constants["ZoweAccountName"] # Setup - copy profile to fake filesystem created by pyfakefs @@ -617,7 +618,7 @@ def test_profile_loading_with_valid_schema(self, get_pass_func): prof_manager = ProfileManager(appname="zowe") prof_manager.config_dir = self.custom_dir props: dict = prof_manager.load(profile_name="zosmf") - + @patch("keyring.get_password", side_effect=keyring_get_password) def test_profile_loading_with_invalid_schema(self, get_pass_func): """ From e355a7c3ebe8aee9557f266f6f9fc549183d1178 Mon Sep 17 00:00:00 2001 From: zFernand0 <37381190+zFernand0@users.noreply.github.com> Date: Wed, 27 Sep 2023 20:37:22 +0000 Subject: [PATCH 32/33] cannot assume that env/lib is the reason why the merging didn't work Signed-off-by: zFernand0 <37381190+zFernand0@users.noreply.github.com> --- tests/unit/test_zowe_core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_zowe_core.py b/tests/unit/test_zowe_core.py index 39e093ad..9f49de79 100644 --- a/tests/unit/test_zowe_core.py +++ b/tests/unit/test_zowe_core.py @@ -207,7 +207,7 @@ def setUp(self): self.original_invalidUri_schema_file_path = os.path.join( FIXTURES_PATH, "invalidUri.zowe.schema.json" ) - self.fs.add_real_directory(os.path.join(FIXTURES_PATH, "../../../env/lib")) + # self.fs.add_real_directory(os.path.join(FIXTURES_PATH, "../../../env/lib")) self.fs.add_real_file(self.original_file_path) self.fs.add_real_file(self.original_user_file_path) self.fs.add_real_file(self.original_nested_file_path) From b824ac0e518b57c30610db6a71078e2c6c8ded59 Mon Sep 17 00:00:00 2001 From: zFernand0 <37381190+zFernand0@users.noreply.github.com> Date: Wed, 27 Sep 2023 21:02:35 +0000 Subject: [PATCH 33/33] try to load jsonschema Signed-off-by: zFernand0 <37381190+zFernand0@users.noreply.github.com> --- tests/unit/test_zowe_core.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/tests/unit/test_zowe_core.py b/tests/unit/test_zowe_core.py index 9f49de79..bf71ef6f 100644 --- a/tests/unit/test_zowe_core.py +++ b/tests/unit/test_zowe_core.py @@ -2,18 +2,20 @@ # Including necessary paths import base64 +import commentjson +import importlib.util import json +import keyring import os import shutil -import keyring -import sys import unittest + +from jsonschema import validate, ValidationError, SchemaError +from pyfakefs.fake_filesystem_unittest import TestCase from unittest import mock from unittest.mock import patch -from jsonschema import validate, ValidationError, SchemaError + from zowe.core_for_zowe_sdk.validators import validate_config_json -import commentjson -from pyfakefs.fake_filesystem_unittest import TestCase from zowe.core_for_zowe_sdk import ( ApiConnection, ConfigFile, @@ -207,7 +209,11 @@ def setUp(self): self.original_invalidUri_schema_file_path = os.path.join( FIXTURES_PATH, "invalidUri.zowe.schema.json" ) - # self.fs.add_real_directory(os.path.join(FIXTURES_PATH, "../../../env/lib")) + + loader = importlib.util.find_spec('jsonschema') + module_path = loader.origin + self.fs.add_real_directory(os.path.dirname(module_path)) + self.fs.add_real_file(self.original_file_path) self.fs.add_real_file(self.original_user_file_path) self.fs.add_real_file(self.original_nested_file_path)