From e55e846ab182e7086dc3079191150bc1910e4179 Mon Sep 17 00:00:00 2001 From: nsheff Date: Wed, 6 Dec 2023 09:16:46 -0500 Subject: [PATCH 01/14] import load_yaml --- yacman/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yacman/__init__.py b/yacman/__init__.py index 3ba6395..c7fb4d1 100644 --- a/yacman/__init__.py +++ b/yacman/__init__.py @@ -2,6 +2,6 @@ from .alias import * # For transition -from .yacman1 import YAMLConfigManager, select_config +from .yacman1 import YAMLConfigManager, select_config, load_yaml from .yacman import * From 5d076a39b4766ddfe3506ca3fc9c79cf140dffd7 Mon Sep 17 00:00:00 2001 From: nsheff Date: Thu, 1 Feb 2024 11:28:00 -0500 Subject: [PATCH 02/14] upgrades --- README.md | 55 ++++ yacman/__init__.py | 9 +- yacman/yacman_future.py | 571 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 633 insertions(+), 2 deletions(-) create mode 100644 yacman/yacman_future.py diff --git a/README.md b/README.md index bccee3b..7dfdf41 100644 --- a/README.md +++ b/README.md @@ -8,3 +8,58 @@ Yacman is a YAML configuration manager. It provides some convenience tools for dealing with YAML configuration files. Please see [this](docs/usage.md) Python notebook for features and usage instructions and [this](docs/api_docs.md) document for API documentation. + +## Upgrading guide + +How to upgrade to yacman v1.0.0. + +There are some transition objects in the 0.9.3 to help with transition. + +### Use the FutureYAMLConfigManager in 0.9.3 + +1. Import the FutureYAMLConfigManager + +Change from: + +``` +from yacman import YAMLConfigManager +``` + +to + +``` +from yacman import FutureYAMLConfigManager as YAMLConfigManager +``` + +Once we switch from `v0.9.3` to `v1.X.X`, you will need to switch back. + +2. Update any context managers to use `write_lock` or `read_lock` + +``` +from yacman import write_lock, read_lock +``` + +Change + +``` +with ym as locked_ym: + locked_ym.write() + +``` + +to + + +``` +with write_lock(ym) as locked_ym: + locked_ym.write() +``` + + +## From v0.9.3 (using future) to v1.X.X: + +Switch back to: + +``` +from yacman import YAMLConfigManager +``` diff --git a/yacman/__init__.py b/yacman/__init__.py index c7fb4d1..0ff6551 100644 --- a/yacman/__init__.py +++ b/yacman/__init__.py @@ -1,7 +1,12 @@ from ._version import __version__ from .alias import * -# For transition +# Origina version +from .yacman import * + +# For transition, mostly backwards-compatible version from .yacman1 import YAMLConfigManager, select_config, load_yaml -from .yacman import * +# Future version (not backwards-compatible) +from .yacman_future import FutureYAMLConfigManager +from ubiquerg import read_lock, write_lock diff --git a/yacman/yacman_future.py b/yacman/yacman_future.py new file mode 100644 index 0000000..1d591e8 --- /dev/null +++ b/yacman/yacman_future.py @@ -0,0 +1,571 @@ +import logging +import os +import yaml + +from collections.abc import Iterable, Mapping +from jsonschema import validate as _validate +from jsonschema.exceptions import ValidationError +from sys import _getframe +from ubiquerg import ( + expandpath, + is_url, + ThreeLocker, + ensure_locked, + locked_read_file, + READ, + WRITE, +) + +from ._version import __version__ + +_LOGGER = logging.getLogger(__name__) +_LOGGER.debug(f"Using yacman version {__version__}") + +# Hack for yaml string indexes +# Credit: Anthon +# https://stackoverflow.com/questions/50045617 +# https://stackoverflow.com/questions/5121931 +# The idea is: if you have yaml keys that can be interpreted as an int or a float, +# then the yaml loader will convert them into an int or a float, and you would +# need to access them with dict[2] instead of dict['2']. But since we always +# expect the keys to be strings, this doesn't work. So, here we are adjusting +# the loader to keep everything as a string. + +# Only do once. +if not hasattr(yaml.SafeLoader, "patched_yaml_loader"): + _LOGGER.debug("Patching yaml loader") + + def my_construct_mapping(self, node, deep=False): + data = self.construct_mapping_org(node, deep) + return { + (str(key) if isinstance(key, float) or isinstance(key, int) else key): data[ + key + ] + for key in data + } + + yaml.SafeLoader.construct_mapping_org = yaml.SafeLoader.construct_mapping + yaml.SafeLoader.construct_mapping = my_construct_mapping + yaml.SafeLoader.patched_yaml_loader = True + +# Constants: to do, remove these + +DEFAULT_WAIT_TIME = 60 +# LOCK_PREFIX = "lock." +SCHEMA_KEY = "schema" +FILEPATH_KEY = "file_path" + +from collections.abc import MutableMapping + +# Since read and write are now different context managers, we have to +# separate them like this, instead of using __enter__ and __exit__ on the class +# itself, which only allows one type of context manager. + + +class FutureYAMLConfigManager(MutableMapping): + """ + A YAML configuration manager, providing file locking, loading, + writing, etc. for YAML configuration files. + """ + + def __init__( + self, + entries=None, + wait_max=DEFAULT_WAIT_TIME, + strict_ro_locks=False, + schema_source=None, + validate_on_write=False, + ): + """ + Object constructor + + :param Iterable[(str, object)] | Mapping[str, object] entries: YAML collection + of key-value pairs. + :param str yamldata: YAML-formatted string + :param int wait_max: how long to wait for creating an object when the file + that data will be read from is locked + :param bool strict_ro_locks: By default, we allow RO filesystems that can't be locked. + Turn on strict_ro_locks to error if locks cannot be enforced on readonly filesystems. + :param bool skip_read_lock: whether the file should not be locked for reading + when object is created in read only mode + :param str schema_source: path or a URL to a jsonschema in YAML format to use + for optional config validation. If this argument is provided the object + is always validated at least once, at the object creation stage. + :param bool validate_on_write: a boolean indicating whether the object should be + validated every time the `write` method is executed, which is + a way of preventing invalid config writing + + """ + + # Settings for this config object + self.filepath = None + self.wait_max = wait_max + self.schema_source = schema_source + self.validate_on_write = validate_on_write + self.strict_ro_locks = strict_ro_locks + self.locker = None + + # We store the values in a dict under .data + if isinstance(entries, list): + self.data = entries + else: + self.data = dict(entries or {}) + if schema_source is not None: + assert isinstance(schema_source, str), TypeError( + f"Path to the schema to validate the config must be a string" + ) + sp = expandpath(schema_source) + assert os.path.exists(sp), FileNotFoundError( + f"Provided schema file does not exist: {schema_source}." + f" Also tried: {sp}" + ) + # validate config + setattr(self, SCHEMA_KEY, load_yaml(sp)) + self.validate() + + @classmethod + def from_obj(cls, entries: object, **kwargs): + """ + Initialize from a Python object (dict, list, or primitive). + + :param obj entries: object to initialize from. + :param kwargs: Keyword arguments to pass to the constructor. + """ + return cls(entries, **kwargs) + + @classmethod + def from_yaml_data(cls, yamldata, **kwargs): + """ + Initialize from a YAML string. + + :param str yamldata: YAML-formatted string. + :param kwargs: Keyword arguments to pass to the constructor. + """ + entries = yaml.load(yamldata, yaml.SafeLoader) + return cls(entries, **kwargs) + + @classmethod + def from_yaml_file(cls, filepath: str, create_file: bool = False, **kwargs): + """ + Initialize from a YAML file. + + :param str filepath: Path to the YAML config file. + :param str create_file: Create a file at filepath if it doesn't exist. + :param kwargs: Keyword arguments to pass to the constructor. + """ + + file_contents = locked_read_file(filepath, create_file=create_file) + entries = yaml.load(file_contents, yaml.SafeLoader) + ref = cls(entries, **kwargs) + ref.locker = ThreeLocker(filepath) + return ref + + def update_from_yaml_file(self, filepath=None): + self.data.update(load_yaml(filepath)) + return + + def update_from_yaml_data(self, yamldata=None): + self.data.update(yaml.load(yamldata, yaml.SafeLoader)) + return + + def update_from_obj(self, entries=None): + self.data.update(entries) + return + + @property + def settings(self): + return { + "wait_max": self.wait_max, + "schema_source": self.schema_source, + "validate_on_write": self.validate_on_write, + "locked": self.locked, + "strict_ro_locks": self.strict_ro_locks, + } + + def __del__(self): + if hasattr(self, "locker"): + del self.locker + + def __repr__(self): + # Render the data in a nice way + return self.to_yaml(self.data) + + def __enter__(self): + raise NotImplementedError("Use the 'read_lock' and 'write_lock' context managers.") + + def __exit__(self): + raise NotImplementedError("Use the 'read_lock' and 'write_lock' context managers.") + + @ensure_locked(READ) + def rebase(self, filepath=None): + """ + Reload the object from file, then update with current information + + :param str filepath: path to the file that should be read + """ + fp = filepath or self.locker.filepath + if fp is not None: + local_data = self.data + self.data = load_yaml(fp) + deep_update(self.data, local_data) + else: + _LOGGER.warning("Rebase has no effect if no filepath given") + + return self + + @ensure_locked(READ) + def reset(self, filepath=None): + """ + Reset dict contents to file contents, or to empty dict if no filepath found. + """ + fp = filepath or self.locker.filepath + if fp is not None: + self.data = load_yaml(fp) + else: + self.data = {} + return self + + def validate(self, schema=None, exclude_case=False): + """ + Validate the object against a schema + + :param dict schema: a schema object to use to validate, it overrides the one + that has been provided at object construction stage + :param bool exclude_case: whether to exclude validated objects + from the error. Useful when used with large configs + """ + try: + _validate(self.to_dict(expand=True), schema or getattr(self, SCHEMA_KEY)) + except ValidationError as e: + _LOGGER.error( + f"{self.__class__.__name__} object did not pass schema validation" + ) + # if getattr(self, FILEPATH_KEY, None) is not None: + # need to unlock locked files in case of validation error so that no + # locks are left in place + # self.make_readonly() + # commented out because I think this is taken care of my context managers now + if not exclude_case: + raise + raise ValidationError( + f"{self.__class__.__name__} object did not pass schema validation: " + f"{e.message}" + ) + _LOGGER.debug("Validated successfully") + + @ensure_locked(WRITE) + def write(self, schema=None, exclude_case=False): + """ + Write the contents to the file backing this object. + + :param dict schema: a schema object to use to validate, it overrides the one + that has been provided at object construction stage + :raise OSError: when the object has been created in a read only mode or other + process has locked the file + :raise TypeError: when the filepath cannot be determined. This takes place only + if YacAttMap initialized with a Mapping as an input, not read from file. + :raise OSError: when the write is called on an object with no write capabilities + or when writing to a file that is locked by a different object + :return str: the path to the created files + """ + if not self.locker.filepath: + raise OSError("Must provide a filepath to write.") + + _check_filepath(self.locker.filepath) + _LOGGER.debug(f"Writing to file '{self.locker.filepath}'") + with open(self.locker.filepath, "w") as f: + f.write(self.to_yaml()) + + if schema is not None or self.validate_on_write: + self.validate(schema=schema, exclude_case=exclude_case) + + abs_path = os.path.abspath(self.locker.filepath) + _LOGGER.debug(f"Wrote to a file: {abs_path}") + return os.path.abspath(abs_path) + + def write_copy(self, filepath=None): + """ + Write the contents to an external file. + + :param str filepath: a file path to write to + """ + + _LOGGER.debug(f"Writing to file '{filepath}'") + with open(filepath, "w") as f: + f.write(self.to_yaml()) + return filepath + + def to_yaml(self, trailing_newline=False, expand=False): + """ + Get text for YAML representation. + + :param bool trailing_newline: whether to add trailing newline + :param bool expand: whether to expand paths in values + :return str: YAML text representation of this instance. + """ + + if expand: + return yaml.dump(self.exp, default_flow_style=False) + return yaml.dump(self.data, default_flow_style=False) + ( + "\n" if trailing_newline else "" + ) + + def to_dict(self, expand=True): + # Seems like it's probably not necessary; can just use the object now. + # but for backwards compatibility. + return self.data + + def __setitem__(self, item, value): + self.data[item] = value + + def __getitem__(self, item): + """ + Fetch the value of given key. + + :param hashable item: key for which to fetch value + :return object: value mapped to given key, if available + :raise KeyError: if the requested key is unmapped. + """ + return self.data[item] + + @property + def exp(self) -> dict: + """ + Returns a copy of the object's data elements with env vars and user vars + expanded. Use it like: object.exp["item"] + """ + return _safely_expand_path(self.data) + + def __iter__(self): + return iter(self.data) + + def __len__(self): + return len(self.data) + + def __delitem__(self, key): + value = self[key] + del self.data[key] + self.pop(value, None) + + def priority_get( + self, + arg_name: str, + env_var: str = None, + default: str = None, + override: str = None, + strict: bool = False, + ): + """ + Helper function to select a value from a config, or, if missing, then go to an env var. + + :param str arg_name: Argument to retrieve + :param bool strict: Should missing args raise an error? False=warning + :param env_var: Env var to retrieve from should it be missing from the cfg + """ + if override: + return override + if self.data.get(arg_name) is not None: + return self.data[arg_name] + if env_var is not None: + arg = os.getenv(env_var, None) + if arg is not None: + _LOGGER.debug(f"Value '{arg}' sourced from '{env_var}' env var") + return expandpath(arg) + if default is not None: + return default + if strict: + message = ( + "Value for required argument '{arg_name}' could not be determined." + ) + _LOGGER.warning(message) + raise Exception(message) + + +# A big issue here is: if you route the __getitem__ through this, +# then it returns a copy of the data, rather than the data itself. +# That's the point, so we don't adjust it. But then you can't use multi-level +# item setting, like ycm["x"]["y"] = value, because ycm['x'] returns a different +# dict, and so you're updating that copy of it. +# The solution is that we have to route expansion through a separate property, +# so the setitem syntax can remain intact while preserving original values. +def _safely_expand_path(x): + if isinstance(x, str): + return expandpath(x) + elif isinstance(x, Mapping): + return {k: _safely_expand_path(v) for k, v in x.items()} + return x + + +def _unsafely_expand_path(x): + if isinstance(x, str): + return expandpath(x) + elif isinstance(x, Mapping): + for k in x.keys(): + x[k] = _safely_expand_path(x[k]) + return x + # return {k: _safely_expand_path(v) for k, v in x.items()} + return x + + +def _check_filepath(filepath): + """ + Validate if the filepath is a str + + :param str filepath: object to validate + :return str: validated filepath + :raise TypeError: if the filepath is not a string + """ + # might be useful if we want to have multiple locked paths in the future + # def _check_string(obj): + # """ check if object is a string or a list of strings """ + # return bool(obj) and all(isinstance(elem, str) for elem in obj) + if not isinstance(filepath, str): + raise TypeError( + f"No valid filepath provided. It must be a str, got: {filepath.__class__.__name__}" + ) + return filepath + + +def _warn_deprecated(obj): + fun_name = _getframe().f_back.f_code.co_name + warnings.warn( + f"The '{fun_name}' property is deprecated and will be removed in a future release." + f' Use {obj.__class__.__name__}["{IK}"]["{fun_name}"] instead.', + UserWarning, + stacklevel=4, + ) + + +def load_yaml(filepath): + """Load a yaml file into a python dict""" + + def read_yaml_file(filepath): + """ + Read a YAML file + + :param str filepath: path to the file to read + :return dict: read data + """ + with open(filepath, "r") as f: + data = yaml.safe_load(f) + return data + + if is_url(filepath): + _LOGGER.debug(f"Got URL: {filepath}") + try: # python3 + from urllib.error import HTTPError + from urllib.request import urlopen + except: # python2 + from urllib2 import URLError as HTTPError + from urllib2 import urlopen + try: + response = urlopen(filepath) + except HTTPError as e: + raise e + data = response.read() # a `bytes` object + text = data.decode("utf-8") + return yaml.safe_load(text) + else: + return read_yaml_file(filepath) + + +def select_config( + config_filepath: str = None, + config_env_vars=None, + default_config_filepath: str = None, + check_exist: bool = True, + on_missing=lambda fp: IOError(fp), + strict_env: bool = False, + config_name=None, +) -> str: + """ + Selects the config file to load. + + This uses a priority ordering to first choose a config filepath if it's given, + but if not, then look in a priority list of environment variables and choose + the first available filepath to return. + + :param str | NoneType config_filepath: direct filepath specification + :param Iterable[str] | NoneType config_env_vars: names of environment + variables to try for config filepaths + :param str default_config_filepath: default value if no other alternative + resolution succeeds + :param bool check_exist: whether to check for path existence as file + :param function(str) -> object on_missing: what to do with a filepath if it + doesn't exist + :param bool strict_env: whether to raise an exception if no file path provided + and environment variables do not point to any files + raise: OSError: when strict environment variables validation is not passed + """ + + # First priority: given file + if type(config_name) is str: + config_name = f"{config_name} " + else: + config_name = "" + + if type(config_env_vars) is str: + config_env_vars = [config_env_vars] + + if config_filepath: + config_filepath = os.path.expandvars(config_filepath) + if not check_exist or os.path.isfile(config_filepath): + return os.path.abspath(config_filepath) + _LOGGER.error(f"{config_name}config file path isn't a file: {config_filepath}") + result = on_missing(config_filepath) + if isinstance(result, Exception): + raise result + return os.path.abspath(result) + + _LOGGER.debug(f"No local {config_name}config file was provided.") + selected_filepath = None + + # Second priority: environment variables (in order) + if config_env_vars: + _LOGGER.debug( + f"Checking environment variables '{config_env_vars}' for {config_name}config" + ) + + for env_var in config_env_vars: + result = os.environ.get(env_var, None) + if result == None: + _LOGGER.debug(f"Env var '{env_var}' not set.") + continue + elif result == "": + _LOGGER.debug(f"Env var '{env_var}' exists, but value is empty.") + continue + elif not os.path.isfile(result): + _LOGGER.debug(f"Env var '{env_var}' file not found: {result}") + continue + else: + _LOGGER.debug(f"Found {config_name}config file in {env_var}: {result}") + selected_filepath = result + + if selected_filepath is None: + # Third priority: default filepath + if default_config_filepath: + _LOGGER.info( + f"Using default {config_name}config. You may specify in env var: {str(config_env_vars)}" + ) + return default_config_filepath + else: + if strict_env: + raise OSError("Unable to select config file.") + + _LOGGER.info(f"Could not locate {config_name}config file.") + return None + return ( + os.path.abspath(selected_filepath) if selected_filepath else selected_filepath + ) + + +def deep_update(old, new): + """ + Recursively update nested dict, modifying source + """ + for key, value in new.items(): + if isinstance(value, Mapping) and value: + old[key] = deep_update(old.get(key, {}), value) + else: + old[key] = new[key] + return old From 721ed2930690f4df6857023b39bf654832079d0c Mon Sep 17 00:00:00 2001 From: nsheff Date: Thu, 1 Feb 2024 11:39:05 -0500 Subject: [PATCH 03/14] readme --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 7dfdf41..6ba16a4 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,10 @@ with write_lock(ym) as locked_ym: locked_ym.write() ``` +3. Update any constructors to use the `from_{x}` functions + +You can no longer just create a `YAMLConfigManager` object directly; now you need to use the constructor helpers. + ## From v0.9.3 (using future) to v1.X.X: From 2e889ae6a4bb9a7a406f2208a4364d3d8cdc4d69 Mon Sep 17 00:00:00 2001 From: nsheff Date: Thu, 1 Feb 2024 17:00:16 +0000 Subject: [PATCH 04/14] readme --- README.md | 113 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 111 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 6ba16a4..d13c392 100644 --- a/README.md +++ b/README.md @@ -12,8 +12,19 @@ Please see [this](docs/usage.md) Python notebook for features and usage instruct ## Upgrading guide How to upgrade to yacman v1.0.0. +Yacman v1 provides 2 feature upgrades: -There are some transition objects in the 0.9.3 to help with transition. +1. Constructors take the form of `yacman.YAMLConfigManager.from_x(...)` functions, to make it clearer how to +create a new `ym` object. +2. It separates locks into read locks and write locks, to allow mutliple simultaneous readers. + +The v0.9.3 transition release would has versions, really: +- attmap-based version (YacAttMap) +- non-attmap-but-mostly-compatible (YAMLConfigManager) +- new future object (FutureYAMLConfigManager...), which is not-backwards-compatible. + +In v1.0.0, FutureYAMLConfigManager will be renamed to YAMLConfigManager and the old stuff will be removed. +Here's how to transition your code: ### Use the FutureYAMLConfigManager in 0.9.3 @@ -44,7 +55,6 @@ Change ``` with ym as locked_ym: locked_ym.write() - ``` to @@ -55,10 +65,71 @@ with write_lock(ym) as locked_ym: locked_ym.write() ``` + + +More examples: + +``` + +from yacman import FutureYAMLConfigManager as YAMLConfigManager + + +data = {"my_list": [1,2,3], "my_int": 8, "my_str": "hello world!", "my_dict": {"nested_val": 15}} + +ym = YAMLConfigManager(data) + +ym["my_list"] +ym["my_int"] +ym["my_dict"] + +# Use in a context manager to write to the file + +ym["new_var"] = 15 + +with write(ym) as locked_ym: + locked_ym.rebase() + locked_ym.write() + +with read(ym) as locked_ym: + locked_ym.rebase() + +``` + + + + 3. Update any constructors to use the `from_{x}` functions You can no longer just create a `YAMLConfigManager` object directly; now you need to use the constructor helpers. +Examples: + +``` +from yacman import FutureYAMLConfigManager as YAMLConfigManager + +data = {"my_list": [1,2,3], "my_int": 8, "my_str": "hello world!", "my_dict": {"nested_val": 15}} +file_path = "tests/data/full.yaml" +yaml_data = "myvar: myval" + +yacman.YAMLConfigManager.from_yaml_file(file_path) +yacman.YAMLConfigManager.from_yaml_data(yaml_data) +yacman.YAMLConfigManager.from_obj(data) + +``` + +In the past, you could load from a file and overwrite some attributes with a dict of variables, all from the constructor. +Now it would is more explicit: + +``` +ym = yacman.YacMan.from_yaml_file(file_path) +ym.update_from_obj(data) +``` + +To exppand environment variables in values, use `.exp`. + +``` +ym.exp["text_expand_home_dir"] +``` ## From v0.9.3 (using future) to v1.X.X: @@ -67,3 +138,41 @@ Switch back to: ``` from yacman import YAMLConfigManager ``` + + + + + +## Demos + +Some interactive demos + +``` +from yacman import FutureYAMLConfigManager as YAMLConfigManager +ym = yacman.YAMLConfigManager(entries=["a", "b", "c"]) +ym.to_dict() +ym + +print(ym.to_yaml()) + +ym = YAMLConfigManager(entries={"top": {"bottom": ["a", "b"], "bottom2": "a"}, "b": "c"}) +ym +print(ym.to_yaml()) + +ym = YAMLConfigManager(filepath="tests/data/conf_schema.yaml") +print(ym.to_yaml()) +ym + +ym = YAMLConfigManager(filepath="tests/data/empty.yaml") +print(ym.to_yaml()) + +ym = YAMLConfigManager(filepath="tests/data/list.yaml") +print(ym.to_yaml()) + +ym = YAMLConfigManager(YAMLConfigManager(filepath="tests/data/full.yaml").exp) +print(ym.to_yaml()) + +ym = YAMLConfigManager(filepath="tests/data/full.yaml") +print(ym.to_yaml(expand=True)) + +``` \ No newline at end of file From 63a91c8ed6a39f18479ce43cb78b36a2b524a93d Mon Sep 17 00:00:00 2001 From: nsheff Date: Thu, 1 Feb 2024 14:18:10 -0500 Subject: [PATCH 05/14] lint --- yacman/yacman_future.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/yacman/yacman_future.py b/yacman/yacman_future.py index 1d591e8..84fd46f 100644 --- a/yacman/yacman_future.py +++ b/yacman/yacman_future.py @@ -191,10 +191,14 @@ def __repr__(self): return self.to_yaml(self.data) def __enter__(self): - raise NotImplementedError("Use the 'read_lock' and 'write_lock' context managers.") + raise NotImplementedError( + "Use the 'read_lock' and 'write_lock' context managers." + ) def __exit__(self): - raise NotImplementedError("Use the 'read_lock' and 'write_lock' context managers.") + raise NotImplementedError( + "Use the 'read_lock' and 'write_lock' context managers." + ) @ensure_locked(READ) def rebase(self, filepath=None): From 1aa4eefd9f0a29b4b7cd95bd115c763f8282f8ce Mon Sep 17 00:00:00 2001 From: nsheff Date: Thu, 1 Feb 2024 14:19:15 -0500 Subject: [PATCH 06/14] lint with latest version --- tests/conftest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/conftest.py b/tests/conftest.py index d710a94..3f7b2c5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,5 @@ """ Test suite shared objects and setup """ + import os from glob import glob From 36dca03adc97a6ef4cd8ec76a593093ce5877701 Mon Sep 17 00:00:00 2001 From: nsheff Date: Thu, 1 Feb 2024 16:11:41 -0500 Subject: [PATCH 07/14] update locking tests, fix rebase bug, version bump to 0.9.3 --- .gitignore | 1 + locking_tests/locking_tests.py | 33 ++++++++++++++++++++++----- locking_tests/locking_tests_attmap.py | 21 +++++++++++++++++ yacman/_version.py | 2 +- yacman/yacman_future.py | 6 ++++- 5 files changed, 55 insertions(+), 8 deletions(-) create mode 100755 locking_tests/locking_tests_attmap.py diff --git a/.gitignore b/.gitignore index 53aca34..99b3c7a 100644 --- a/.gitignore +++ b/.gitignore @@ -74,3 +74,4 @@ dist/ *.egg-info/ *ipynb_checkpoints* +locking_tests/test.yaml \ No newline at end of file diff --git a/locking_tests/locking_tests.py b/locking_tests/locking_tests.py index 0d825ae..cb33f08 100755 --- a/locking_tests/locking_tests.py +++ b/locking_tests/locking_tests.py @@ -1,11 +1,24 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python3.11 import sys +import os + from argparse import ArgumentParser from random import random from time import sleep from yacman import YacAttMap +from yacman import FutureYAMLConfigManager as YAMLConfigManager +from yacman import write_lock + +import logging +_LOGGER = logging.getLogger() # root logger +stream = logging.StreamHandler(sys.stdout) +fmt = logging.Formatter("%(levelname)s %(asctime)s | %(name)s:%(module)s:%(lineno)d > %(message)s ") +stream.setFormatter(fmt) +_LOGGER.setLevel(os.environ.get("LOGLEVEL", "DEBUG")) +_LOGGER.addHandler(stream) + parser = ArgumentParser(description="Test script") @@ -13,9 +26,17 @@ parser.add_argument("-i", "--id", help="process id", required=True) parser.add_argument("-w", "--wait", help="max wait time", type=int, required=True) args = parser.parse_args() -yam = YacAttMap(filepath=args.path, wait_max=args.wait) -with yam as y: - sleep(random()) - y.update({args.id: 1}) +ym = YAMLConfigManager.from_yaml_file(args.path, wait_max=args.wait) + +with write_lock(ym) as locked_y: + locked_y.rebase() + random_wait_time = random() + _LOGGER.debug(f"Sleeping for {random_wait_time} to simulate process {args.id} updating the file") + sleep(random_wait_time) + locked_y.update({args.id: 1}) + _LOGGER.debug(f"Writing to file for process {args.id}.") + locked_y.write() + -sys.exit(0) +raise SystemExit +# sys.exit(0) diff --git a/locking_tests/locking_tests_attmap.py b/locking_tests/locking_tests_attmap.py new file mode 100755 index 0000000..0d825ae --- /dev/null +++ b/locking_tests/locking_tests_attmap.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python3 + +import sys +from argparse import ArgumentParser +from random import random +from time import sleep + +from yacman import YacAttMap + +parser = ArgumentParser(description="Test script") + +parser.add_argument("-p", "--path", help="path to the test file", required=True) +parser.add_argument("-i", "--id", help="process id", required=True) +parser.add_argument("-w", "--wait", help="max wait time", type=int, required=True) +args = parser.parse_args() +yam = YacAttMap(filepath=args.path, wait_max=args.wait) +with yam as y: + sleep(random()) + y.update({args.id: 1}) + +sys.exit(0) diff --git a/yacman/_version.py b/yacman/_version.py index a2fecb4..c598173 100644 --- a/yacman/_version.py +++ b/yacman/_version.py @@ -1 +1 @@ -__version__ = "0.9.2" +__version__ = "0.9.3" diff --git a/yacman/yacman_future.py b/yacman/yacman_future.py index 84fd46f..7f33efa 100644 --- a/yacman/yacman_future.py +++ b/yacman/yacman_future.py @@ -211,7 +211,11 @@ def rebase(self, filepath=None): if fp is not None: local_data = self.data self.data = load_yaml(fp) - deep_update(self.data, local_data) + _LOGGER.debug(f"Rebased {local_data} with {self.data} from {fp}") + if self.data is None: + self.data = local_data + else: + deep_update(self.data, local_data) else: _LOGGER.warning("Rebase has no effect if no filepath given") From 708db66737c02e3c1e2848c4399b1ed6f6491ecf Mon Sep 17 00:00:00 2001 From: nsheff Date: Thu, 1 Feb 2024 16:16:12 -0500 Subject: [PATCH 08/14] lint --- locking_tests/locking_tests.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/locking_tests/locking_tests.py b/locking_tests/locking_tests.py index cb33f08..c4f2953 100755 --- a/locking_tests/locking_tests.py +++ b/locking_tests/locking_tests.py @@ -12,9 +12,12 @@ from yacman import write_lock import logging + _LOGGER = logging.getLogger() # root logger stream = logging.StreamHandler(sys.stdout) -fmt = logging.Formatter("%(levelname)s %(asctime)s | %(name)s:%(module)s:%(lineno)d > %(message)s ") +fmt = logging.Formatter( + "%(levelname)s %(asctime)s | %(name)s:%(module)s:%(lineno)d > %(message)s " +) stream.setFormatter(fmt) _LOGGER.setLevel(os.environ.get("LOGLEVEL", "DEBUG")) _LOGGER.addHandler(stream) @@ -31,7 +34,9 @@ with write_lock(ym) as locked_y: locked_y.rebase() random_wait_time = random() - _LOGGER.debug(f"Sleeping for {random_wait_time} to simulate process {args.id} updating the file") + _LOGGER.debug( + f"Sleeping for {random_wait_time} to simulate process {args.id} updating the file" + ) sleep(random_wait_time) locked_y.update({args.id: 1}) _LOGGER.debug(f"Writing to file for process {args.id}.") From f2b048ddae3cf0d12b7b9c01e8a13e427dccf8ad Mon Sep 17 00:00:00 2001 From: nsheff Date: Thu, 1 Feb 2024 16:16:44 -0500 Subject: [PATCH 09/14] test python 3.12 --- .github/workflows/run-pytest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-pytest.yml b/.github/workflows/run-pytest.yml index e9f43a2..7f3f453 100644 --- a/.github/workflows/run-pytest.yml +++ b/.github/workflows/run-pytest.yml @@ -11,7 +11,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - python-version: ["3.8", "3.10"] + python-version: ["3.8", "3.12"] os: [ubuntu-latest] steps: From e024ebbcc1644b434fbb3dc672b8a4f52cc323e1 Mon Sep 17 00:00:00 2001 From: nsheff Date: Thu, 1 Feb 2024 16:17:47 -0500 Subject: [PATCH 10/14] update versions, pytest action --- .github/workflows/run-pytest.yml | 6 +++--- setup.py | 5 +++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/run-pytest.yml b/.github/workflows/run-pytest.yml index 7f3f453..65183a5 100644 --- a/.github/workflows/run-pytest.yml +++ b/.github/workflows/run-pytest.yml @@ -20,12 +20,12 @@ jobs: - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: - python-version: ${{ matrix.python-version }} + python-version: "3.x" - - name: Install dev dependancies + - name: Install dev dependencies run: if [ -f requirements/requirements-dev.txt ]; then pip install -r requirements/requirements-dev.txt; fi - - name: Install test dependancies + - name: Install test dependencies run: if [ -f requirements/requirements-test.txt ]; then pip install -r requirements/requirements-test.txt; fi - name: Install yacman diff --git a/setup.py b/setup.py index 71e79a3..f73e24f 100644 --- a/setup.py +++ b/setup.py @@ -39,10 +39,11 @@ classifiers=[ "Development Status :: 4 - Beta", "License :: OSI Approved :: BSD License", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Topic :: Scientific/Engineering :: Bio-Informatics", ], license="BSD2", From 9bff34b6f41e570fb38e8d577b16534772a45da3 Mon Sep 17 00:00:00 2001 From: nsheff Date: Thu, 1 Feb 2024 16:18:47 -0500 Subject: [PATCH 11/14] simplify locking tests --- .github/workflows/run-locking-tests.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/run-locking-tests.yaml b/.github/workflows/run-locking-tests.yaml index 7560fd1..f8835d6 100644 --- a/.github/workflows/run-locking-tests.yaml +++ b/.github/workflows/run-locking-tests.yaml @@ -12,7 +12,7 @@ jobs: strategy: matrix: python-version: ["3.9"] - proc: [10, 50, 100] + proc: [10, 100] wait: [10, 40] steps: - uses: actions/checkout@v3 @@ -20,9 +20,9 @@ jobs: - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: - python-version: ${{ matrix.python-version }} + python-version: "3.x" - - name: Install dev dependancies + - name: Install dev dependencies run: if [ -f requirements/requirements-dev.txt ]; then pip install -r requirements/requirements-dev.txt; fi - name: Install yacman From 953637c4b6072a621922c0373bd2a77c1086ad2b Mon Sep 17 00:00:00 2001 From: nsheff Date: Thu, 1 Feb 2024 16:19:50 -0500 Subject: [PATCH 12/14] fix locking test python pointer --- locking_tests/locking_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/locking_tests/locking_tests.py b/locking_tests/locking_tests.py index c4f2953..9715aa8 100755 --- a/locking_tests/locking_tests.py +++ b/locking_tests/locking_tests.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3.11 +#!/usr/bin/env python3 import sys import os From 48d0ebd46cf76cc6fdad1322ab22d529e3390029 Mon Sep 17 00:00:00 2001 From: nsheff Date: Thu, 1 Feb 2024 16:30:31 -0500 Subject: [PATCH 13/14] ubiquerg req, changelog --- docs/changelog.md | 7 +++++++ requirements/requirements-all.txt | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/changelog.md b/docs/changelog.md index 95e95b9..1ea36f6 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -2,6 +2,13 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) format. +## [0.9.3] -- 2024-02-01 + +### Added +- New `FutureYAMLConfigManager` object, prep for v1. +- Improved file locking system with `read_lock` and `write_lock` context managers +- New `from_x` object construction API. + ## [0.9.2] -- 2023-10-05 ## Added diff --git a/requirements/requirements-all.txt b/requirements/requirements-all.txt index 6502c46..e66e224 100644 --- a/requirements/requirements-all.txt +++ b/requirements/requirements-all.txt @@ -2,4 +2,4 @@ attmap>=0.13.0 jsonschema>=3.2.0 oyaml pyyaml>=3.13 -ubiquerg>=0.6.1 +ubiquerg>=0.7.0 From 9168d16670400c3f7c57fd57a4e8adc6c4fabfa6 Mon Sep 17 00:00:00 2001 From: nsheff Date: Thu, 1 Feb 2024 16:31:39 -0500 Subject: [PATCH 14/14] update python publish --- .github/workflows/python-publish.yml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 0c36fed..9c031fb 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -11,21 +11,20 @@ jobs: deploy: runs-on: ubuntu-latest - + permissions: + id-token: write # IMPORTANT: this permission is mandatory for trusted publishing steps: - uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: '3.x' - name: Install dependencies run: | python -m pip install --upgrade pip pip install setuptools wheel twine - - name: Build and publish - env: - TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + - name: Build package run: | python setup.py sdist bdist_wheel - twine upload dist/* + - name: Publish package distributions to PyPI + uses: pypa/gh-action-pypi-publish@release/v1