From ad325adb81826f8a5de2a45d3eeda1b2a4045e2b Mon Sep 17 00:00:00 2001 From: Josh Orr Date: Sat, 11 Feb 2023 16:21:29 -0700 Subject: [PATCH] feat: renamed/reconfigured dependcies, got unit tests working. --- LICENSE | 24 ++++ Readme.md | 174 ++++++++++++------------ poetry.lock | 120 +++++++++------- pyproject.toml | 31 +++-- setup.cfg | 8 -- tests/__init__.py | 0 tests/conftest.py | 27 ---- tests/normal_tests/__init__.py | 0 tests/normal_tests/conftest.py | 88 ++++++++++++ tests/{ => normal_tests}/test_config.py | 157 ++++++--------------- tests/test_errors.py | 76 +++++++++++ xcon/__init__.py | 14 +- xcon/config.py | 156 ++++++++++----------- xcon/directory.py | 16 +-- xcon/provider.py | 56 ++++---- xcon/providers/common.py | 10 +- xcon/providers/default.py | 2 +- xcon/providers/dynamo.py | 129 +++++++----------- xcon/providers/environmental.py | 10 +- xcon/providers/secrets_manager.py | 10 +- xcon/providers/ssm_param_store.py | 6 +- xcon/pytest_plugin.py | 11 +- xcon/types.py | 26 ++++ 23 files changed, 617 insertions(+), 534 deletions(-) create mode 100644 LICENSE create mode 100644 tests/__init__.py delete mode 100644 tests/conftest.py create mode 100644 tests/normal_tests/__init__.py create mode 100644 tests/normal_tests/conftest.py rename tests/{ => normal_tests}/test_config.py (84%) create mode 100644 tests/test_errors.py create mode 100644 xcon/types.py diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..fdddb29 --- /dev/null +++ b/LICENSE @@ -0,0 +1,24 @@ +This is free and unencumbered software released into the public domain. + +Anyone is free to copy, modify, publish, use, compile, sell, or +distribute this software, either in source code form or as a compiled +binary, for any purpose, commercial or non-commercial, and by any +means. + +In jurisdictions that recognize copyright laws, the author or authors +of this software dedicate any and all copyright interest in the +software to the public domain. We make this dedication for the benefit +of the public at large and to the detriment of our heirs and +successors. We intend this dedication to be an overt act of +relinquishment in perpetuity of all present and future rights to this +software under copyright law. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. + +For more information, please refer to diff --git a/Readme.md b/Readme.md index a5611d4..683509b 100644 --- a/Readme.md +++ b/Readme.md @@ -123,7 +123,7 @@ Now, you simply use it. From the get-go, environmental variables will 'just wor Here are a few basic examples: ```python -from xyn_config import config +from xcon import config # If you had an environmental variable called `SOME_CONFIG_VARIABLE`, this would find it: my_config_value = config.get('SOME_CONFIG_VARIABLE') @@ -141,12 +141,12 @@ By default, unit tests will always start with a Config object that has caching d and only uses the environmental provider (ie: only looks at environmental variables). This is accomplished via an autouse fixture in a pytest plugin module -(see plugin module `xyn_config.pytest_plugin` or fixture `xyn_config.pytest_plugin.xyn_config`). +(see plugin module `xcon.pytest_plugin` or fixture `xcon.pytest_plugin.xcon`). If a project has `xyn-config` as a dependency, pytest will find this plugin module and automatically use it. Nothing more to do. -As an FYI/side-note: There is a `xyn_resource.pytest_plugin.xyn_context` that will also +As an FYI/side-note: There is a `xinject.pytest_plugin.xyn_context` that will also automatically configure a blank context for each unit test. This does mean you must configure Config using a fixture or at the top of your unit test method, @@ -286,17 +286,17 @@ followed by a link to more details. - Grabbing Values - Simply ask a config object for a value via upper-case attribute: - `config.CLIENT_ID` - - Or use `get` method `xyn_config.config.Config.get`. + - Or use `get` method `xcon.config.Config.get`. - get method lets you pass in a default value, just like `dict.get`. - Current Config / Resources - There is a concept that there is a 'current/active' default config object that can be used at any time. - - This is accomplished via xyn-resource library (see `xyn-resource`). - - You can get the current config object via `Config.resource()`. + - This is accomplished via xinject library (see `xinject`). + - You can get the current config object via `Config.grab()`. - There is a convenient proxy config object you can use that represents the current Config object. - The proxy can be used as if it's the current config object. - Below you see us importing the config proxy and then using it to get a value: - - `from xyn_config import config` + - `from xcon import config` - `config.get('some_config_var_name')` - For details see [Current Config](#current-config) - Parents @@ -313,7 +313,7 @@ followed by a link to more details. - It's usually more convenient to just use the current/default config. - For details see [Parent Chain](#parent-chain). - Overrides - - Some overrides can happen as part of `xyn_config.config.Config.__init__`. + - Some overrides can happen as part of `xcon.config.Config.__init__`. - Such as `service`, `environment`, `providers`, etc. - You can also change them after Config object is created via attributes. - For normal configuration values, you can override thoese as well. @@ -335,7 +335,7 @@ followed by a link to more details. ## Service/Environment Names {#service-environment-names} -There are two special variables that `xyn_config.config.Config` treats special: +There are two special variables that `xcon.config.Config` treats special: 1. `Config.SERVICE_NAME` - Normally comes from an environmental variable `SERVICE_NAME`. @@ -367,7 +367,7 @@ Config will only look in these locations for the special variables 1. First, [Overrides] (including and any overrides in the [Parent Chain][parent-chain]). 2. Environmental variables next (directly via `os.getenv`, NOT the provider). - **This is how most projects normally do it.** - - Even if the `xyn_config.providers.environmental.EnvironmentalProvider` is **NOT** in the + - Even if the `xcon.providers.environmental.EnvironmentalProvider` is **NOT** in the [Provider Chain][provider-chain] we will still look for `SERVICE_NAME`/`APP_ENV` in the environmental variables (all other config values would not). 3. [Defaults] last (including any defaults in the [Parent Chain][parent-chain]). @@ -380,7 +380,7 @@ Let's start with a very simple example: ```python # Import the default config object, which is an 'alias' to the # currently active config object. -from xyn_config import config +from xcon import config # Get a value from the currently active config object, this special # config object will always lookup the currently active config object @@ -388,13 +388,13 @@ from xyn_config import config value = config.SOME_CONFIG_VALUE ``` -This will look up the current `xyn_config.config.Config` class and ask it for the +This will look up the current `xcon.config.Config` class and ask it for the `SOME_CONFIG_VALUE` value. It will either give you the value or a None if it does not exist. The general idea is: The underlying 'app/service' setup will provide the properly setup -ready-to-use `xyn_config.config.Config` as a resource (`xyn_resource.resource.Resource`). -So you can just import this special `xyn_config.config.config` variable to easily always -use current `xyn_config.config.Config.current` resource. +ready-to-use `xcon.config.Config` as a resource (`xinject.dependency.Dependency`). +So you can just import this special `xcon.config.config` variable to easily always +use current `xcon.config.Config.current` resource. ## Naming Guidelines @@ -415,7 +415,7 @@ use current `xyn_config.config.Config.current` resource. them in code and generally lower-case them in the various aws providers. - Directory paths are case **sensitive** (see [Directory Paths][standard-directory-paths]); like this: `/myCoolService/joshOrrEnv/...`. - - The service name and env name that make up the `xyn_config.directory.Directory.path` + - The service name and env name that make up the `xcon.directory.Directory.path` is case-sensitive. But the part after that for the config name is **NOT**. ## Standard Lookup Order @@ -425,19 +425,19 @@ By Default, Config will look at the following locations by default 1. Config [Overrides](#overrides) 2. Environmental Variables - - via `xyn_config.providers.environmental.EnvironmentalProvider`. + - via `xcon.providers.environmental.EnvironmentalProvider`. 3. Dynamo flat config cache, Details: - [Info About Caching][caching] - - via `xyn_config.providers.dynamo.DynamoCacher` -4. AWS Secrets Provider via `xyn_config.providers.secrets_manager.SecretsManagerProvider`. -5. AWS SSM Param Store via `xyn_config.providers.ssm_param_store.SsmParamStoreProvider`. + - via `xcon.providers.dynamo.DynamoCacher` +4. AWS Secrets Provider via `xcon.providers.secrets_manager.SecretsManagerProvider`. +5. AWS SSM Param Store via `xcon.providers.ssm_param_store.SsmParamStoreProvider`. 6. Config [Defaults](#defaults) ## Standard Directory Paths [standard-directory-paths]: #standard-directory-paths Most of the providers have a 'path' you can use with them. I call the path up until just -before the config variable name a directory (see `xyn_config.directory.Directory`). +before the config variable name a directory (see `xcon.directory.Directory`). If no `Config.SERVICE_NAME` has been provided or is set to `None` (either from a lack of an environmental variable `SERVICE_NAME`, or via @@ -470,14 +470,14 @@ The `Config` class is more dynamic... you can think of it as more of a 'view' or So this "view" and/or "lens" can now be easily changed. You can do an override like this: ```python -from xyn_config import config +from xcon import config config.SERVICE_NAME = "someServiceName" ``` Or set a default (if it can't find the value anywhere else): ```python -from xyn_config import config +from xcon import config config.set_default("service_name", "someServiceName") ``` @@ -520,8 +520,8 @@ still be very fast, as the resources it uses behind the scenes stay allocated an have the value for `SOME_NAME` if it's been asked for previously. ```python -from xyn_config import Config -from xyn_types import Default +from xcon import Config +from xsentinels import Default def my_function_is_called_a_lot(): my_config = Config(directories=[f"/some/dir_path", Default]) the_value_I_want = my_config.SOME_NAME @@ -529,25 +529,25 @@ def my_function_is_called_a_lot(): ## Current Config -The Config class is a xyn-resource, `xyn_resource.resource.Resource`; +The Config class is a xinject, `xinject.dependency.Dependency`; meaning that there is a concept that there is a 'current' or 'default' Config object that can always be used. -You can get it your self easily anywhere asking `Config` for it's `.resource()`. +You can get it your self easily anywhere asking `Config` for it's `.grab()`. ```python # Import Config class -from xyn_config import Config +from xcon import Config # Ask Config class for the current one. -config = Config.resource() +config = Config.grab() ``` Most of the time, it's more convenient to use a special ActiveResourceProxy object that you can import and use directly. You can use it as if it's the current config object: ```python -from xyn_config import config +from xcon import config # Use it as if it's the current/default config object, # it will proxy what you ask it to the real object @@ -557,8 +557,8 @@ config.get('SOME_CONFIG_NAME') ## Basics -We have a list of `xyn_config.provider.Provider` that we query, in a priority order. -We also have a list of `xyn_config.directory.Directory` in priority order as well +We have a list of `xcon.provider.Provider` that we query, in a priority order. +We also have a list of `xcon.directory.Directory` in priority order as well (see [Provider Chain][provider-chain] and [Directory Chain][directory-chain]). For each directory, we ask each provider for @@ -574,7 +574,7 @@ and be fast to retrieve. You can use `Config` as if the config-var is directly on the object: ```python -from xyn_config import Config +from xcon import Config value = Config().SOME_VAR ``` @@ -584,14 +584,14 @@ it's used it will lookup the current config object and direct the retrieval to i Here is an example: ```python -from xyn_config import config +from xcon import config value = config.SOME_VAR ``` This is equivalent of doing `Config.current().SOME_VAR`. You can call any method you want on config that Config supports as well: ```python -from xyn_config import config +from xcon import config value = config.get("SOME_VAR", "some default value") ``` @@ -602,10 +602,10 @@ Here is the order we check things in when retrieving a value: 1. [Overrides](#overrides) - Value is set directly on `Config` or one of Config's parent(s). - For more details about parents, see [Parent Chain][parent-chain]. -2. `xyn_config.providers.environmental.EnvironmentalProvider` first if that provider is +2. `xcon.providers.environmental.EnvironmentalProvider` first if that provider is configured to be used. We don't cache things from the envirometnal provider, so it's always consutled before the cache. See topic [Provider Chain][provider-chain] or the - `xyn_config.provider.ProviderChain` class for more details. + `xcon.provider.ProviderChain` class for more details. 3. High-Level flattened cache if it was not disabled (see [Caching][caching]). 4. All other [Providers][provider-chain] / [Directories](#directory-chain) - Looked up based first on [Directory Order][directory-chain] @@ -619,7 +619,7 @@ Here is the order we check things in when retrieving a value: Basic, average/normal example: ```python -from xyn_config import config +from xcon import config assert config.APP_ENV == "testing" @@ -648,7 +648,7 @@ assert config.SOME_NAME == "Dynamo-V-2" Here is an example of setting and using an override: ```python -from xyn_config import config +from xcon import config # if we have values: config.SOME_NAME = "some parent value" @@ -676,7 +676,7 @@ Example of using defaults. I am using a more complex example here, to illustrate how parents and defaults work: ```python -from xyn_config import Config, config +from xcon import Config, config # If we have these defaults in the 'parent' config: config.set_default(f"SOME_OTHER_NAME", "parent-default-value") @@ -710,8 +710,8 @@ Here is an example of modifying the current config to add a directory in it's cu [Directory Chain][directory-chain]. ```python -from xyn_config import Config -from xyn_config.directory import Directory +from xcon import Config +from xcon.directory import Directory # Even if this function is called a lot, what we do with # config should still be fast enough. @@ -738,7 +738,7 @@ the current config object may be the root-config object, and therefore have no p The parent chain is generally consulted when: - We are getting the list of providers, directories, getting the cacher, and so on; and we encounter - a `xyn_types.default.Default` value while doing this.We then consult the next parent in the + a `xsentinels.default.Default` value while doing this.We then consult the next parent in the To Resole this `Default` value, Config consults the current parent-chain. If when reaching the last parent in the chain, we still have a `Default` value, sensible/default values are constructed and used. @@ -750,10 +750,10 @@ The parent chain is generally consulted when: If the Config object has their `use_parent == True` (it defaults to True) then it will allow the parent-chain to grow past it's self in the past/previously activated Config objects. -Config is a xyn-resource Resource. Resource uses a `xyn_resource.context.Context` object to +Config is a xinject Dependency. Dependency uses a `xinject.context.XContext` object to keep track of current and past resources. -The parent-chain starts with the current config resource (the one in the current Context). +The parent-chain starts with the current config resource (the one in the current XContext). If that context has a parent context, we next grab the Config resource from that parent context and check it's `Config.use_parent`. If `True` we keep doing this until we reach a Config object without a parent or a `Config.use_parent` @@ -762,13 +762,13 @@ that is False. If the `Config.use_parent` is `False` on the Config object that is currently being asked for a config value: -- If it does not find it's self in the parent-chain (via Context) then the parent-chain +- If it does not find it's self in the parent-chain (via XContext) then the parent-chain will be empty at that moment. This means it will only consult its self and no other Config object. The idea here is the Config object is not a resource in the - `xyn_resource.context.Context.parent_chain` + `xinject.context.XContext.parent_chain` and so is by its self (ie: alone) and should be isolated in this case. - If it finds its self, it will allow the parent-chain to grow to the point it finds its - self in the Context parent-chain. The purpose of this behavior is to allow all the 'child' + self in the XContext parent-chain. The purpose of this behavior is to allow all the 'child' config objects to be in the parent-chain. If one of these children has the use_parent=False, it will stop at that point and **NOT** have any more child config objects included in the parent-chain. @@ -780,55 +780,55 @@ config value: We take out of the chain any config object that is myself. The only objects in the chain are other Config object instances. -Each config object is consulted until we get an answer that is not a `xyn_types.Default`; +Each config object is consulted until we get an answer that is not a `xsentinels.Default`; once that is found that is what is used. Example: If we had two Config object, `A` and `B`. And when `B` was originally constructed, -directory was left at it's `xyn_types.Default` value. +directory was left at it's `xsentinels.Default` value. And `A` is the parent of `B` at the time `B` was asked for its directory_chain -(ie: `xyn_config.config.Config.directory_chain`). This would cause `B` to ask `A` for their +(ie: `xcon.config.Config.directory_chain`). This would cause `B` to ask `A` for their directory_chain because `A` is in `B`'s parent-chain. The directory_chain from `A` is what `B` -would use for it's list of `xyn_config.directory.Directory`'s to look through when resolving +would use for it's list of `xcon.directory.Directory`'s to look through when resolving a configuration value (see `Config.get`). Here is an example: ```python -from xyn_config import Config +from xcon import Config # This is the current config A = Config.current() # We make a new Config, and we DON'T make it 'current'. -# This means it's not tied to or inside any Context [like `A` above is]. +# This means it's not tied to or inside any XContext [like `A` above is]. B = Config() assert B.directory_chain == A.directory_chain # Import the special config object that always 'acts' like the current config # which in this case should be `A`. -from xyn_config import config +from xcon import config assert B.directory_chain == config.directory_chain ``` See [Directory Chain][directory-chain] (later) for what a -`xyn_config.directory.DirectoryChain` is. +`xcon.directory.DirectoryChain` is. ## Provider Chain [provider-chain]: #provider-chain -`Config` uses an abstract base class `xyn_config.provider.Provider` to allow for various -configuration providers. You can see these providers under the `xyn_config.providers` module. +`Config` uses an abstract base class `xcon.provider.Provider` to allow for various +configuration providers. You can see these providers under the `xcon.providers` module. Each `Config` class has an ordered list of these providers in the form of a -`xyn_config.provider.ProviderChain`. This chain is queried when looking for a config value. +`xcon.provider.ProviderChain`. This chain is queried when looking for a config value. Once a value is found, it will be cached by default [if not disabled] via a -`xyn_config.providers.dynamo.DynamoCacher`. +`xcon.providers.dynamo.DynamoCacher`. The dynamo cacher will cache values that are -looked up externally, such as by `xyn_config.providers.ssm_param_store.SsmParamStoreProvider`, -for example. If we use a provider such as `xyn_config.providers.environmental.EnvironmentalProvider`, +looked up externally, such as by `xcon.providers.ssm_param_store.SsmParamStoreProvider`, +for example. If we use a provider such as `xcon.providers.environmental.EnvironmentalProvider`, since this found it locally in a process environmental variable it does not cache it. The providers are queried in the order they are defined in the `Config.provider_chain`. @@ -839,11 +839,11 @@ By `Default`, the `Config.provider_chain` is inherited from the [Parent Chain][p ### Supported Providers -- `xyn_config.providers.environmental.EnvironmentalProvider` -- `xyn_config.providers.dynamo.DynamoProvider` -- `xyn_config.providers.dynamo.DynamoCacher` -- `xyn_config.providers.ssm_param_store.SsmParamStoreProvider` -- `xyn_config.providers.secrets_manager.SecretsManagerProvider` +- `xcon.providers.environmental.EnvironmentalProvider` +- `xcon.providers.dynamo.DynamoProvider` +- `xcon.providers.dynamo.DynamoCacher` +- `xcon.providers.ssm_param_store.SsmParamStoreProvider` +- `xcon.providers.secrets_manager.SecretsManagerProvider` .. todo:: Need to document how to setup permissions in a serverless project to provide correct access to the specific providers for ssm/dynamo/etc. @@ -854,8 +854,8 @@ By `Default`, the `Config.provider_chain` is inherited from the [Parent Chain][p Some providers have a path/directory concept, where they have various different sets of config name/values at a specific path. The path is what we call a specific -`xyn_config.directory.Directory`. We can get the list of directories that will be queried -via `Config.directory_chain`. It returns a `xyn_config.directory.DirectoryChain` that has +`xcon.directory.Directory`. We can get the list of directories that will be queried +via `Config.directory_chain`. It returns a `xcon.directory.DirectoryChain` that has a list of directories in a specific order. We search a specific directory on all of our providers before searching the next directory. @@ -887,7 +887,7 @@ There are two types of caching in py-xyn-config: ### Internal Local Memory Cacher -The `xyn_config.provider.InternalLocalProviderCache` is a resource that is centrally used +The `xcon.provider.InternalLocalProviderCache` is a resource that is centrally used by the other providers (including the DynamoCacher provider) to store what values they have retrieved from their service locally, in a sort of local-memory-cache. @@ -909,9 +909,9 @@ You can change the amount of time via two ways: - When an instance of `InternalLocalProviderCache` is created, it will look for the environmental variable named `CONFIG_INTERNAL_CACHE_EXPIRATION_MINUTES`. If it exists and is true-like it's converted into an `int` and then used as the number of minutes before the cache expires. -- Modifying `xyn_config.provider.InternalLocalProviderCache.expire_time_delta`. +- Modifying `xcon.provider.InternalLocalProviderCache.expire_time_delta`. You can easily modify it by getting the current resource instance and changing the attribute. - (via `InternalLocalProviderCache.resource().expire_time_delta`) + (via `InternalLocalProviderCache.grab().expire_time_delta`) - `expire_time_delta` is a `datetime.timedelta` object. You can use whatever time-units you want by allocating a new `timedelta` object. - Example: `timedelta(minutes=5, seconds=10)`, for 5 minutes and 10 seconds. @@ -920,11 +920,11 @@ If environmental variable is not set and nothing changes the `expire_time_delta` it defaults to 15 minutes. You can always reset the entire cache by calling this method on the current resource instance: -`xyn_config.provider.InternalLocalProviderCache.reset_cache`. +`xcon.provider.InternalLocalProviderCache.reset_cache`. #### Local Memory Caching Side Notes -There is also an option on `xyn_config.config.Config.get` that allows you to ignore the local +There is also an option on `xcon.config.Config.get` that allows you to ignore the local memory cache (as a convenience option). Right now it does this by resetting the entire cache for you before lookup. @@ -940,7 +940,7 @@ The cache is meant to provide a fast-way to lookup configuration values, and is in which fast/scaled-executing processes such as Lambda's will probably get their values. The cache has a built-in TTL (time-to-live) after which it will be deleted from the cache. -In addition, the `xyn_config.providers.dynamo.DynamoCacher` generates a random number and +In addition, the `xcon.providers.dynamo.DynamoCacher` generates a random number and subtracts that from the TTL when querying for values. That way it may see things as not in the cache sooner then it normally would without that. The purpose behind this is to not flood SSM or other configuration services with a bunch of requests at the same time when the cache @@ -953,7 +953,7 @@ get the value and put it into the cache if it's a cacheable value [ie: not an en variable]. The cache is a flattened list of all of the configuration values for a specific set of -`xyn_config.providers` and `xyn_config.directory.Directory`'s. +`xcon.providers` and `xcon.directory.Directory`'s. Because the order of the Directories and Providers determine which values we find and ultimately cache... the cache's Dynamo hash key is made up of: @@ -984,8 +984,8 @@ First, let's talk about how to disable caching via environmental variables: - `CONFIG_DISABLE_DEFAULT_CACHER`, if 'true': - Only by default will the cache will be disabled. - - This only happens while resolving the `Default` on `xyn_config.config.Config.cacher`. - - If you set `DynamoCacher` directly on `xyn_config.config.Config.cacher` via code, + - This only happens while resolving the `Default` on `xcon.config.Config.cacher`. + - If you set `DynamoCacher` directly on `xcon.config.Config.cacher` via code, caching will still be used regardless. - Using this option disables the cacher without having to also disable the providers, this means it will still lookup params from SSM and so on, just not use the cached version. @@ -1004,9 +1004,9 @@ you don't want to modify the code it's self to disable caching. You can set an environmental variable called `CONFIG_DISABLE_DEFAULT_CACHER` to `True` if you want to easily disable caching by default. -..important:: The code will use `xyn_config.providers.environmental.EnvironmentalProvider` for this. +..important:: The code will use `xcon.providers.environmental.EnvironmentalProvider` for this. So if you change this environmental variable WHILE in the middle of running the code - `xyn_config.providers.environmental.EnvironmentalProvider` via debugger or other means, + `xcon.providers.environmental.EnvironmentalProvider` via debugger or other means, provider may have already taken its snapshot of the environmental variables and Config won't see the change. @@ -1027,21 +1027,21 @@ The `CONFIG_DISABLE_DEFAULT_CACHER` will only disable it if `cacher=Default` If you want to permanently disable cacher via code, do the following instead: {There is an autouse fixture that will disable the cacher during unit tests, -as an example real-world use-case in xyn-config's pytest_plugin module: `xyn_config.pytest_plugin.xyn_config`} +as an example real-world use-case in xyn-config's pytest_plugin module: `xcon.pytest_plugin.xcon`} ```python -from xyn_config import Config, config +from xcon import Config, config # Globally/Permanently: config.cacher = None # Temporarily via `with`: -from xyn_config import Config +from xcon import Config with Config(cacher=None): pass # Temporarily via decorator: -from xyn_config import Config +from xcon import Config @Config(cacher=None) def some_method(): pass @@ -1076,7 +1076,7 @@ You can override a value on a `Config` object in two ways: 2. Setting it directly as an attribute, ie: 2. Setting it directly as an attribute, ie: ``` -from xyn_config import config +from xcon import config config.SOME_CONFIG_NAME = "some config value" ``` @@ -1099,7 +1099,7 @@ the value via the providers/cacher like normal (see [Fundamentals][fundamentals] If you only want to temporarily override a value, you can do something like this: ``` -from xyn_config import config +from xcon import config # Activate a new Config object instance: with Config(): @@ -1141,7 +1141,7 @@ child config object by simply setting the default on the child. > :warning: todo: Put an example in here about how it goes though each directory/provider when finding a value. -> :warning: todo: Move this into xyn_config, I think this overview would be better suited there +> :warning: todo: Move this into xcon, I think this overview would be better suited there since it talks about the other sub-modules, like providers, cacher, etc. > :warning: todo: Document/implement new cache key scheme where the RANGE key has the diff --git a/poetry.lock b/poetry.lock index 6c40460..d00ba77 100644 --- a/poetry.lock +++ b/poetry.lock @@ -4,7 +4,7 @@ name = "appnope" version = "0.1.3" description = "Disable App Nap on macOS >= 10.9" -category = "dev" +category = "main" optional = false python-versions = "*" files = [ @@ -16,7 +16,7 @@ files = [ name = "asttokens" version = "2.2.1" description = "Annotate AST trees with source code positions" -category = "dev" +category = "main" optional = false python-versions = "*" files = [ @@ -53,7 +53,7 @@ tests-no-zope = ["cloudpickle", "cloudpickle", "hypothesis", "hypothesis", "mypy name = "backcall" version = "0.2.0" description = "Specifications for callback functions passed in to an API" -category = "dev" +category = "main" optional = false python-versions = "*" files = [ @@ -65,7 +65,7 @@ files = [ name = "black" version = "23.1.0" description = "The uncompromising code formatter." -category = "dev" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -135,7 +135,7 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] name = "boto3-stubs" version = "1.26.69" description = "Type annotations for boto3 1.26.69 generated with mypy-boto3-builder 7.12.3" -category = "dev" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -523,7 +523,7 @@ crt = ["awscrt (==0.16.9)"] name = "botocore-stubs" version = "1.29.69" description = "Type annotations and code completion for botocore" -category = "dev" +category = "main" optional = false python-versions = ">=3.7,<4.0" files = [ @@ -550,7 +550,7 @@ files = [ name = "cffi" version = "1.15.1" description = "Foreign Function Interface for Python calling C code." -category = "dev" +category = "main" optional = false python-versions = "*" files = [ @@ -807,7 +807,7 @@ files = [ name = "cryptography" version = "39.0.1" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." -category = "dev" +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -851,7 +851,7 @@ tox = ["tox"] name = "decorator" version = "5.1.1" description = "Decorators for Humans" -category = "dev" +category = "main" optional = false python-versions = ">=3.5" files = [ @@ -878,7 +878,7 @@ test = ["pytest (>=6)"] name = "executing" version = "1.2.0" description = "Get the currently executing AST node of a frame, and other information" -category = "dev" +category = "main" optional = false python-versions = "*" files = [ @@ -911,7 +911,7 @@ dev = ["flake8", "markdown", "twine", "wheel"] name = "gitdb" version = "4.0.10" description = "Git Object Database" -category = "dev" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -926,7 +926,7 @@ smmap = ">=3.0.1,<6" name = "gitpython" version = "3.1.30" description = "GitPython is a python library used to interact with Git repositories" -category = "dev" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -941,7 +941,7 @@ gitdb = ">=4.0.1,<5" name = "griffe" version = "0.25.4" description = "Signatures for entire Python programs. Extract the structure, the frame, the skeleton of your project, to generate API documentation or find breaking changes in your API." -category = "dev" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1003,7 +1003,7 @@ files = [ name = "ipdb" version = "0.13.11" description = "IPython-enabled pdb" -category = "dev" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -1020,7 +1020,7 @@ tomli = {version = "*", markers = "python_version > \"3.6\" and python_version < name = "ipython" version = "8.10.0" description = "IPython: Productive Interactive Computing" -category = "dev" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -1059,7 +1059,7 @@ test-extra = ["curio", "matplotlib (!=3.2.0)", "nbformat", "numpy (>=1.21)", "pa name = "jedi" version = "0.18.2" description = "An autocompletion tool for Python that can be used for text editors." -category = "dev" +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1109,7 +1109,7 @@ files = [ name = "mako" version = "1.2.4" description = "A super-fast templating language that borrows the best ideas from the existing templating languages." -category = "dev" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1207,7 +1207,7 @@ files = [ name = "matplotlib-inline" version = "0.1.6" description = "Inline Matplotlib backend for Jupyter" -category = "dev" +category = "main" optional = false python-versions = ">=3.5" files = [ @@ -1285,7 +1285,7 @@ min-versions = ["babel (==2.9.0)", "click (==7.0)", "colorama (==0.4)", "ghp-imp name = "mkdocs-autorefs" version = "0.4.1" description = "Automatically link across pages in MkDocs." -category = "dev" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1301,7 +1301,7 @@ mkdocs = ">=1.1" name = "mkdocs-git-revision-date-plugin" version = "0.3.2" description = "MkDocs plugin for setting revision date from git per markdown file." -category = "dev" +category = "main" optional = false python-versions = ">=3.4" files = [ @@ -1352,7 +1352,7 @@ files = [ name = "mkdocstrings" version = "0.20.0" description = "Automatic documentation from sources, for MkDocs." -category = "dev" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1378,7 +1378,7 @@ python-legacy = ["mkdocstrings-python-legacy (>=0.2.1)"] name = "mkdocstrings-python" version = "0.8.3" description = "A Python handler for mkdocstrings." -category = "dev" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1394,7 +1394,7 @@ mkdocstrings = ">=0.19" name = "moto" version = "4.1.2" description = "" -category = "dev" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1442,7 +1442,7 @@ xray = ["aws-xray-sdk (>=0.93,!=0.96)", "setuptools"] name = "mypy-boto3-cloudformation" version = "1.26.60" description = "Type annotations for boto3.CloudFormation 1.26.60 service generated with mypy-boto3-builder 7.12.3" -category = "dev" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1457,7 +1457,7 @@ typing-extensions = ">=4.1.0" name = "mypy-boto3-dynamodb" version = "1.26.24" description = "Type annotations for boto3.DynamoDB 1.26.24 service generated with mypy-boto3-builder 7.11.11" -category = "dev" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1472,7 +1472,7 @@ typing-extensions = ">=4.1.0" name = "mypy-boto3-ec2" version = "1.26.69" description = "Type annotations for boto3.EC2 1.26.69 service generated with mypy-boto3-builder 7.12.3" -category = "dev" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1487,7 +1487,7 @@ typing-extensions = ">=4.1.0" name = "mypy-boto3-lambda" version = "1.26.55" description = "Type annotations for boto3.Lambda 1.26.55 service generated with mypy-boto3-builder 7.12.3" -category = "dev" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1502,7 +1502,7 @@ typing-extensions = ">=4.1.0" name = "mypy-boto3-rds" version = "1.26.47" description = "Type annotations for boto3.RDS 1.26.47 service generated with mypy-boto3-builder 7.12.2" -category = "dev" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1517,7 +1517,7 @@ typing-extensions = ">=4.1.0" name = "mypy-boto3-s3" version = "1.26.62" description = "Type annotations for boto3.S3 1.26.62 service generated with mypy-boto3-builder 7.12.3" -category = "dev" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1532,7 +1532,7 @@ typing-extensions = ">=4.1.0" name = "mypy-boto3-sqs" version = "1.26.0.post1" description = "Type annotations for boto3.SQS 1.26.0 service generated with mypy-boto3-builder 7.11.10" -category = "dev" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1571,7 +1571,7 @@ files = [ name = "parso" version = "0.8.3" description = "A Python Parser" -category = "dev" +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1587,7 +1587,7 @@ testing = ["docopt", "pytest (<6.0.0)"] name = "pathspec" version = "0.11.0" description = "Utility library for gitignore style pattern matching of file paths." -category = "dev" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1599,7 +1599,7 @@ files = [ name = "pdoc3" version = "0.10.0" description = "Auto-generate API documentation for Python projects." -category = "dev" +category = "main" optional = false python-versions = ">= 3.6" files = [ @@ -1615,7 +1615,7 @@ markdown = ">=3.0" name = "pexpect" version = "4.8.0" description = "Pexpect allows easy control of interactive console applications." -category = "dev" +category = "main" optional = false python-versions = "*" files = [ @@ -1630,7 +1630,7 @@ ptyprocess = ">=0.5" name = "pickleshare" version = "0.7.5" description = "Tiny 'shelve'-like database with concurrency support" -category = "dev" +category = "main" optional = false python-versions = "*" files = [ @@ -1642,7 +1642,7 @@ files = [ name = "platformdirs" version = "3.0.0" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." -category = "dev" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1674,7 +1674,7 @@ testing = ["pytest", "pytest-benchmark"] name = "prompt-toolkit" version = "3.0.36" description = "Library for building powerful interactive command lines in Python" -category = "dev" +category = "main" optional = false python-versions = ">=3.6.2" files = [ @@ -1689,7 +1689,7 @@ wcwidth = "*" name = "ptyprocess" version = "0.7.0" description = "Run a subprocess in a pseudo terminal" -category = "dev" +category = "main" optional = false python-versions = "*" files = [ @@ -1701,7 +1701,7 @@ files = [ name = "pure-eval" version = "0.2.2" description = "Safely evaluate AST nodes without side effects" -category = "dev" +category = "main" optional = false python-versions = "*" files = [ @@ -1740,7 +1740,7 @@ files = [ name = "pycparser" version = "2.21" description = "C parser in Python" -category = "dev" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -1838,6 +1838,22 @@ pytest = [ {version = ">=6.2.4", markers = "python_version >= \"3.10\""}, ] +[[package]] +name = "pytest-ordering" +version = "0.6" +description = "pytest plugin to run your tests in a specific order" +category = "dev" +optional = false +python-versions = "*" +files = [ + {file = "pytest-ordering-0.6.tar.gz", hash = "sha256:561ad653626bb171da78e682f6d39ac33bb13b3e272d406cd555adb6b006bda6"}, + {file = "pytest_ordering-0.6-py2-none-any.whl", hash = "sha256:27fba3fc265f5d0f8597e7557885662c1bdc1969497cd58aff6ed21c3b617de2"}, + {file = "pytest_ordering-0.6-py3-none-any.whl", hash = "sha256:3f314a178dbeb6777509548727dc69edf22d6d9a2867bf2d310ab85c403380b6"}, +] + +[package.dependencies] +pytest = "*" + [[package]] name = "pytest-pycodestyle" version = "2.3.1" @@ -2054,7 +2070,7 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] name = "responses" version = "0.22.0" description = "A utility library for mocking out the `requests` Python library." -category = "dev" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2105,7 +2121,7 @@ files = [ name = "smmap" version = "5.0.0" description = "A pure Python implementation of a sliding window memory map manager" -category = "dev" +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -2117,7 +2133,7 @@ files = [ name = "stack-data" version = "0.6.2" description = "Extract data from python stack frames and tracebacks for informative displays" -category = "dev" +category = "main" optional = false python-versions = "*" files = [ @@ -2137,7 +2153,7 @@ tests = ["cython", "littleutils", "pygments", "pytest", "typeguard"] name = "toml" version = "0.10.2" description = "Python Library for Tom's Obvious, Minimal Language" -category = "dev" +category = "main" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" files = [ @@ -2161,7 +2177,7 @@ files = [ name = "traitlets" version = "5.9.0" description = "Traitlets Python configuration system" -category = "dev" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2177,7 +2193,7 @@ test = ["argcomplete (>=2.0)", "pre-commit", "pytest", "pytest-mock"] name = "types-awscrt" version = "0.16.4" description = "Type annotations and code completion for awscrt" -category = "dev" +category = "main" optional = false python-versions = ">=3.7,<4.0" files = [ @@ -2189,7 +2205,7 @@ files = [ name = "types-s3transfer" version = "0.6.0.post5" description = "Type annotations and code completion for s3transfer" -category = "dev" +category = "main" optional = false python-versions = ">=3.7,<4.0" files = [ @@ -2204,7 +2220,7 @@ types-awscrt = "*" name = "types-toml" version = "0.10.8.3" description = "Typing stubs for toml" -category = "dev" +category = "main" optional = false python-versions = "*" files = [ @@ -2317,7 +2333,7 @@ watchmedo = ["PyYAML (>=3.10)"] name = "wcwidth" version = "0.2.6" description = "Measures the displayed width of unicode strings in a terminal" -category = "dev" +category = "main" optional = false python-versions = "*" files = [ @@ -2329,7 +2345,7 @@ files = [ name = "werkzeug" version = "2.2.2" description = "The comprehensive WSGI web application library." -category = "dev" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2404,7 +2420,7 @@ files = [ name = "xmltodict" version = "0.13.0" description = "Makes working with XML feel like you are working with JSON" -category = "dev" +category = "main" optional = false python-versions = ">=3.4" files = [ @@ -2465,4 +2481,4 @@ testing = ["flake8 (<5)", "func-timeout", "jaraco.functools", "jaraco.itertools" [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "b6cdfa7b492be819ac59cb7d0a2c965cded9c41981eea9e38c7da401503d30c1" +content-hash = "2e7a7b1b5a15b0fa7bd0504dc18ef53a6a9144cb5a47de068a6968a894464dc9" diff --git a/pyproject.toml b/pyproject.toml index 470558b..6836af6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,14 +6,6 @@ authors = ["Josh Orr "] [tool.poetry.dependencies] python = "^3.8" -#xyn-types = "^1.1.0" -#xyn-utils = "^1.4.0" -#xyn-dateutils = "^1.1.0" -# -#xyn-resource = ">=2.1.2,<4" -#xyn-logging = "^1.1.1" -#xyn-aws = "^1.1.1" -#xyn-settings = "^1.1.0" xbool = "^1.0.0" xloop = "^1.0.1" xsentinels = "^1.2.1" @@ -27,9 +19,6 @@ pytest-pycodestyle = "^2.3.1" mkdocs = "^1.4.2" mkdocs-material = "^9.0.12" mike = "^1.1.2" - - -[tool.poetry.dev-dependencies] ipdb = "^0" pdoc3 = "*" black = {version = "*", allow-prereleases = true} @@ -38,6 +27,26 @@ boto3-stubs = { extras = ["essential"], version = "*" } mkdocstrings = { extras = ["python"], version = "^0" } mkdocs-autorefs = "^0" mkdocs-git-revision-date-plugin = "^0" +ciso8601 = "^2.3.0" + +[tool.poetry.group.dev.dependencies] +pytest-ordering = "^0.6" + +[tool.pytest.ini_options] +minversion = "6.0" + +# By default, reuse db schema +# (speeds up unit test starts after first unit test run) +# If test-db schmea gets messed up, drop `--reuse-db` +# and it will recreate db next time you run unit tests. +addopts = "--verbose --pycodestyle" + +testpaths = ["tests", "xcon"] +python_files = "tests.py test_*.py *_tests.py tests/*" +norecursedirs = "lib/* tests/scripts .serverless .eggs dist/* node_modules" + +[tool.poetry.plugins] +pytest11 = { xcon_pytest_plugin = "xcon.pytest_plugin"} [build-system] requires = ["poetry-core>=1.0.0"] diff --git a/setup.cfg b/setup.cfg index 75c8e8a..fb061ec 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,17 +1,9 @@ # WARNING: This file is managed by our repository manager tool https://github.com/xyngular/repoman # and changes will be overwritten when "repo install" is executed -[aliases] -test = pytest - [pycodestyle] ignore = E121,E123,E126,E203,E226,E242,E704,E731,W391,W503,W504,C0103 max-line-length = 99 statistics = True exclude = setup.py,**/migrations/*,lib/*,.git,__pycache__,node_modules,.venv,.eggs/*,.serverless/** -# Including a pytest.ini file in project will cause this to be ignored -[tool:pytest] -addopts = --verbose --pycodestyle -python_files = tests.py test_*.py *_tests.py tests/* -norecursedirs = lib/* tests/scripts .serverless .eggs dist/* node_modules diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/conftest.py b/tests/conftest.py deleted file mode 100644 index 20fe688..0000000 --- a/tests/conftest.py +++ /dev/null @@ -1,27 +0,0 @@ -import os - -import pytest -from xyn_config import config - -# have a semi-really looking environmental variable to test with. -os.environ["DJANGO_SETTINGS_MODULE"] = "somemodule.some_app.settings.tests" - -service_name_at_import_time = str(config.SERVICE_NAME) -app_env_at_import_time = str(config.APP_ENV) - - -def test_ensure_config_is_at_baseline_at_module_import_time(): - # Ensure that config is configured at conftest import time. - assert service_name_at_import_time == 'testing' - assert app_env_at_import_time == 'unit' - - -@pytest.fixture -@pytest.mark.order(-70) -def directory(xyn_config): - """ Returns the currently configured full test Directory [with the proper service/env set]. - This uses the `config` fixture to an isolated config, and `config` fixture will get - an isolated Context for you as well. - """ - from xyn_config.directory import Directory - return Directory(service=xyn_config.SERVICE_NAME, env=xyn_config.APP_ENV) diff --git a/tests/normal_tests/__init__.py b/tests/normal_tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/normal_tests/conftest.py b/tests/normal_tests/conftest.py new file mode 100644 index 0000000..95f3572 --- /dev/null +++ b/tests/normal_tests/conftest.py @@ -0,0 +1,88 @@ +import os + +import moto +import pytest +from xcon import config +from xboto.resource import dynamodb + +# have a semi-really looking environmental variable to test with. +os.environ["DJANGO_SETTINGS_MODULE"] = "somemodule.some_app.settings.tests" + +service_name_at_import_time = str(config.SERVICE_NAME) +app_env_at_import_time = str(config.APP_ENV) + + +def test_ensure_config_is_at_baseline_at_module_import_time(): + # Ensure that config is configured at conftest import time. + assert service_name_at_import_time == 'testing' + assert app_env_at_import_time == 'unit' + + +@pytest.fixture +def directory(): + """ Returns the currently configured full test Directory [with the proper service/env set]. + This uses the `config` fixture to an isolated config, and `config` fixture will get + an isolated XContext for you as well. + """ + from xcon.directory import Directory + return Directory(service=config.SERVICE_NAME, env=config.APP_ENV) + + +@pytest.fixture(autouse=True) +def start_moto(): + with moto.mock_dynamodb(): + with moto.mock_ssm(): + with moto.mock_secretsmanager(): + yield 'a' + + +@pytest.fixture(autouse=True) +def dynamo_cache_table(start_moto): + return dynamodb.create_table( + TableName='global-configCache', + KeySchema=[ + # Partition Key + {'AttributeName': 'directory', 'KeyType': 'HASH'}, + # Sort Key + {'AttributeName': 'name', 'KeyType': 'RANGE'} + ], + AttributeDefinitions=[ + {'AttributeName': 'directory', 'AttributeType': 'S'}, + {'AttributeName': 'name', 'AttributeType': 'S'} + ], + # todo: + # YOu need to use a newer-boto3 for this to work than what lamda provides. + # HOWEVER, the config table should always exist, so we should not have to really + # worry about it. If the able already exists we won't attempt to create it. + BillingMode='PAY_PER_REQUEST', + Tags=[{'Key': 'DDBTableGroupKey', 'Value': 'xyn_config'}], + SSESpecification={ + "Enabled": True + } + ) + + +@pytest.fixture(autouse=True) +def dynamo_provider_table(start_moto): + return dynamodb.create_table( + TableName='global-config', + KeySchema=[ + # Partition Key + {'AttributeName': 'directory', 'KeyType': 'HASH'}, + # Sort Key + {'AttributeName': 'name', 'KeyType': 'RANGE'} + ], + AttributeDefinitions=[ + {'AttributeName': 'directory', 'AttributeType': 'S'}, + {'AttributeName': 'name', 'AttributeType': 'S'} + ], + # todo: + # YOu need to use a newer-boto3 for this to work than what lamda provides. + # HOWEVER, the config table should always exist, so we should not have to really + # worry about it. If the able already exists we won't attempt to create it. + BillingMode='PAY_PER_REQUEST', + Tags=[{'Key': 'DDBTableGroupKey', 'Value': 'xyn_config'}], + SSESpecification={ + "Enabled": True + } + ) diff --git a/tests/test_config.py b/tests/normal_tests/test_config.py similarity index 84% rename from tests/test_config.py rename to tests/normal_tests/test_config.py index 627c4f2..ea7c4aa 100644 --- a/tests/test_config.py +++ b/tests/normal_tests/test_config.py @@ -4,25 +4,25 @@ import time from typing import Type -from xyn_config.providers.common import AwsProvider +from xcon.providers.common import AwsProvider -from xyn_aws import aws_clients +from xboto import boto_clients import moto import pytest -from xyn_types import Default -from xyn_utils.loop import loop +from xsentinels import Default +from xloop import xloop -from xyn_config import Config, config -from xyn_config.directory import DirectoryItem, Directory -from xyn_config.provider import ProviderCacher, InternalLocalProviderCache -from xyn_config.providers import ( +from xcon import Config, config +from xcon.directory import DirectoryItem, Directory +from xcon.provider import ProviderCacher, InternalLocalProviderCache +from xcon.providers import ( EnvironmentalProvider, DynamoProvider, SsmParamStoreProvider, SecretsManagerProvider, DynamoCacher, default_provider_types, ) -from xyn_config.providers.dynamo import _ConfigDynamoTable +from xcon.providers.dynamo import _ConfigDynamoTable DEFAULT_TESTING_PROVIDERS = [EnvironmentalProvider, SecretsManagerProvider, SsmParamStoreProvider] @@ -102,7 +102,7 @@ def test_env_provider(): assert config.get_value("some_other_non_existant_env_var") is None -@moto.mock_dynamodb2 +@moto.mock_dynamodb @moto.mock_ssm @moto.mock_secretsmanager def test_config_disable_via_env_var(): @@ -127,20 +127,17 @@ def test_config_disable_via_env_var(): assert isinstance(config.resolved_cacher, DynamoCacher) -@moto.mock_dynamodb2 +@moto.mock_dynamodb @moto.mock_ssm @moto.mock_secretsmanager @config_with_env_dyn_ssm_secrets_providers def test_direct_class_access(directory: Directory): config.TEST_NAME = "myTestValue" # When you access a config var via the default config, it should lookup the current - # default config automatically [via current Context] and use that to get the var. + # default config automatically [via current XContext] and use that to get the var. assert config.TEST_NAME == 'myTestValue' -@moto.mock_dynamodb2 -@moto.mock_ssm -@moto.mock_secretsmanager @config_with_env_dyn_ssm_secrets_providers def test_basic_configs(directory: Directory): # Basic defaults-test. @@ -151,9 +148,6 @@ def test_basic_configs(directory: Directory): assert config.TEST_NAME == 'myTestValue' -@moto.mock_dynamodb2 -@moto.mock_ssm -@moto.mock_secretsmanager @config_with_env_dyn_ssm_secrets_providers def test_config_for_unconfiged_param(directory: Directory): # Basic defaults-test. @@ -162,9 +156,6 @@ def test_config_for_unconfiged_param(directory: Directory): assert config.TEST_NAME_OTHER is None -@moto.mock_dynamodb2 -@moto.mock_ssm -@moto.mock_secretsmanager @config_with_env_dyn_ssm_secrets_providers def test_direct_parent_behavior(directory: Directory): config.set_default("A_DEFAULT", 'parent-default') @@ -187,10 +178,14 @@ def test_direct_parent_behavior(directory: Directory): config_providers = [ p for p in config.provider_chain.providers if not isinstance(p, ProviderCacher) ] - assert list(loop(child.provider_chain.providers)) == list(loop(config_providers)) + assert list(xloop(child.provider_chain.providers)) == list(xloop(config_providers)) child = Config(directories=["/hello/another"]) - assert list(loop(child.directory_chain.directories)) == [Directory.from_path("/hello/another")] + + assert list( + xloop(child.directory_chain.directories) + ) == [Directory.from_path("/hello/another")] + assert child.cacher is Default assert child.A_DEFAULT == 'parent-default' child.A_DEFAULT = 'child-override' @@ -202,14 +197,11 @@ def test_direct_parent_behavior(directory: Directory): p if not isinstance(p, ProviderCacher) else child.cacher for p in config.provider_chain.providers ] - child_providers = list(loop(child.provider_chain.providers)) - parent_providers = list(loop(config.provider_chain.providers)) + child_providers = list(xloop(child.provider_chain.providers)) + parent_providers = list(xloop(config.provider_chain.providers)) assert child_providers == parent_providers -@moto.mock_dynamodb2 -@moto.mock_ssm -@moto.mock_secretsmanager @config_with_env_dyn_ssm_secrets_providers def test_env_are_higher_priority_than_cacher(directory: Directory): # Re-enable the cacher by default, so we can test cacher-related features @@ -222,7 +214,7 @@ def test_env_are_higher_priority_than_cacher(directory: Directory): config.set_default('TEST_CACHER_NOT_USED', 'default-value') # Put some test-data in ssm. - client = aws_clients.ssm + client = boto_clients.ssm client.put_parameter( Name=f'{directory.path}/test_cacher_not_used', Value="wrongValue", Type="String" ) @@ -234,7 +226,7 @@ def test_env_are_higher_priority_than_cacher(directory: Directory): with EnvironmentalProvider(env_vars={'TEST_CACHER_NOT_USED': 'rightValue'}): # Ensure the original ssm value made it into the cacher; # we are verifying that the cache got updated correctly with this check. - cacher_obj = DynamoCacher.resource() + cacher_obj = DynamoCacher.grab() assert ( cacher_obj.get_value( name='TEST_CACHER_NOT_USED', @@ -250,13 +242,10 @@ def test_env_are_higher_priority_than_cacher(directory: Directory): assert config.TEST_CACHER_NOT_USED == "rightValue" -@moto.mock_dynamodb2 -@moto.mock_ssm -@moto.mock_secretsmanager @config_with_env_dyn_ssm_secrets_providers def test_basic_config(directory: Directory): # Put some test-data in ssm - client = aws_clients.ssm + client = boto_clients.ssm client.put_parameter(Name=f'{directory.path}/test_name', Value="testValue2", Type="String") # Make sure moto is working.... @@ -268,9 +257,6 @@ def test_basic_config(directory: Directory): assert v2 == 'testValue2' -@moto.mock_dynamodb2 -@moto.mock_ssm -@moto.mock_secretsmanager @config_with_env_dyn_ssm_secrets_providers @pytest.mark.parametrize( "expected_values", @@ -280,14 +266,14 @@ def test_basic_config(directory: Directory): ], ) def test_ssm_and_dynamo(directory: Directory, expected_values): - client = aws_clients.ssm + client = boto_clients.ssm client.put_parameter( Name=f'{directory.path}/test_name', Value="ssmValue", Type="String", ) - table = _ConfigDynamoTable() + table = _ConfigDynamoTable(table_name='global-config') item = DirectoryItem( name=expected_values['cache_item_name'], directory=directory, @@ -350,7 +336,6 @@ def test_basic_confg_features_with_parent_chain(directory: Directory): assert current_config.SOME_OTHER_NAME == "parent-default-value" -@moto.mock_ssm @Config(providers=[SsmParamStoreProvider]) # Not using EnvironmentalProvider, only Ssm...Provider. def test_exported_values(directory: Directory): # This is for making sure the default value does not somehow override other ways @@ -361,7 +346,7 @@ def test_exported_values(directory: Directory): config.add_export(service=another_service) # Put some test-data in ssm. - client = aws_clients.ssm + client = boto_clients.ssm client.put_parameter( Name=f'/{another_service}/export/{directory.env}/some_exported_name', Value="an-exported-value", @@ -375,9 +360,6 @@ def test_exported_values(directory: Directory): # values don't override any of them. -@moto.mock_dynamodb2 -@moto.mock_ssm -@moto.mock_secretsmanager def test_env_and_defaults_do_not_go_into_cache(directory: Directory): # Re-enable the cacher by default, so we can test cacher-related features # (it's set to None by default for unit tests) @@ -397,7 +379,7 @@ def test_env_and_defaults_do_not_go_into_cache(directory: Directory): # Ensure the values did not go into the cache # we are verifying that the cache got updated correctly with this check. - cacher_obj = DynamoCacher.resource() + cacher_obj = DynamoCacher.grab() assert ( cacher_obj.get_value( name='TEST_CACHER_NOT_USED', @@ -410,9 +392,6 @@ def test_env_and_defaults_do_not_go_into_cache(directory: Directory): ) -@moto.mock_dynamodb2 -@moto.mock_ssm -@moto.mock_secretsmanager @config_with_env_dyn_ssm_secrets_providers def test_env_and_defaults_do_not_go_into_cache(directory: Directory): # Re-enable the cacher by default, so we can test cacher-related features @@ -439,7 +418,7 @@ def test_env_and_defaults_do_not_go_into_cache(directory: Directory): # Ensure the values did not go into the cache # we are verifying that the cache got updated correctly with this check. - cacher_obj = DynamoCacher.resource() + cacher_obj = DynamoCacher.grab() cached_item = cacher_obj.get_item( name='TEST_CACHER_NOT_USED', directory=directory, @@ -452,7 +431,7 @@ def test_env_and_defaults_do_not_go_into_cache(directory: Directory): assert cached_item.directory.is_non_existent # Put some test-data in ssm. - client = aws_clients.ssm + client = boto_clients.ssm client.put_parameter( Name=f'{directory.path}/test_cacher_not_used', Value="ssmValue", Type="String" ) @@ -487,59 +466,9 @@ def test_config_item_not_logging_value(directory: Directory): assert 'my-value' in item.__repr__() -@pytest.mark.parametrize( - "provider_type", - [ - SecretsManagerProvider, - DynamoProvider, - SsmParamStoreProvider - ], -) -# Must run it before the others, or boto will 'cache' the config-file; -# If the config file does not exist, it looks it up each time. -# So the other tests should run fine and be able to find a default region. -@pytest.mark.order(1) -def test_providers_is_ok_without_region(provider_type: Type[AwsProvider]): - old_region = None - try: - # Point boto to a non-existent file; - # the easiest way I found to get boto to raise a NoRegion error. - os.environ['AWS_CONFIG_FILE'] = '/dev/null' - if 'AWS_DEFAULT_REGION' in os.environ: - old_region = os.environ['AWS_DEFAULT_REGION'] - del os.environ['AWS_DEFAULT_REGION'] - - os.environ['AWS_CONFIG_FILE'] = '/dev/null' - - provider = provider_type() - - # If should not get an exception, it should be handled for us this this should return None - item = provider.get_item( - name='some-name', - directory=Directory.from_path("/a/b"), - directory_chain=None, - provider_chain=None, - environ=Directory(service='unittest', env='unittest') - ) - - # It should look nothing up. - assert item is None - - # See if we handled the correct error and not some other error. - from xyn_config.providers.common import aws_error_classes_to_ignore - assert provider.botocore_error_ignored_exception - assert type(provider.botocore_error_ignored_exception) in aws_error_classes_to_ignore - finally: - # Cleanup, this could contaminate other unit tests, remove it. - del os.environ['AWS_CONFIG_FILE'] - if old_region is not None: - os.environ['AWS_DEFAULT_REGION'] = old_region - - -@moto.mock_secretsmanager def test_secrets_manager_provider(): config.providers = [EnvironmentalProvider, SecretsManagerProvider] - aws_clients.secretsmanager.create_secret( + boto_clients.secretsmanager.create_secret( Name='/testing/unit/my_secret', SecretString='my-secret-value', ) @@ -548,10 +477,9 @@ def test_secrets_manager_provider(): assert config.get_value('MY_SECRET') == 'my-secret-value' -@moto.mock_secretsmanager def test_secrets_manager_provider_case_insensative(): config.providers = [EnvironmentalProvider, SecretsManagerProvider] - aws_clients.secretsmanager.create_secret( + boto_clients.secretsmanager.create_secret( Name='/testing/unit/MY_secret', SecretString='my-secret-value', ) @@ -560,16 +488,13 @@ def test_secrets_manager_provider_case_insensative(): assert config.get_value('MY_SECRET') == 'my-secret-value' -@moto.mock_dynamodb2 -@moto.mock_ssm -@moto.mock_secretsmanager @Config(providers=DEFAULT_TESTING_PROVIDERS, cacher=DynamoCacher) def test_expire_internal_local_cache(directory: Directory): # Basic defaults-test. - InternalLocalProviderCache.resource().expire_time_delta = dt.timedelta(milliseconds=250) + InternalLocalProviderCache.grab().expire_time_delta = dt.timedelta(milliseconds=250) path = f'/{config.SERVICE_NAME}/{config.APP_ENV}/exp_test_value' - aws_clients.ssm.put_parameter( + boto_clients.ssm.put_parameter( Name=path, Value="expiringTestValue", Type="String", @@ -579,7 +504,7 @@ def test_expire_internal_local_cache(directory: Directory): assert config.get('exp_test_value') == 'expiringTestValue' # Change value in SSM - aws_clients.ssm.put_parameter( + boto_clients.ssm.put_parameter( Name=path, Value="expiringTestValue2", Type="String", @@ -610,14 +535,11 @@ def test_expire_internal_local_cache(directory: Directory): assert config.get('exp_test_value') == 'expiringTestValue2' -@moto.mock_dynamodb2 -@moto.mock_ssm -@moto.mock_secretsmanager @Config(providers=DEFAULT_TESTING_PROVIDERS, cacher=DynamoCacher) def test_cache_uses_env_vars_by_default(directory: Directory): # First, put in value in SSM path = f'/{config.SERVICE_NAME}/{config.APP_ENV}/exp_test_value_3' - aws_clients.ssm.put_parameter( + boto_clients.ssm.put_parameter( Name=path, Value="expiringTestValue3", Type="String", @@ -660,16 +582,13 @@ def test_cache_uses_env_vars_by_default(directory: Directory): os.environ.pop('APP_ENV') -@moto.mock_dynamodb2 -@moto.mock_ssm -@moto.mock_secretsmanager @Config(providers=DEFAULT_TESTING_PROVIDERS, cacher=DynamoCacher) def test_dynamo_cacher_retrieves_new_values_after_local_cache_expires(directory: Directory): # Basic defaults-test. - InternalLocalProviderCache.resource().expire_time_delta = dt.timedelta(milliseconds=250) + InternalLocalProviderCache.grab().expire_time_delta = dt.timedelta(milliseconds=250) path = f'/{config.SERVICE_NAME}/{config.APP_ENV}/exp_test_value' - aws_clients.ssm.put_parameter( + boto_clients.ssm.put_parameter( Name=path, Value="expiringTestValue", Type="String", @@ -679,7 +598,7 @@ def test_dynamo_cacher_retrieves_new_values_after_local_cache_expires(directory: assert config.get('exp_test_value') == 'expiringTestValue' # Change value in SSM - aws_clients.ssm.put_parameter( + boto_clients.ssm.put_parameter( Name=path, Value="expiringTestValue2", Type="String", diff --git a/tests/test_errors.py b/tests/test_errors.py new file mode 100644 index 0000000..12a20f4 --- /dev/null +++ b/tests/test_errors.py @@ -0,0 +1,76 @@ +import datetime as dt +import functools +import os +import time +from typing import Type + +from xcon.providers.common import AwsProvider + +from xboto import boto_clients +import moto +import pytest +from xsentinels import Default +from xloop import xloop + +from xcon import Config, config +from xcon.directory import DirectoryItem, Directory +from xcon.provider import ProviderCacher, InternalLocalProviderCache +from xcon.providers import ( + EnvironmentalProvider, + DynamoProvider, + SsmParamStoreProvider, + SecretsManagerProvider, + DynamoCacher, default_provider_types, +) +from xcon.providers.dynamo import _ConfigDynamoTable + +DEFAULT_TESTING_PROVIDERS = [EnvironmentalProvider, SecretsManagerProvider, SsmParamStoreProvider] + + +@pytest.mark.parametrize( + "provider_type", + [ + SecretsManagerProvider, + DynamoProvider, + SsmParamStoreProvider + ], +) +# Must run it before the others, or boto will 'cache' the config-file; +# If the config file does not exist, it looks it up each time. +# So the other tests should run fine and be able to find a default region. +@pytest.mark.run(order=-10) +def test_providers_is_ok_without_region(provider_type: Type[AwsProvider]): + old_region = None + try: + # Point boto to a non-existent file; + # the easiest way I found to get boto to raise a NoRegion error. + os.environ['AWS_CONFIG_FILE'] = '/dev/null' + if 'AWS_DEFAULT_REGION' in os.environ: + old_region = os.environ['AWS_DEFAULT_REGION'] + del os.environ['AWS_DEFAULT_REGION'] + + os.environ['AWS_CONFIG_FILE'] = '/dev/null' + + provider = provider_type() + + # If should not get an exception, it should be handled for us this should return None + item = provider.get_item( + name='some-name', + directory=Directory.from_path("/a/b"), + directory_chain=None, + provider_chain=None, + environ=Directory(service='unittest', env='unittest') + ) + + # It should look nothing up. + assert item is None + + # See if we handled the correct error and not some other error. + from xcon.providers.common import aws_error_classes_to_ignore + assert provider.botocore_error_ignored_exception + assert type(provider.botocore_error_ignored_exception) in aws_error_classes_to_ignore + finally: + # Cleanup, this could contaminate other unit tests, remove it. + del os.environ['AWS_CONFIG_FILE'] + if old_region is not None: + os.environ['AWS_DEFAULT_REGION'] = old_region diff --git a/xcon/__init__.py b/xcon/__init__.py index 6518686..6908f3a 100644 --- a/xcon/__init__.py +++ b/xcon/__init__.py @@ -5,23 +5,23 @@ # Importable Attributes -Here are a few special attributes at the top-level `xyn_config` module that you can easily import. +Here are a few special attributes at the top-level `xcon` module that you can easily import. Go to [How To Use](#how-to-use) for more details on how to use this library. -- `xyn_config.config.Config`: Is the main class in by Config module, you can import easily it via +- `xcon.config.Config`: Is the main class in by Config module, you can import easily it via ```python - from xyn_config import Config + from xcon import Config ``` -- `xyn_config.config.config`: It represents the currently active `xyn_config.config.Config` +- `xcon.config.config`: It represents the currently active `xcon.config.Config` object. You can grab it via: ``` - from xyn_config import config + from xcon import config ``` For more details see: @@ -30,11 +30,11 @@ - More about what the 'current config' is: [Current Config](#current-config). -- `xyn_config.providers`: Easy access to the provider classes. +- `xcon.providers`: Easy access to the provider classes. See [Supported Providers](#supported-providers) for a list of providers. -- `xyn_config.config.ConfigSettings`: Used in projects to create a 'ConfigSettings' subclass. +- `xcon.config.ConfigSettings`: Used in projects to create a 'ConfigSettings' subclass. The subclass would allow you easily specify project settings to lazily lookup via Config. # How To Use diff --git a/xcon/config.py b/xcon/config.py index 0f2328d..1caf36b 100644 --- a/xcon/config.py +++ b/xcon/config.py @@ -1,10 +1,10 @@ """ -Main config module. Key pieces such as `xyn_config.config.Config` and -`xyn_config.config.config` are imported directly into "xyn_config". +Main config module. Key pieces such as `xcon.config.Config` and +`xcon.config.config` are imported directly into "xcon". Very quick example, this will grab `SOME_CONFIG_ATTR` for you: ->>> from xyn_config.config import config +>>> from xcon.config import config >>> config.SOME_CONFIG_ATTR .. todo:: Link to the readme.md docs @@ -21,14 +21,16 @@ # Note: pdoc3 can't resolve type-hints inside of method parameters with this enabled. # disabling it. (leaving it here for future reference for others) # from __future__ import annotations -from xyn_logging.loggers import XynLogger -from xyn_resource import Context, ActiveResourceProxy, Resource -from xyn_settings import Settings, SettingsField -from xyn_settings.settings import SettingsRetriever -from xyn_types import Default, OrderedDefaultSet, OrderedSet -from xyn_types.default import DefaultType -from xyn_utils.bool import bool_value -from xyn_utils.loop import loop +from xinject import XContext, Dependency +from xsettings import Settings, SettingsField +from xsettings.retreivers import SettingsRetrieverProtocol +from xsentinels import Default +from .types import OrderedDefaultSet, OrderedSet +from xsentinels.default import DefaultType +from xbool import bool_value +from xloop import xloop + +from logging import getLogger from .directory import Directory, DirectoryItem, DirectoryListing, DirectoryOrPath, DirectoryChain from .exceptions import ConfigError @@ -37,7 +39,7 @@ from .providers import default_provider_types from .providers.dynamo import DynamoCacher -xlog = XynLogger(__name__) +xlog = getLogger(__name__) T = TypeVar('T') @@ -66,7 +68,7 @@ def __post_init__(self): parents = self.parents if not isinstance(parents, tuple): # Convert to a tuple - object.__setattr__(self, 'parents', tuple(loop(parents))) + object.__setattr__(self, 'parents', tuple(xloop(parents))) def start_cursor(self) -> Optional[_ParentCursor]: """ @@ -95,7 +97,7 @@ def _check_proper_cacher_or_raise_error(cacher): ) -class Config(Resource): +class Config(Dependency): """ Lets you easily get configuration values from various sources. @@ -156,11 +158,11 @@ class Config(Resource): @classmethod def current(cls): - """ Calls 'cls.resource()', just am alternative name for the same thing, may make things + """ Calls 'cls.grab()', just am alternative name for the same thing, may make things a bit more self-documenting, since `Config` could be used in a lot of places. """ - return cls.resource() + return cls.grab() def __init__( self, *, @@ -181,9 +183,9 @@ def __init__( Parameters --------- - directories: Union[Iterable[xyn_config.directory.DirectoryOrPath], xyn_types.Default] + directories: Union[Iterable[xcon.directory.DirectoryOrPath], xsentinels.Default] List of directories/paths to search when querying for a name. - If `xyn_types.Default`: Uses the first one from [Parent Chain](#parent-chain). + If `xsentinels.Default`: Uses the first one from [Parent Chain](#parent-chain). If everyone in the parent chain is set to `Default`, uses `standard_directories()`. Various ways to change what directories to use: @@ -214,12 +216,12 @@ def __init__( and then if it still can't find the var it will next look at the current app/service for the var. - providers: Union[Iterable[Type[[xyn_config.provider.Provider]], [xyn_types.Default] + providers: Union[Iterable[Type[[xcon.provider.Provider]], [xsentinels.Default] List of provider types to use. If set to `Default`, uses the first one from [Parent Chain](#parent-chain). If everyone in the parent chain is set to `Default`, - uses `xyn_config.providers.default.default_provider_types`. + uses `xcon.providers.default.default_provider_types`. - cacher: Type[xyn_config.provider.ProviderCacher] + cacher: Type[xcon.provider.ProviderCacher] In the future, I may allow other cachers to be passed in via this param, but right now only the DynamoCacher is used and the only values you can use are: @@ -227,7 +229,7 @@ def __init__( - No flattened high-level caching will be used. The individual providers will still cache things internally per-directory/provider. - - If left as `xyn_types.Default`: + - If left as `xsentinels.Default`: - Must have a service/enviroment we can use (ie: APP_ENV / SERVICE_NAME). If so, we will attempt to read/write to a special Dynamo table that has a flattened list of name/value pairs that are tied to the current service, @@ -253,7 +255,7 @@ def __init__( [Parent Chain](#parent-chain) is used to find: - Overridden config values; these are values that set directly - on the Config object; ie: `xyn_config.config.config`.CONFIG_NAME = "Some Value to Override With" + on the Config object; ie: `xcon.config.config`.CONFIG_NAME = "Some Value to Override With" - Default values; these are used when config can find no other value for a particular .CONFIG_NAME. See `set_default` @@ -273,15 +275,15 @@ def __init__( in the [Parent Chain](#parent-chain) has `use_parent==false`, the parent-chain will stop there. - By `xyn_types.Default`: - We lookup the parent by getting the current Config via current Context; + By `xsentinels.Default`: + We lookup the parent by getting the current Config via current XContext; If that's ourselves, then we grab the parent context's Config resource. This lookup occurs every time we are asked for a .CONFIG_NAME to see if there is an override for it, etc. [see `parent is used to find` section above]. That means the Config's parent can change depending on the current context the time the .CONFIG_NAME is asked for. - defaults: Union[xyn_config.directory.DirectoryListing, Dict[str, Any], xyn_types.Default] + defaults: Union[xcon.directory.DirectoryListing, Dict[str, Any], xsentinels.Default] Side Note: If a default is not provided for "APP_ENV", a "dev" default will be added for it. You can set your own default either via `defaults['APP_ENV'] = 'whatever'` or you @@ -374,7 +376,7 @@ def __init__( # This property will lazily be used to create self.provider_chain when the chain # is requested for the first time. - self._providers = {x: None for x in loop(providers, iterate_dicts=True)} + self._providers = {x: None for x in xloop(providers)} # We lazy-lookup cacher if it's Default or a Type. # See 'self.cacher' property. @@ -400,7 +402,7 @@ def providers(self) -> Union[DefaultType, Iterable[Union[Type[Provider], Default If set to Default, it means we look to our [Parent Chain](#parent-chain) first, and if one of them don't have any set to then use sensible defaults. - Otherwise it's a list of `xyn_config.provider.Provider` types and/or Default. + Otherwise it's a list of `xcon.provider.Provider` types and/or Default. """ return self._providers.keys() @@ -411,7 +413,7 @@ def directories(self) -> Union[DefaultType, Iterable[Union[Directory, DefaultTyp If set to Default, it means we look to our [Parent Chain](#parent-chain) first, and if one of them don't have any set to then use sensible defaults. - Otherwise it's a list of `xyn_config.directory.Directory` and/or Default. + Otherwise it's a list of `xcon.directory.Directory` and/or Default. """ return self._directories.keys() @@ -426,7 +428,7 @@ def directories( """ # make an ordered-set out of this. dirs: OrderedDefaultSet = {} - for x in loop(value, iterate_dicts=True): + for x in xloop(value): if x is not Default: x = Directory.from_path(x) dirs[x] = None @@ -440,7 +442,7 @@ def providers(self, value: Union[DefaultType, Iterable[Union[DefaultType, Type[P when you ask for the `Config.provider_chain`. """ # make an ordered-set out of this. - self._providers = {x: None for x in loop(value)} + self._providers = {x: None for x in xloop(value)} def add_provider(self, provider: Type[Provider]): """ Adds a provider type to end of my provider type list [you can see what it is for @@ -454,7 +456,7 @@ def add_provider(self, provider: Type[Provider]): if provider in self._providers: return - # Add Provider type; using dict as an 'ordered set'; see xyn_types.OrderedSet. + # Add Provider type; using dict as an 'ordered set'; see xsentinels.OrderedSet. self._providers[provider] = None def add_directory(self, directory: Union[Directory, str, DefaultType]): @@ -468,7 +470,7 @@ def add_directory(self, directory: Union[Directory, str, DefaultType]): if directory in self._directories: return - # Add Directory; using dict as an 'ordered set'; see xyn_types.OrderedSet + # Add Directory; using dict as an 'ordered set'; see xsentinels.OrderedSet self._directories[directory] = None def add_export(self, *, service: str): @@ -486,7 +488,7 @@ def add_export(self, *, service: str): By default, the export list is just this: - ( `xyn_types.Default`, ) + ( `xsentinels.Default`, ) When you add more exports via `Config.add_export`, it will append to the end of this list. That way we still add whatever we need from parent and then an explicitly added to self. @@ -514,17 +516,17 @@ def set_exports(self, *, services: Iterable[Union[str, DefaultType]]): This replaces all current services. By default, the export list is this: - ( `xyn_types.Default`, ) + ( `xsentinels.Default`, ) Which means, we ask the parent chain for an exports. If you set the exports without including this then the parent-chain won't be consulted. Args: - services (Iterable[Union[str, `xyn_types.Default`]]): List of exports you want + services (Iterable[Union[str, `xsentinels.Default`]]): List of exports you want to add by service name. If you don't add the `Default` somewhere in this list then we will NOT check the parent-chain """ - self._exports = {x: None for x in loop(services, iterate_dicts=True)} + self._exports = {x: None for x in xloop(services)} def get_exports_by_service(self): """ List of services we current check their export's for. This only lists the exports @@ -542,7 +544,7 @@ def set_override(self, name, value: [Any, Default]): You can also set an override by setting a value for a config-name directly on `Config` via this syntax: - >>> from xyn_config.config import config + >>> from xcon.config import config >>> config.SOME_OVERRIDE_NAME = "my override value" @@ -559,7 +561,7 @@ def set_override(self, name, value: [Any, Default]): Args: name: Name of the item to remove, case-insensitive. - value (Union[Any, xyn_types.Default]): Can be any value. If Default is used + value (Union[Any, xsentinels.Default]): Can be any value. If Default is used we will instead call `Config.remove_override(name)` for you to remove the value. """ if value is Default: @@ -586,9 +588,9 @@ def get_override(self, name) -> Union[Any, DefaultType, None]: - `config.set_override` - `config.SOME_VAR = "a-value"` - The returned value is `xyn_types.Default` if no override is found; + The returned value is `xsentinels.Default` if no override is found; this is so you can distinguish between overriding to None or no override set at all - (`xyn_types.Default` evaluates to `False`, just like how `None` works). + (`xsentinels.Default` evaluates to `False`, just like how `None` works). .. warning:: Only returns a value if overrides was directly set on self! @@ -603,10 +605,10 @@ def get_override(self, name) -> Union[Any, DefaultType, None]: name (str): Name to use to get override [case-insensitive]. Returns: - Union[Any, xyn_types.Default]: The value, or `xyn_types.Default` if no value was + Union[Any, xsentinels.Default]: The value, or `xsentinels.Default` if no value was set for `name`. This allows you to distinguish between overriding a value to `None` and no override being set in the first place - (`xyn_types.Default` evaluates to `False`, just like how `None` works). + (`xsentinels.Default` evaluates to `False`, just like how `None` works). """ item = self._override.get_item(name) if item: @@ -635,8 +637,8 @@ def remove_override(self, name): You can remove overrides in various ways, such as: ```python - from xyn_config import config - from xyn_types import Default + from xcon import config + from xsentinels import Default # Alternate Method 1: config.SOME_NAME = Default @@ -695,7 +697,7 @@ def service(self) -> Union[DefaultType, str]: take a look at `config.SERVICE_NAME`. That will check the [Parent Chain](#parent-chain) if needed. You could also ask for the `Config.directory_chain` and see what the first Directory in the chain has for the service - `xyn_config.directory.Directory.service`. + `xcon.directory.Directory.service`. """ item = self._override.get_item('service_name') if item: @@ -739,7 +741,7 @@ def environment(self) -> str: take a look at `config.APP_ENV`. That will check the [Parent Chain](#parent-chain) if needed. You could also ask for the `Config.directory_chain` and see what the first Directory in the chain has for the environment - `xyn_config.directory.Directory.env`. + `xcon.directory.Directory.env`. """ item = self._override.get_item('app_env') if item: @@ -787,9 +789,9 @@ def resolved_cacher(self) -> Optional[ProviderCacher]: @property def provider_chain(self) -> ProviderChain: """ - `xyn_config.provider.ProviderChain` we are currently using. + `xcon.provider.ProviderChain` we are currently using. This is effected by what was passed into Config when it was created. - If it was left as `xyn_types.Default`, we will get the value via the + If it was left as `xsentinels.Default`, we will get the value via the [Parent Chain](#parent-chain). See `Config` for more details. @@ -803,9 +805,9 @@ def provider_chain(self) -> ProviderChain: @property def directory_chain(self) -> DirectoryChain: """ - `xyn_config.directory.DirectoryChain` we are currently using. + `xcon.directory.DirectoryChain` we are currently using. This is effected by what was passed into Config when it was created. - If it was left as `xyn_types.Default`, we will get the value via the + If it was left as `xsentinels.Default`, we will get the value via the [Parent Chain](#parent-chain). See `Config` for more details. @@ -816,7 +818,7 @@ def directory_chain(self) -> DirectoryChain: def use_parent(self) -> bool: """ If `True`: we will use the [Parent Chain](#parent-chain) when looking up things such as the - `Config.provider_chain`. as an example; if it was left as `xyn_types.Default` + `Config.provider_chain`. as an example; if it was left as `xsentinels.Default` when `Config` object was created. If `False`: the parent will not be consulted, and anything that was not set at creation @@ -872,9 +874,9 @@ def get_default(self, name: str) -> Optional[Any]: name (str): Name to use to get default [case-insensitive]. Returns: - Union[Any, None, xyn_types.Default]: The value, or Default if no default + Union[Any, None, xsentinels.Default]: The value, or Default if no default is set for name. This allows you to distinguish between defaulting a value to - None and no default being set in the first place (`xyn_types.Default` + None and no default being set in the first place (`xsentinels.Default` looks like `False`, just like how `None` works). """ item = self._defaults.get_item(name) @@ -892,8 +894,8 @@ def remove_default(self, name): You can also call this other ways, such as: ```python - from xyn_config import config - from xyn_types import Default + from xcon import config + from xsentinels import Default # Alternate Method 1: config.set_default("SOME_NAME", Default) @@ -913,7 +915,7 @@ def get( ) -> Optional[str]: """ Similar to dict.get(), provide name [case-insensitive] and we call `Config.get_item()` - and return the `xyn_config.directory.DirectoryItem.value` of the item returned, + and return the `xcon.directory.DirectoryItem.value` of the item returned, or passed in `default=None` if no item was found. See documentation for `Config.get_item()` for more details and to find out more @@ -945,7 +947,7 @@ def get( for now. """ if ignore_local_caches: - InternalLocalProviderCache.resource().reset_cache() + InternalLocalProviderCache.grab().reset_cache() item = self.get_item(name=name, skip_providers=skip_providers, skip_logging=skip_logging) if item: @@ -991,10 +993,10 @@ def get_item( ) -> Optional[DirectoryItem]: """ Gets a DirectoryItem for name. If the value does not exist, we will still return a - `xyn_config.directory.DirectoryItem` with a - `xyn_config.directory.DirectoryItem.value` == `None`. This is because we cache the + `xcon.directory.DirectoryItem` with a + `xcon.directory.DirectoryItem.value` == `None`. This is because we cache the non-existence of items for performance reasons. This allows you to see - where the None value came from via the `xyn_config.directory.DirectoryItem.directory` + where the None value came from via the `xcon.directory.DirectoryItem.directory` attribute. Attributes: @@ -1207,7 +1209,7 @@ def _cacher_with_cursor( # Lower-casing it because `EnvironmentalProvider` will do that for us when looking it # up (it looks it up in a case-insensitive manner). # Trying to make it a tiny bit more efficient since this is called a lot. - env_provider = EnvironmentalProvider.resource() + env_provider = EnvironmentalProvider.grab() if bool_value(env_provider.get_value_without_environ("config_disable_default_cacher")): return None cacher = DynamoCacher @@ -1223,14 +1225,14 @@ def _cacher_with_cursor( f"Trying to get the cacher, but the type the user wants to use is not a " f"DynamoCacher type: ({cacher}) In the future I may support other cacher " f"types; but right now we only support either None, Default or " - f"xyn_config.providers.dynamo.DynamoCacher." + f"xcon.providers.dynamo.DynamoCacher." ) # Grab the current cacher resource, it's a ProviderCacher type of some sort and - # so is a xyn_resource.resource.Resource + # so is a xinject.dependency.Dependency # (right now, cacher can only be a DynamoCacher type; # although we can change that in the future if we decide to change how caching works) - return cacher.resource() + return cacher.grab() def _get_item( self, name: str, *, @@ -1385,13 +1387,13 @@ def _parent_chain(self) -> _ParentChain: See [Parent Chain](#parent-chain). There is a concept of a parent-chain with Config if the Config object has - their `use_parent == True` [it defaults to True]. We use the current Context to + their `use_parent == True` [it defaults to True]. We use the current XContext to construct this parent-chain. See [parent-chain]. - The parent-chain starts with the config resource in the current Context. + The parent-chain starts with the config resource in the current XContext. If that context has a parent context, we next grab the Config resource from that parent context and check it's `Config.use_parent`. If True we keep doing - this until we reach a Context without a parent or a `Config.use_parent` that is False. + this until we reach a XContext without a parent or a `Config.use_parent` that is False. We take out of the chain any config object that is myself. The only objects in the chain are other Config object instances. @@ -1405,7 +1407,7 @@ def _parent_chain(self) -> _ParentChain: skip_adding_more_parents = False chain = [] - for config_resource in Context.current().resource_chain(Config, create=True): + for config_resource in XContext.grab().dependency_chain(Config, create=True): if config_resource is self: found_self = True if not use_parent: @@ -1525,7 +1527,7 @@ def _get_special_non_provider_item_with_cursor( return item return item.value - item = EnvironmentalProvider.resource().get_item_without_environ(name) + item = EnvironmentalProvider.grab().get_item_without_environ(name) if not item or not item.value: item = DirectoryItem( directory="/_default/hard-coded", @@ -1561,7 +1563,7 @@ def __setattr__(self, key, value): # for details on what this is and how to use it. # # noinspection PyRedeclaration -config = ActiveResourceProxy.wrap(Config) +config = Config.proxy() """ This will be an alias for the current Config object. Every time you ask it for something, it looks up the current Config object and gets it from that. This means you can use this @@ -1571,12 +1573,12 @@ def __setattr__(self, key, value): Example use case: ```python - from xyn_config import config + from xcon import config value = config.SOME_VAR ``` """ -Config.resource() +Config.grab() def _env_only_is_turned_on() -> bool: @@ -1602,7 +1604,7 @@ def standard_directories(*, service: str, env: str) -> OrderedSet[Directory]: directory paths to check beyond the defaults, you can call this method and append them to a config object, like so: - >>> from xyn_config.config import config + >>> from xcon.config import config >>> for directory in standard_directories(service="customService", env="customEnv"): ... config.add_directory(directory) @@ -1680,11 +1682,11 @@ def standard_directories(*, service: str, env: str) -> OrderedSet[Directory]: _std_directory_chain_cache: Dict[str, OrderedSet[Directory]] = dict() -class ConfigRetriever(SettingsRetriever): +class ConfigRetriever(SettingsRetrieverProtocol): """Retrieving the setting from config""" - def retrieve_value(self, field: SettingsField) -> Any: + def __call__(self, *, field: SettingsField, settings: 'Settings') -> Any: return config.get(field.name) -class ConfigSettings(Settings, default_retriever=ConfigRetriever()): +class ConfigSettings(Settings, default_retrievers=[ConfigRetriever()]): pass diff --git a/xcon/directory.py b/xcon/directory.py index 562bbd4..9c5d5c3 100644 --- a/xcon/directory.py +++ b/xcon/directory.py @@ -12,13 +12,13 @@ "DirectoryItem.__str__": True, } -from xyn_dateutils.dates import parse_iso_datetime +import ciso8601 -from xyn_types import Default -from xyn_types.default import JsonDict -from xyn_utils.loop import loop +from xsentinels import Default +from .types import JsonDict +from xloop import xloop -from xyn_config.exceptions import ConfigError +from xcon.exceptions import ConfigError @dataclass(eq=True, frozen=True) @@ -224,7 +224,7 @@ def _service_env_from_path(path: str) -> Tuple[Optional[str], Optional[str]]: DirectoryItemValue = Union[JsonDict, list, str, int, None] """ A type indicating the of values a `DirectoryItem.value` could return. - Generally, it's either a `xyn_types.JsonDict` or a `list`/`str`/`int`/`None`. + Generally, it's either a `xsentinels.JsonDict` or a `list`/`str`/`int`/`None`. Basically, the basic str/int in combination with what you generally could store in JSON. """ @@ -483,7 +483,7 @@ def from_json(cls, json: JsonDict, append_source: str = '', from_cacher: bool = cache_concat_provider_names = json.get('cache_concat_provider_names') created_at = json.get('created_at', None) - created_at = parse_iso_datetime(created_at) if created_at else None + created_at = ciso8601.parse_datetime(created_at) if created_at else None return DirectoryItem( directory=directory, @@ -557,7 +557,7 @@ class DirectoryListing: def __init__(self, directory: Directory = None, items: Iterable[DirectoryItem] = None): self.directory = directory self._items = {} - for item in loop(items): + for item in xloop(items): self.add_item(item) def get_any_item(self) -> Optional[DirectoryItem]: diff --git a/xcon/provider.py b/xcon/provider.py index d5f75cd..612d6a5 100644 --- a/xcon/provider.py +++ b/xcon/provider.py @@ -9,28 +9,28 @@ from typing import Union from botocore.exceptions import BotoCoreError -from xyn_logging.loggers import XynLogger -from xyn_resource import Resource, Context -from xyn_utils.loop import loop +from xinject import Dependency, XContext +from xloop import xloop from .directory import Directory, DirectoryOrPath, DirectoryItem, DirectoryChain, DirectoryListing import datetime as dt +from logging import getLogger -xlog = XynLogger(__name__) +log = getLogger(__name__) -class Provider(Resource, ABC): +class Provider(Dependency, ABC): """ Represents a Provider, which wraps a resource that can be used to store Config values based on a provided directory. It caches these directories so future lookups don't keep having to - fetch them from the Resource again in the future, while the process is still running. + fetch them from the Dependency again in the future, while the process is still running. Most of the time there is no need for a several of the same provider in the same process/app. The providers generally keep a cache of every directory they already looked up, so it's nice to share that cache with other Child context's by default via the - `xyn_resource.resource.Resource` mechanism. + `xinject.dependency.Dependency` mechanism. Config will use the current/active provider when it needs to consult a Provider for values. @@ -42,25 +42,17 @@ class Provider(Resource, ABC): - boto clients/resources are also not thread-safe. - Sessions from requests library can't be used cross-thread. - For boto, if you use `xyn_aws` to grab the boto client/resource, + For boto, if you use `xboto` to grab the boto client/resource, it will allow you to lazily get a shared object that is guaranteed to only be shared for the current thread. This allows boto to reuse connections for things running on the same thread, - but xyn_aws will lazily create a new client if your on a seperate thread. - - As for request sessions, we will eventually have something in the `xyn_requests` library, - but for now you'll have to do it your self. - - You can look at `xyn_model_rest.session.Session`. - It's a resource that lets you get the 'current' requests session - (which allows re-use of TCP connections that are already connected, - but will also lazily return a session that can always be used on current thread). + but `xboto` will lazily create a new client if your on a separate thread. """ name = "?" """ This is the value that will normally be set to the items - `xyn_config.directory.DirectoryItem.source`, also displayed + `xcon.directory.DirectoryItem.source`, also displayed when logging out the names of providers when something can't be found. """ @@ -85,7 +77,7 @@ class Provider(Resource, ABC): needs_directory = True """ By default, providers can't really use a `None` for a directory when calling `get_item()`. If you CAN work with a None directory then set this to False (for example - `xyn_config.providers.environmental.EnvironmentalProvider` uses this). + `xcon.providers.environmental.EnvironmentalProvider` uses this). A `None` normally means that we could not determine the proper directory to use. This can happen if no SERVICE_NAME and APP_ENV are defined. But some providers don't @@ -125,7 +117,7 @@ def get_item( cacher. But it's used by the other providers. This cacher acts just like a provider and so accepts the parameter. directory_chain: Current directory chain that is being used to lookup value. - provider_chain (xyn_config.provider.ProviderChain): Current provider chain + provider_chain (xcon.provider.ProviderChain): Current provider chain that is being used to lookup value. environ: This is supposed to have the full service and environment name. @@ -133,7 +125,7 @@ def get_item( Example Directory Path: `/hubspot/testing` Returns: - xyn_config.directory.DirectoryItem: If we have the item, this is it. + xcon.directory.DirectoryItem: If we have the item, this is it. None: Otherwise we return None indicating we don't know about it. """ raise NotImplementedError(f"Need to implement in ({self}).") @@ -162,7 +154,7 @@ def log_about_items( ): # We could be called before application has configured it's logging; # ensure logging has been configured before we log out. - # Otherwise log message may never get logged out + # Other-wise log message may never get logged out # (Python defaults to Warning log level). # Use cache_range_key if it exists, otherwise use name. @@ -171,9 +163,9 @@ def log_about_items( provider_class = self.__class__.__name__ thread_name = threading.current_thread().name - xlog.info( - "{msg_prefix} values via provider ({provider}/{provider_class}) " - "for path ({path}), for thread ({thread_name}), for names ({names}).", + log.info( + f"{msg_prefix} values via provider ({self.name}/{provider_class}) " + f"for path ({path}), for thread ({thread_name}), for names ({names}).", extra=dict( msg_prefix=msg_prefix, provider=self.name, @@ -230,7 +222,7 @@ class AwsProvider(Provider, ABC): @property def local_cache(self) -> Dict[Directory, DirectoryListing]: - cacher = InternalLocalProviderCache.resource() + cacher = InternalLocalProviderCache.grab() return cacher.get_cache_for_provider(provider=self, cache_constructor=lambda c: dict()) @@ -276,7 +268,7 @@ class ProviderChain: Starting with the first provider in my list has `Provider.query_before_cache_if_possible` set to False (default) we will consider them cachable. - Normally, the `xyn_config.providers.environmental.EnvironmentalProvider` provider + Normally, the `xcon.providers.environmental.EnvironmentalProvider` provider is the only non-cacheable provider, and normally it's listed first. This means that we will normally not cache values from this EnvironmentalProvider. @@ -306,13 +298,13 @@ class ProviderChain: def __post_init__(self): providers = list() - context = Context.current() + context = XContext.grab() provider_key_names = [] query_before_finished = False - for p in loop(self.providers, iterate_dicts=True): + for p in xloop(self.providers): # Check to see if any of them are classes [and type's resources needs to be grabbed]. if isclass(p): - p = context.resource(p) + p = context.dependency(p) providers.append(p) if not query_before_finished: @@ -470,7 +462,7 @@ def retrieved_items_map( return final_map -class InternalLocalProviderCache(Resource): +class InternalLocalProviderCache(Dependency): """ Used by the providers for a place to store/cache things they retrieve from the systems they provide configuration values from. @@ -506,7 +498,7 @@ class InternalLocalProviderCache(Resource): The providers do this every time they are asked for a value. In addition to changing this directly - (via `InternalLocalProviderCache.resource().expire_time_delta` = ...) + (via `InternalLocalProviderCache.grab().expire_time_delta` = ...) you can also override this via an environmental variable: `CONFIG_INTERNAL_CACHE_EXPIRATION_MINUTES` diff --git a/xcon/providers/common.py b/xcon/providers/common.py index 5e95b28..1529720 100644 --- a/xcon/providers/common.py +++ b/xcon/providers/common.py @@ -25,13 +25,13 @@ def handle_aws_exception(exception: Exception, provider: AwsProvider, directory: Directory): - """ Used by the `xyn_config.config.Config` providers that connect to AWS. + """ Used by the `xcon.config.Config` providers that connect to AWS. A common set of code to handle exceptions that happen while getting configuration values from various AWS resources. If we ignore an error, we will log a warning, and then set the directory as an error'd directory via `log_ignored_aws_exception`; which in turn calls - `xyn_config.provider.AwsProvider.mark_errored_directory` on the provider. + `xcon.provider.AwsProvider.mark_errored_directory` on the provider. This informs the provider so they don't keep asking for this directory in the future. """ # First check to see if we have a specific `BotoCoreError` subclass of some sort... @@ -76,13 +76,13 @@ def log_ignored_aws_exception( ): """ We will log a warning, and then set the directory as an error'd directory via - `xyn_config.provider.AwsProvider.mark_errored_directory` on the provider. + `xcon.provider.AwsProvider.mark_errored_directory` on the provider. This informs the provider so they don't keep asking for this directory in the future. Args: exception: Exception that tells us about the error. - provider (xyn_config.provider.AwsProvider): AwsProvider that had the error. - directory (xyn_config.directory.Directory): Directory that had the error. + provider (xcon.provider.AwsProvider): AwsProvider that had the error. + directory (xcon.directory.Directory): Directory that had the error. error_detail: Some extra human redable details about the error. Returns: diff --git a/xcon/providers/default.py b/xcon/providers/default.py index cc079ca..ade60fe 100644 --- a/xcon/providers/default.py +++ b/xcon/providers/default.py @@ -11,5 +11,5 @@ SecretsManagerProvider, SsmParamStoreProvider ] -""" Set of default provider types and order to use them in for `xyn_config.config.Config`. +""" Set of default provider types and order to use them in for `xcon.config.Config`. """ diff --git a/xcon/providers/dynamo.py b/xcon/providers/dynamo.py index f51001a..dc296cf 100644 --- a/xcon/providers/dynamo.py +++ b/xcon/providers/dynamo.py @@ -10,14 +10,14 @@ from typing import Mapping from boto3.dynamodb import conditions -from xyn_aws.dynamodb import DynamoDB -from xyn_types import Default -from xyn_utils.loop import loop +from xboto.resource import dynamodb +from xsentinels import Default +from xloop import xloop -from xyn_config.directory import Directory, DirectoryListing, DirectoryOrPath, DirectoryItem, \ +from xcon.directory import Directory, DirectoryListing, DirectoryOrPath, DirectoryItem, \ DirectoryChain -from xyn_config.exceptions import ConfigError -from xyn_config.provider import ProviderCacher, ProviderChain, AwsProvider, \ +from xcon.exceptions import ConfigError +from xcon.provider import ProviderCacher, ProviderChain, AwsProvider, \ InternalLocalProviderCache from .common import handle_aws_exception @@ -30,16 +30,13 @@ class DynamoProvider(AwsProvider): This provider allows one to have a structured list or dictionary. It supports JSON and will parse/decode it when it gets it from Dynamo into a real Python dict/list/str/etc! """ - attributes_to_skip_while_copying = ['_table'] - - _directories: Dict[Directory, DirectoryListing] - _table: _ConfigDynamoTable - name = "dynamo" + _directories: Dict[Directory, DirectoryListing] - def __init__(self): - super().__init__() - self._table = _ConfigDynamoTable() + @property + def _table(self) -> _ConfigDynamoTable: + # todo: make table name configurable + return _ConfigDynamoTable(table_name='global-config') def get_item( self, @@ -57,7 +54,7 @@ def get_item( if listing: return listing.get_item(name) - # We need to lookup the directory listing from Dynamo. + # We need to look up the directory listing from Dynamo. items = [] try: @@ -105,7 +102,7 @@ class DynamoCacher(ProviderCacher): 1. Hash key: Is the `environ` method parameter that gets passed to the methods on me. It's normally a string in this format: `/{SERVICE_NAME}/{APP_ENV}`. Also, normally apps/services are only given access to one specific hash-key. - This hash-key should represent the app and it's current environment. + This hash-key should represent the app and its current environment. 2. Range/Sort key: Contains the variable name, providers and directory paths used to lookup value. This makes it so the app can do various queries using various different providers/directories and the cacher can cache those results correctly @@ -115,26 +112,31 @@ class DynamoCacher(ProviderCacher): You don't need to parse the rage/sort-key. All of its components are also separate attributes in the table on the row. - ## Resource Details + ## Dependency Details - Right now the `DynamoCacher` is a `xyn_resource.resource.Resource` - resource, you can grab the current one by calling `DynamoCacher.resource()`. + Right now the `DynamoCacher` is a `xinject.dependency.Dependency` + resource, you can grab the current one by calling `DynamoCacher.grab()`. - More specifically: we are a `xyn_resource.resource.Resource`, which means that there is - normally only one of us around. See xyn-resource library for more details. + More specifically: we are a `xinject.dependency.Dependency`, which means that there is + normally only one of us around. See xinject library for more details. """ + name = "cacher" + _ttl: dt.datetime + def retrieved_items_map(self, directory: DirectoryOrPath) -> Mapping[str, DirectoryItem]: """ This is mostly useful for getting this to cache, so I am not going to implement it - in the cacher (if we ever do, the `xyn_config.provider.ProviderChain` will have to + in the cacher (if we ever do, the `xcon.provider.ProviderChain` will have to figure out how to skip us). """ return {} - name = "cacher" - attributes_to_skip_while_copying = ['_table'] - _table: _ConfigDynamoTable - _ttl: dt.datetime + @property + def _table(self) -> _ConfigDynamoTable: + # todo: make table name configurable + table = _ConfigDynamoTable(table_name='global-configCache', cache_table=True) + table.append_source = " - via cacher" + return table @dataclasses.dataclass class _LocalCache: @@ -149,9 +151,9 @@ class _LocalCache: @property def local_cache(self) -> _LocalCache: - # Using default dict so I don't have to worry about allocating the dict's my self later. + # Using default dict, so I don't have to worry about allocating the dict's my self later. maker = lambda c: DynamoCacher._LocalCache() - cacher = InternalLocalProviderCache.resource() + cacher = InternalLocalProviderCache.grab() return cacher.get_cache_for_provider(provider=self, cache_constructor=maker) def __init__(self): @@ -172,7 +174,6 @@ def __init__(self): 12 hours in the future with a random +/- 1500 seconds added on is what we currently do. Thinking about making it a shorter period of time [a couple of hours]. """ - self._table = _ConfigDynamoTable(table_name='global-configCache', cache_table=True) self._table.append_source = " - via cacher" super().__init__() @@ -278,7 +279,7 @@ def get_item( a provider and so accepts the parameter. directory_chain: Current directory chain that is being used to lookup value. Used as part of the rang-key in the Dynamo table. - provider_chain (xyn_config.provider.ProviderChain): Current provider chain + provider_chain (xcon.provider.ProviderChain): Current provider chain that is being used to lookup value. Used as part of the rang-key in the Dynamo table. environ: @@ -288,7 +289,7 @@ def get_item( Example Directory Path: `/hubspot/testing` Returns: - xyn_config.directory.DirectoryItem: If we have a cached item, this is it. + xcon.directory.DirectoryItem: If we have a cached item, this is it. None: Otherwise we return None indicating nothing has been cached. """ # Cache needs all of this stuff to do proper caching. @@ -344,7 +345,7 @@ def _get_items_for_environ(self, environ: Directory) -> Iterable[DirectoryItem]: items = self._table.get_items_for_directory(directory=environ, expire_time=expire_time) # Ensure we have a list, and not a generator. - items = tuple(loop(items)) + items = tuple(xloop(items)) # Log about stuff we retrieved from the cache table. self.log_about_items(items=items, path=environ.path) @@ -397,12 +398,18 @@ def _get_environ_to_use( # in this library (it's a bit heavy). So for now, we are duplicating some of that functionality # for use here, in a much simpler (but WAY less feature-rich) way: class _ConfigDynamoTable: - append_source = "dynamo" + """ + Meant to be a simple abstract around dynamo table, just enough for our needs in this + `dynamo.py` module file... - @property - def db(self) -> DynamoDB: - """ DynamoDB resource. """ - return DynamoDB.resource() + After doing all the needed work for getting/updating items and so forth, + you should throw-away the `_ConfigDynamoTable` object and lazily create a new one next time a + call from the user comes in that needs the table. + + This helps support dependency injection of the dynamodb boto3 resource via xinject + (always uses dependency when called, so it can be changed/injected by user). + """ + append_source = "dynamo" @property def table_name(self) -> str: @@ -414,18 +421,17 @@ def table(self): """ DynamoDB table resource. We lazily get the resource, so we don't have to verify/create it if not needed. """ - table = self._table if table is not None: return table - table = self.db.table(name=self.table_name, table_creator=self._create_table) + table = dynamodb.Table(self.table_name) self._table = table return table def __init__( self, - table_name: str = "global-config", + table_name: str, cache_table: bool = False ): super().__init__() @@ -490,9 +496,9 @@ def get_items_for_directory( that DynamoDB only guarantees an expired item will be deleted within 48 hours, so it will only be returned if DynamoDB has not deleted the item yet. If dt.datetime: - I'll use the provided datetime for the expire time. Items will only be returned + I'll use the provided datetime for the expiry time. Items will only be returned if they don't have an expiration time, or if their expiration time is greater - then the provided date/time. + than the provided date/time. :return: """ @@ -566,45 +572,6 @@ def _with_batch_writer(self): _table_name: str _verified_table_status: bool - - def _create_table(self, dynamo: DynamoDB = None): - # This is mainly here to create table when mocking aws for unit tests. If the table - # really does not exist in reality, this also can create it. - - # Time To Live Notes: - # - # We can't enable TimeToLive during table creation, we have to wait until after it's - # created. This is a minor issue, since the table will still function correctly, - # the queries will still filter out expired items like normal. The only difference - # is we could get charged extra for storage we are not using. We need to still filter - # items out of our queries because deletion does not happen immediately [could take up - # to 48 hours]. - # - # At this point, it's expected that you'll have to go into the AWS dynamo console - # to setup automatic TimeToLive item deletion for a table. - return dynamo.db.create_table( - TableName=self.table_name, - KeySchema=[ - # Partition Key - {'AttributeName': 'directory', 'KeyType': 'HASH'}, - # Sort Key - {'AttributeName': 'name', 'KeyType': 'RANGE'} - ], - AttributeDefinitions=[ - {'AttributeName': 'directory', 'AttributeType': 'S'}, - {'AttributeName': 'name', 'AttributeType': 'S'} - ], - # todo: - # YOu need to use a newer-boto3 for this to work than what lamda provides. - # HOWEVER, the config table should always exist, so we should not have to really - # worry about it. If the able already exists we won't attempt to create it. - BillingMode='PAY_PER_REQUEST', - Tags=[{'Key': 'DDBTableGroupKey', 'Value': 'xyn_config'}], - SSESpecification={ - "Enabled": True - } - ) - _batch_writer = None def _paginate_all_items_generator( diff --git a/xcon/providers/environmental.py b/xcon/providers/environmental.py index 50cef4f..f81733b 100644 --- a/xcon/providers/environmental.py +++ b/xcon/providers/environmental.py @@ -3,10 +3,10 @@ import os from typing import Optional, Mapping, Dict, Any -from xyn_config.directory import ( +from xcon.directory import ( DirectoryOrPath, DirectoryItem, DirectoryChain, Directory, DirectoryListing ) -from xyn_config.provider import Provider, ProviderChain, InternalLocalProviderCache +from xcon.provider import Provider, ProviderChain, InternalLocalProviderCache class EnvironmentalProvider(Provider): @@ -47,7 +47,7 @@ def local_cache(self) -> DirectoryListing: return self._user_provided_cache maker = lambda c: self._create_snapshot(None, c) - cacher = InternalLocalProviderCache.resource() + cacher = InternalLocalProviderCache.grab() return cacher.get_cache_for_provider(provider=self, cache_constructor=maker) def __init__(self, env_vars: Optional[Dict[str, Any]] = None): @@ -85,7 +85,7 @@ def _create_snapshot( internal_cache_provider: If provided, we will use this as the object to store our environmental snapshot on. - If not provided, we will get the `InternalLocalProviderCache.resource()` + If not provided, we will get the `InternalLocalProviderCache.grab()` (current instance). In either case, we won't use a `InternalLocalProviderCache` if you pass @@ -100,7 +100,7 @@ def _create_snapshot( if from_env_dict is None: # If an internal cacher not provided, get current one. if not internal_cache_provider: - internal_cache_provider = InternalLocalProviderCache.resource() + internal_cache_provider = InternalLocalProviderCache.grab() from_env_dict = os.environ msg_prefix = "Snapshotted os.environ" diff --git a/xcon/providers/secrets_manager.py b/xcon/providers/secrets_manager.py index 96ff616..10f425e 100644 --- a/xcon/providers/secrets_manager.py +++ b/xcon/providers/secrets_manager.py @@ -9,8 +9,8 @@ from .common import handle_aws_exception from ..directory import Directory, DirectoryListing, DirectoryOrPath, DirectoryItem, DirectoryChain from botocore.exceptions import ClientError -from xyn_config.provider import AwsProvider, ProviderChain, InternalLocalProviderCache -from xyn_aws import aws_clients +from xcon.provider import AwsProvider, ProviderChain, InternalLocalProviderCache +from xboto import boto_clients log = logging.getLogger(__name__) @@ -66,7 +66,7 @@ class SecretsManagerProvider(AwsProvider): def local_cache(self) -> _LocalSecretsManagerCache: # Using default dict so I don't have to worry about allocating the dict's my self later. maker = lambda c: _LocalSecretsManagerCache() - cacher = InternalLocalProviderCache.resource() + cacher = InternalLocalProviderCache.grab() return cacher.get_cache_for_provider(provider=self, cache_constructor=maker) def _available_names_for_directory(self) -> Dict[Directory, DirectoryListing]: @@ -85,7 +85,7 @@ def _available_names_for_directory(self) -> Dict[Directory, DirectoryListing]: log.info("Getting full listing of available path/names in AWS Secrets Manager.") dir_to_item_map = {} try: - paginator = aws_clients.secretsmanager.get_paginator('list_secrets') + paginator = boto_clients.secretsmanager.get_paginator('list_secrets') response = paginator.paginate() for page in response: @@ -170,7 +170,7 @@ def get_item( secret = None try: log.info(f"Getting value at SecretsManagerProvider path ({item_path})") - item_value: Dict[str, Any] = aws_clients.secretsmanager.get_secret_value( + item_value: Dict[str, Any] = boto_clients.secretsmanager.get_secret_value( SecretId=item_path ) secret = item_value.get('SecretString') diff --git a/xcon/providers/ssm_param_store.py b/xcon/providers/ssm_param_store.py index 9b15182..ab47ba8 100644 --- a/xcon/providers/ssm_param_store.py +++ b/xcon/providers/ssm_param_store.py @@ -2,9 +2,9 @@ from typing import Dict, Optional, Mapping -from xyn_aws.proxy import aws_clients +from xboto import boto_clients -from xyn_config.provider import AwsProvider, ProviderChain +from xcon.provider import AwsProvider, ProviderChain from .common import handle_aws_exception from ..directory import Directory, DirectoryListing, DirectoryOrPath, DirectoryItem, DirectoryChain @@ -19,7 +19,7 @@ class SsmParamStoreProvider(AwsProvider): def _get_params_paginator(self): paginator = self._store_get_params_paginator if not paginator: - paginator = aws_clients.ssm.get_paginator('get_parameters_by_path') + paginator = boto_clients.ssm.get_paginator('get_parameters_by_path') self._store_get_params_paginator = paginator return paginator diff --git a/xcon/pytest_plugin.py b/xcon/pytest_plugin.py index 25571b1..840f220 100644 --- a/xcon/pytest_plugin.py +++ b/xcon/pytest_plugin.py @@ -17,13 +17,12 @@ import pytest -from xyn_config.providers import EnvironmentalProvider -from xyn_config import Config +from xcon.providers import EnvironmentalProvider +from xcon import Config @pytest.fixture(autouse=True) -@pytest.mark.order(-1200) -def xyn_config(xyn_context): +def xcon_test_config(xinject_test_context): """ Important: This fixture should automatically be imported and auto-used as long as `xyn-config` is installed as a dependency in any project. @@ -87,12 +86,12 @@ def xyn_config(xyn_context): # (The xyn_context fixture throws always all resource objects before each test, # so configuring config with base-line values before each unit test) _setup_config_for_testing() - return Config.resource() + return Config.grab() def _setup_config_for_testing(): # Get config object from current/new context: - config = Config.resource() + config = Config.grab() # We default to ONLY use 'EnvironmentalProvider'. # We tell it not to use a cacher or parent, not testing those aspects in this test. diff --git a/xcon/types.py b/xcon/types.py new file mode 100644 index 0000000..5b9a499 --- /dev/null +++ b/xcon/types.py @@ -0,0 +1,26 @@ +from typing import Dict, Any +from xsentinels.default import DefaultType +from typing import Dict, Union, Type, TypeVar + + +T = TypeVar('T') + +JsonDict = Dict[str, Any] + +# todo: Put my `OrderedSet` class into library and use that instead. +OrderedSet = Dict[T, Type[None]] +""" +Internally we are using a dict as an ordered-set; python 3.7 guarantees dicts +keep their insertion order. So these are ordered sets of values inside a dict. +They can also have the Default value as one of their values. It's replaced by +the parent's values when resolved/used. + +.. todo:: Perhaps make an OrderedSet as a dict-subclass, with a few extra nice set-based methods? +""" + +OrderedDefaultSet = OrderedSet[Union[T, DefaultType]] +""" +Same as `OrderedSet`, except it adds in a `Default` for use as well. It's common internally +in `xynlib.config.config.Config` to use this type. Might be useful elsewhere. +The `Default` type is mean to be used to say 'insert default stuff where this placeholder is at'. +"""