Merge branch 'master' of https://github.com/StackStorm/st2 into py38_…

…u2004
StackStorm · Mar 6, 2021 · 9bd1488 · 9bd1488
2 parents d74f2a5 + 144e51b
commit 9bd1488
Show file tree

Hide file tree

Showing 984 changed files with 54,370 additions and 42,175 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -1,5 +1,5 @@
 # Setup in CircleCI account the following ENV variables:
-# PACKAGECLOUD_ORGANIZATION (default: stackstorm) 
+# PACKAGECLOUD_ORGANIZATION (default: stackstorm)
 # PACKAGECLOUD_TOKEN
 version: 2
 jobs:

diff --git a/.git-blame-ignore-rev b/.git-blame-ignore-rev
@@ -0,0 +1,5 @@
+# Code was auto formatted using black
+8496bb2407b969f0937431992172b98b545f6756
+
+# Files were auto formatted to remove trailing whitespace
+969793f1fdbdd2c228e59ab112189166530d2680
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -0,0 +1,42 @@
+# pre-commit hook which runs all the various lint checks + black auto formatting on the added
+# files.
+# This hook relies on development virtual environment being present in virtualenv/.
+default_language_version:
+  python: python3.6
+
+exclude: '(build|dist)'
+repos:
+  - repo: local
+    hooks:
+    - id: black
+      name: black
+      entry: ./virtualenv/bin/python -m black --config pyproject.toml
+      language: script
+      types: [file, python]
+  - repo: local
+    hooks:
+    - id: flake8
+      name: flake8
+      entry: ./virtualenv/bin/python -m flake8 --config lint-configs/python/.flake8
+      language: script
+      types: [file, python]
+  - repo: local
+    hooks:
+    - id: pylint
+      name: pylint
+      entry: ./virtualenv/bin/python -m pylint -E --rcfile=./lint-configs/python/.pylintrc
+      language: script
+      types: [file, python]
+  - repo: local
+    hooks:
+    - id: bandit
+      name: bandit
+      entry: ./virtualenv/bin/python -m bandit -lll -x build,dist
+      language: script
+      types: [file, python]
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v2.5.0
+    hooks:
+    - id: trailing-whitespace
+      exclude: (^conf/|^st2common/st2common/openapi.yaml|^st2client/tests/fixtures|^st2tests/st2tests/fixtures)
+    - id: check-yaml
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -4,6 +4,13 @@ Changelog
 in development
 --------------
 
+Changed
+~~~~~~~
+
+* All the code has been refactored using black and black style is automatically enforced and
+  required for all the new code. (#5156)
+
+  Contributed by @Kami.
 
 3.4.0 - March 02, 2021
 ----------------------
@@ -22,7 +29,8 @@ Added
 * Added st2-auth-ldap pip requirements for LDAP auth integartion. (new feature) #5082
   Contributed by @hnanchahal
 
-* Added --register-recreate-virtualenvs flag to st2ctl reload to recreate virtualenvs from scratch. (part of upgrade instructions) [#5167]
+* Added --register-recreate-virtualenvs flag to st2ctl reload to recreate virtualenvs from scratch.
+  (part of upgrade instructions) [#5167]
   Contributed by @winem and @blag
 
 Changed
@@ -34,7 +42,7 @@ Changed
 * Improve the st2-self-check script to echo to stderr and exit if it isn't run with a
   ST2_AUTH_TOKEN or ST2_API_KEY environment variable. (improvement) #5068
 
-* Added timeout parameter for packs.install action to help with long running installs that exceed the 
+* Added timeout parameter for packs.install action to help with long running installs that exceed the
   default timeout of 600 sec which is defined by the python_script action runner (improvement) #5084
 
   Contributed by @hnanchahal

diff --git a/Makefile b/Makefile
@@ -326,6 +326,73 @@ schemasgen: requirements .schemasgen
 	. $(VIRTUALENV_DIR)/bin/activate; pylint -j $(PYLINT_CONCURRENCY) -E --rcfile=./lint-configs/python/.pylintrc --load-plugins=pylint_plugins.api_models tools/*.py || exit 1;
 	. $(VIRTUALENV_DIR)/bin/activate; pylint -j $(PYLINT_CONCURRENCY) -E --rcfile=./lint-configs/python/.pylintrc pylint_plugins/*.py || exit 1;
 
+# Black task which checks if the code comforts to black code style
+.PHONY: black-check
+black: requirements .black-check
+
+.PHONY: .black-check
+.black-check:
+	@echo
+	@echo "================== black-check ===================="
+	@echo
+	# st2 components
+	@for component in $(COMPONENTS); do\
+		echo "==========================================================="; \
+		echo "Running black on" $$component; \
+		echo "==========================================================="; \
+		. $(VIRTUALENV_DIR)/bin/activate ; black --check --config pyproject.toml $$component/ || exit 1; \
+	done
+	# runner modules and packages
+	@for component in $(COMPONENTS_RUNNERS); do\
+		echo "==========================================================="; \
+		echo "Running black on" $$component; \
+		echo "==========================================================="; \
+		. $(VIRTUALENV_DIR)/bin/activate ; black --check --config pyproject.toml $$component/ || exit 1; \
+	done
+	. $(VIRTUALENV_DIR)/bin/activate; black --check --config pyproject.toml contrib/ || exit 1;
+	. $(VIRTUALENV_DIR)/bin/activate; black --check --config pyproject.toml scripts/*.py || exit 1;
+	. $(VIRTUALENV_DIR)/bin/activate; black --check --config pyproject.toml tools/*.py || exit 1;
+	. $(VIRTUALENV_DIR)/bin/activate; black --check --config pyproject.toml pylint_plugins/*.py || exit 1;
+
+# Black task which reformats the code using black
+.PHONY: black-format
+black: requirements .black-format
+
+.PHONY: .black-format
+.black-format:
+	@echo
+	@echo "================== black ===================="
+	@echo
+	# st2 components
+	@for component in $(COMPONENTS); do\
+		echo "==========================================================="; \
+		echo "Running black on" $$component; \
+		echo "==========================================================="; \
+		. $(VIRTUALENV_DIR)/bin/activate ; black --config pyproject.toml $$component/ || exit 1; \
+	done
+	# runner modules and packages
+	@for component in $(COMPONENTS_RUNNERS); do\
+		echo "==========================================================="; \
+		echo "Running black on" $$component; \
+		echo "==========================================================="; \
+		. $(VIRTUALENV_DIR)/bin/activate ; black --config pyproject.toml  $$component/ || exit 1; \
+	done
+	. $(VIRTUALENV_DIR)/bin/activate; black --config pyproject.toml contrib/ || exit 1;
+	. $(VIRTUALENV_DIR)/bin/activate; black --config pyproject.toml scripts/*.py || exit 1;
+	. $(VIRTUALENV_DIR)/bin/activate; black --config pyproject.toml tools/*.py || exit 1;
+	. $(VIRTUALENV_DIR)/bin/activate; black --config pyproject.toml pylint_plugins/*.py || exit 1;
+
+.PHONY: pre-commit-checks
+black: requirements .pre-commit-checks
+
+# Ensure all files contain no trailing whitespace + that all YAML files are valid.
+.PHONY: .pre-commit-checks
+.pre-commit-checks:
+	@echo
+	@echo "================== pre-commit-checks ===================="
+	@echo
+	. $(VIRTUALENV_DIR)/bin/activate; pre-commit run trailing-whitespace --all --show-diff-on-failure
+	. $(VIRTUALENV_DIR)/bin/activate; pre-commit run check-yaml --all --show-diff-on-failure
 .PHONY: lint-api-spec
 lint-api-spec: requirements .lint-api-spec
 
@@ -418,7 +485,7 @@ bandit: requirements .bandit
 lint: requirements .lint
 
 .PHONY: .lint
-.lint: .generate-api-spec .flake8 .pylint .st2client-dependencies-check .st2common-circular-dependencies-check .rst-check .st2client-install-check
+.lint: .generate-api-spec .black-check .pre-commit-checks .flake8 .pylint .st2client-dependencies-check .st2common-circular-dependencies-check .rst-check .st2client-install-check
 
 .PHONY: clean
 clean: .cleanpycs
@@ -979,7 +1046,7 @@ debs:
 ci: ci-checks ci-unit ci-integration ci-packs-tests
 
 .PHONY: ci-checks
-ci-checks: .generated-files-check .pylint .flake8 check-requirements check-sdist-requirements .st2client-dependencies-check .st2common-circular-dependencies-check circle-lint-api-spec .rst-check .st2client-install-check check-python-packages
+ci-checks: .generated-files-check .black-check .pre-commit-checks .pylint .flake8 check-requirements check-sdist-requirements .st2client-dependencies-check .st2common-circular-dependencies-check circle-lint-api-spec .rst-check .st2client-install-check check-python-packages
 
 .PHONY: .rst-check
 .rst-check:

diff --git a/OWNERS.md b/OWNERS.md
@@ -74,7 +74,7 @@ Thank you, Friends!
 * Johan Dahlberg ([@johandahlberg](https://github.com/johandahlberg)) - Using st2 for Bioinformatics/Science project, providing feedback & contributions in Ansible, Community, Workflows. [Case Study](https://stackstorm.com/case-study-scilifelab/).
 * Johan Hermansson ([@johanherman](https://github.com/johanherman)) - Using st2 for Bioinformatics/Science project, feedback & contributions in Ansible, Community, Workflows. [Case Study](https://stackstorm.com/case-study-scilifelab/).
 * Lakshmi Kannan ([@lakshmi-kannan](https://github.com/lakshmi-kannan)) - early Stormer. Initial Core platform architecture, scalability, reliability, Team Leadership during the project hard times.
-* Lindsay Hill ([@LindsayHill](https://github.com/LindsayHill)) - ex StackStorm product manager that made a significant impact building an ecosystem we see today. 
+* Lindsay Hill ([@LindsayHill](https://github.com/LindsayHill)) - ex StackStorm product manager that made a significant impact building an ecosystem we see today.
 * Manas Kelshikar ([@manasdk](https://github.com/manasdk)) - ex Stormer. Developed (well) early core platform features.
 * Vineesh Jain ([@VineeshJain](https://github.com/VineeshJain)) - ex Stormer. Community, Tests, Core, QA.
 * Warren Van Winckel ([@warrenvw](https://github.com/warrenvw)) - ex Stormer. Docker, Kubernetes, Vagrant, Infrastructure.
diff --git a/README.md b/README.md
@@ -4,12 +4,12 @@
 
 [![Build Status](https://github.com/StackStorm/st2/workflows/ci-checks/badge.svg?branch=master)](https://github.com/StackStorm/st2/actions?query=branch%3Amaster)
 [![Travis Integration Tests Status](https://travis-ci.org/StackStorm/st2.svg?branch=master)](https://travis-ci.org/StackStorm/st2)
-[![Packages Build Status](https://circleci.com/gh/StackStorm/st2/tree/master.svg?style=shield)](https://circleci.com/gh/StackStorm/st2) 
-[![Codecov](https://codecov.io/github/StackStorm/st2/badge.svg?branch=master&service=github)](https://codecov.io/github/StackStorm/st2?branch=master) 
-[![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/1833/badge)](https://bestpractices.coreinfrastructure.org/projects/1833) 
-![Python 3.6](https://img.shields.io/badge/python-3.6-blue) 
-[![Apache Licensed](https://img.shields.io/github/license/StackStorm/st2)](LICENSE) 
-[![Join our community Slack](https://img.shields.io/badge/slack-stackstorm-success.svg?logo=slack)](https://stackstorm.com/community-signup) 
+[![Packages Build Status](https://circleci.com/gh/StackStorm/st2/tree/master.svg?style=shield)](https://circleci.com/gh/StackStorm/st2)
+[![Codecov](https://codecov.io/github/StackStorm/st2/badge.svg?branch=master&service=github)](https://codecov.io/github/StackStorm/st2?branch=master)
+[![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/1833/badge)](https://bestpractices.coreinfrastructure.org/projects/1833)
+![Python 3.6](https://img.shields.io/badge/python-3.6-blue)
+[![Apache Licensed](https://img.shields.io/github/license/StackStorm/st2)](LICENSE)
+[![Join our community Slack](https://img.shields.io/badge/slack-stackstorm-success.svg?logo=slack)](https://stackstorm.com/community-signup)
 [![Forum](https://img.shields.io/discourse/https/forum.stackstorm.com/posts.svg)](https://forum.stackstorm.com/)
 
 ---

diff --git a/conf/st2.conf.sample b/conf/st2.conf.sample
@@ -2,7 +2,7 @@
 # Note: This file is automatically generated using tools/config_gen.py - DO NOT UPDATE MANUALLY
 
 [action_sensor]
-# List of execution statuses for which a trigger will be emitted. 
+# List of execution statuses for which a trigger will be emitted.
 emit_when = succeeded,failed,timeout,canceled,abandoned # comma separated list allowed here.
 # Whether to enable or disable the ability to post a trigger on action.
 enable = True
@@ -170,7 +170,7 @@ trigger_instances_ttl = None
 # Allow encryption of values in key value stored qualified as "secret".
 enable_encryption = True
 # Location of the symmetric encryption key for encrypting values in kvstore. This key should be in JSON and should've been generated using st2-generate-symmetric-crypto-key tool.
-encryption_key_path = 
+encryption_key_path =
 
 [log]
 # Exclusion list of loggers to omit.

diff --git a/contrib/chatops/actions/format_execution_result.py b/contrib/chatops/actions/format_execution_result.py
@@ -23,51 +23,50 @@
 
 class FormatResultAction(Action):
     def __init__(self, config=None, action_service=None):
-        super(FormatResultAction, self).__init__(config=config, action_service=action_service)
-        api_url = os.environ.get('ST2_ACTION_API_URL', None)
-        token = os.environ.get('ST2_ACTION_AUTH_TOKEN', None)
+        super(FormatResultAction, self).__init__(
+            config=config, action_service=action_service
+        )
+        api_url = os.environ.get("ST2_ACTION_API_URL", None)
+        token = os.environ.get("ST2_ACTION_AUTH_TOKEN", None)
         self.client = Client(api_url=api_url, token=token)
         self.jinja = jinja_utils.get_jinja_environment(allow_undefined=True)
-        self.jinja.tests['in'] = lambda item, list: item in list
+        self.jinja.tests["in"] = lambda item, list: item in list
 
         path = os.path.dirname(os.path.realpath(__file__))
-        with open(os.path.join(path, 'templates/default.j2'), 'r') as f:
+        with open(os.path.join(path, "templates/default.j2"), "r") as f:
             self.default_template = f.read()
 
     def run(self, execution_id):
         execution = self._get_execution(execution_id)
-        context = {
-            'six': six,
-            'execution': execution
-        }
+        context = {"six": six, "execution": execution}
         template = self.default_template
         result = {"enabled": True}
 
-        alias_id = execution['context'].get('action_alias_ref', {}).get('id', None)
+        alias_id = execution["context"].get("action_alias_ref", {}).get("id", None)
         if alias_id:
-            alias = self.client.managers['ActionAlias'].get_by_id(alias_id)
+            alias = self.client.managers["ActionAlias"].get_by_id(alias_id)
 
-            context.update({
-                'alias': alias
-            })
+            context.update({"alias": alias})
 
-            result_params = getattr(alias, 'result', None)
+            result_params = getattr(alias, "result", None)
             if result_params:
-                if not result_params.get('enabled', True):
+                if not result_params.get("enabled", True):
                     result["enabled"] = False
                 else:
-                    if 'format' in alias.result:
-                        template = alias.result['format']
-                    if 'extra' in alias.result:
-                        result['extra'] = jinja_utils.render_values(alias.result['extra'], context)
+                    if "format" in alias.result:
+                        template = alias.result["format"]
+                    if "extra" in alias.result:
+                        result["extra"] = jinja_utils.render_values(
+                            alias.result["extra"], context
+                        )
 
-        result['message'] = self.jinja.from_string(template).render(context)
+        result["message"] = self.jinja.from_string(template).render(context)
 
         return result
 
     def _get_execution(self, execution_id):
         if not execution_id:
-            raise ValueError('Invalid execution_id provided.')
+            raise ValueError("Invalid execution_id provided.")
         execution = self.client.liveactions.get_by_id(id=execution_id)
         if not execution:
             return None

diff --git a/contrib/chatops/actions/match.py b/contrib/chatops/actions/match.py
@@ -23,23 +23,16 @@
 class MatchAction(Action):
     def __init__(self, config=None):
         super(MatchAction, self).__init__(config=config)
-        api_url = os.environ.get('ST2_ACTION_API_URL', None)
-        token = os.environ.get('ST2_ACTION_AUTH_TOKEN', None)
+        api_url = os.environ.get("ST2_ACTION_API_URL", None)
+        token = os.environ.get("ST2_ACTION_AUTH_TOKEN", None)
         self.client = Client(api_url=api_url, token=token)
 
     def run(self, text):
         alias_match = ActionAliasMatch()
         alias_match.command = text
-        matches = self.client.managers['ActionAlias'].match(alias_match)
-        return {
-            'alias': _format_match(matches[0]),
-            'representation': matches[1]
-        }
+        matches = self.client.managers["ActionAlias"].match(alias_match)
+        return {"alias": _format_match(matches[0]), "representation": matches[1]}
 
 
 def _format_match(match):
-    return {
-        'name': match.name,
-        'pack': match.pack,
-        'action_ref': match.action_ref
-    }
+    return {"name": match.name, "pack": match.pack, "action_ref": match.action_ref}
diff --git a/contrib/chatops/actions/match_and_execute.py b/contrib/chatops/actions/match_and_execute.py
@@ -19,25 +19,26 @@
 from st2common.runners.base_action import Action
 from st2client.models.action_alias import ActionAliasMatch
 from st2client.models.aliasexecution import ActionAliasExecution
-from st2client.commands.action import (LIVEACTION_STATUS_REQUESTED,
-                                       LIVEACTION_STATUS_SCHEDULED,
-                                       LIVEACTION_STATUS_RUNNING,
-                                       LIVEACTION_STATUS_CANCELING)
+from st2client.commands.action import (
+    LIVEACTION_STATUS_REQUESTED,
+    LIVEACTION_STATUS_SCHEDULED,
+    LIVEACTION_STATUS_RUNNING,
+    LIVEACTION_STATUS_CANCELING,
+)
 from st2client.client import Client
 
 
 class ExecuteActionAliasAction(Action):
     def __init__(self, config=None):
         super(ExecuteActionAliasAction, self).__init__(config=config)
-        api_url = os.environ.get('ST2_ACTION_API_URL', None)
-        token = os.environ.get('ST2_ACTION_AUTH_TOKEN', None)
+        api_url = os.environ.get("ST2_ACTION_API_URL", None)
+        token = os.environ.get("ST2_ACTION_AUTH_TOKEN", None)
         self.client = Client(api_url=api_url, token=token)
 
     def run(self, text, source_channel=None, user=None):
         alias_match = ActionAliasMatch()
         alias_match.command = text
-        alias, representation = self.client.managers['ActionAlias'].match(
-            alias_match)
+        alias, representation = self.client.managers["ActionAlias"].match(alias_match)
 
         execution = ActionAliasExecution()
         execution.name = alias.name
@@ -48,20 +49,20 @@ def run(self, text, source_channel=None, user=None):
         execution.notification_route = None
         execution.user = user
 
-        action_exec_mgr = self.client.managers['ActionAliasExecution']
+        action_exec_mgr = self.client.managers["ActionAliasExecution"]
 
         execution = action_exec_mgr.create(execution)
-        self._wait_execution_to_finish(execution.execution['id'])
-        return execution.execution['id']
+        self._wait_execution_to_finish(execution.execution["id"])
+        return execution.execution["id"]
 
     def _wait_execution_to_finish(self, execution_id):
         pending_statuses = [
             LIVEACTION_STATUS_REQUESTED,
             LIVEACTION_STATUS_SCHEDULED,
             LIVEACTION_STATUS_RUNNING,
-            LIVEACTION_STATUS_CANCELING
+            LIVEACTION_STATUS_CANCELING,
         ]
-        action_exec_mgr = self.client.managers['LiveAction']
+        action_exec_mgr = self.client.managers["LiveAction"]
         execution = action_exec_mgr.get_by_id(execution_id)
         while execution.status in pending_statuses:
             time.sleep(1)