add bandit, fix vulnerabilities & bump pre-commit-config (#444)

* add bandit to pre-commit & bump hooks

* fix bandit vulnerabilities

* refactor: remove key `usedforsecurity`, because not all version pylint support

* fix: try for `autoescape` into `jinja2.Environment

* style: apply

* confidense -> confidence

.pre-commit-config.yaml

@@ -1,8 +1,8 @@
 default_language_version:
   python: python3
 repos:
-  - repo: 'https://github.com/pre-commit/pre-commit-hooks'
-    rev: v4.0.1
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.3.0
     hooks:
       - id: check-added-large-files
       - id: check-case-conflict
@@ -17,8 +17,8 @@ repos:
       - id: mixed-line-ending
       - id: sort-simple-yaml
       - id: trailing-whitespace
-  - repo: 'https://github.com/pycqa/flake8'
-    rev: 4.0.1
+  - repo: https://github.com/pycqa/flake8
+    rev: 5.0.4
     hooks:
       - id: flake8
         args:
@@ -28,16 +28,16 @@ repos:
           - flake8-comprehensions
           - flake8-debugger
           - flake8-string-format
-  - repo: 'https://github.com/psf/black'
-    rev: 22.3.0
+  - repo: https://github.com/psf/black
+    rev: 22.10.0
     hooks:
       - id: black
   - repo: 'https://github.com/PyCQA/isort'
     rev: 5.10.1
     hooks:
       - id: isort
-  - repo: 'https://github.com/pre-commit/mirrors-mypy'
-    rev: v0.942
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v0.982
     hooks:
       - id: mypy
         additional_dependencies:
@@ -54,7 +54,11 @@ repos:
         entry: pylint -v
         language: system
         types: [ python ]
-  # - repo: https://github.com/PyCQA/bandit
-  #   rev: '1.7.0'
-  #   hooks:
-  #     - id: bandit
+  - repo: https://github.com/PyCQA/bandit
+    rev: 1.7.4
+    hooks:
+      - id: bandit
+        exclude: tests/
+        args:
+          - -iii  # high level
+          - -lll  # high confidence

mlem/contrib/docker/context.py

@@ -95,7 +95,7 @@ def _new_whl(path):
         with tempfile.TemporaryDirectory() as whl_dir:
             subprocess.check_output(
                 f"pip wheel . --no-deps -w {whl_dir}",
-                shell=True,
+                shell=True,  # nosec: B602
                 cwd=mlem_src_path,
             )
             whl_path = glob.glob(os.path.join(whl_dir, "*.whl"))[0]

mlem/contrib/pip/base.py

@@ -116,7 +116,9 @@ def build_whl(self, path, target, target_fs):
         logger.debug("Building whl from %s...", path)
         with tempfile.TemporaryDirectory() as whl_dir:
             subprocess.check_output(
-                f"pip wheel . --no-deps -w {whl_dir}", shell=True, cwd=path
+                f"pip wheel . --no-deps -w {whl_dir}",
+                shell=True,  # nosec: B602
+                cwd=path,
             )
             whl_path = glob.glob(os.path.join(whl_dir, "*.whl"))[0]
             whl_name = os.path.basename(whl_path)

mlem/contrib/sagemaker/env_setup.py

@@ -23,7 +23,7 @@ def _tf_command(tf_dir, command, *flags, **args):
 def _tf_get_var(tf_dir, varname):
     return (
         subprocess.check_output(
-            _tf_command(tf_dir, "output", varname), shell=True
+            _tf_command(tf_dir, "output", varname), shell=True  # nosec: B602
         )
         .decode("utf8")
         .strip()
@@ -47,7 +47,10 @@ def sagemaker_terraform(
         os.path.join(os.path.dirname(__file__), MLEM_TF),
         os.path.join(work_dir, MLEM_TF),
     )
-    subprocess.check_output(_tf_command(work_dir, "init"), shell=True)
+    subprocess.check_output(
+        _tf_command(work_dir, "init"),
+        shell=True,  # nosec: B602
+    )
 
     flags = ["-auto-approve"] if not plan else []
 
@@ -62,7 +65,7 @@ def sagemaker_terraform(
                 region_name=region_name,
                 profile=profile,
             ),
-            shell=True,
+            shell=True,  # nosec: B602
         )
     )
 

mlem/core/artifacts.py

@@ -323,7 +323,7 @@ def relative(self, fs: AbstractFileSystem, path: str) -> "FSSpecArtifact":
 
 
 def md5_fileobj(fobj):
-    hash_md5 = hashlib.md5()
+    hash_md5 = hashlib.md5()  # nosec: B324
     for chunk in iter(lambda: fobj.read(CHUNK_SIZE), b""):
         hash_md5.update(chunk)
     return hash_md5.hexdigest()

mlem/core/objects.py

@@ -302,7 +302,9 @@ def update(self):
             self._write_meta(self.location)
 
     def meta_hash(self):
-        return hashlib.md5(safe_dump(self.dict()).encode("utf8")).hexdigest()
+        return hashlib.md5(  # nosec: B324
+            safe_dump(self.dict()).encode("utf8")
+        ).hexdigest()
 
 
 TL = TypeVar("TL", bound="MlemLink")

mlem/utils/templates.py

@@ -2,7 +2,12 @@
 
 from fsspec import AbstractFileSystem
 from fsspec.implementations.local import LocalFileSystem
-from jinja2 import Environment, FileSystemLoader, StrictUndefined
+from jinja2 import (
+    Environment,
+    FileSystemLoader,
+    StrictUndefined,
+    select_autoescape,
+)
 from pydantic import BaseModel
 
 
@@ -22,6 +27,7 @@ def generate(self, **additional):
         j2 = Environment(
             loader=FileSystemLoader(self.templates_dir + [self.TEMPLATE_DIR]),
             undefined=StrictUndefined,
+            autoescape=select_autoescape(),
         )
         template = j2.get_template(self.TEMPLATE_FILE)
         args = self.prepare_dict()