Merge pull request #121 from devopshq/hotfix/disable-fallback-in-recu…

…rsion

Disable recursive search by default for all verbs

.pre-commit-config.yaml

@@ -0,0 +1,14 @@
+exclude: ^tests/data
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v2.3.0
+    hooks:
+    -   id: check-yaml
+    -   id: end-of-file-fixer
+    -   id: trailing-whitespace
+
+-   repo: https://github.com/PyCQA/flake8
+    rev: 4.0.1
+    hooks:
+    -   id: flake8
+        exclude: ^tests/

LICENSE

@@ -19,4 +19,3 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
-

README.md

@@ -50,5 +50,3 @@ and manifest file with the list of packages you need to download.
 
 Configuration file format is YAML, as you could see from its filename, so you free to use yaml hints and tricks,
 as long, as main configuration parameters remains on their levels :)
-
-

crosspm/adapters/artifactoryaql.py

@@ -13,6 +13,7 @@
 from crosspm.adapters.common import BaseAdapter
 from crosspm.helpers.exceptions import *  # noqa
 from crosspm.helpers.package import Package
+from crosspm.helpers.config import CROSSPM_DEPENDENCY_LOCK_FILENAME
 
 
 CHUNK_SIZE = 1024
@@ -272,9 +273,8 @@ def get_packages(self, source, parser, downloader, list_or_file_path, property_v
                 if downloader.do_load:
                     _package.download()
                     _deps_file = _package.get_file(self._config.deps_lock_file_name)
-                    # TODO: turn on this fallback after testing
-                    # if not _deps_file:
-                    #    _deps_file = _package.get_file(CROSSPM_DEPENDENCY_LOCK_FILENAME)
+                    if not _deps_file:
+                        _deps_file = _package.get_file(CROSSPM_DEPENDENCY_LOCK_FILENAME)
                     if downloader.recursive:
                         if _deps_file:
                             _package.find_dependencies(_deps_file, property_validate=False)

crosspm/adapters/files.py

@@ -5,6 +5,7 @@
 from crosspm.adapters.common import BaseAdapter
 from crosspm.helpers.exceptions import *  # noqa
 from crosspm.helpers.package import Package
+from crosspm.helpers.config import CROSSPM_DEPENDENCY_LOCK_FILENAME
 import os
 
 CHUNK_SIZE = 1024
@@ -209,9 +210,8 @@ def get_packages(self, source, parser, downloader, list_or_file_path):
                 if downloader.do_load:
                     _package.download()
                     _deps_file = _package.get_file(self._config.deps_lock_file_name)
-                    # TODO: turn on this fallback after testing
-                    # if not _deps_file:
-                    #    _deps_file = _package.get_file(CROSSPM_DEPENDENCY_LOCK_FILENAME)
+                    if not _deps_file:
+                        _deps_file = _package.get_file(CROSSPM_DEPENDENCY_LOCK_FILENAME)
                     if _deps_file:
                         _package.find_dependencies(_deps_file)
                     elif self._config.deps_file_name:

crosspm/config.py

@@ -1 +1 @@
-__version__ = '1.0.13'
+__version__ = '1.0.14'

crosspm/cpm.py

@@ -43,6 +43,8 @@
 import sys
 import time
 
+from typing import Optional
+
 from docopt import docopt
 
 from crosspm import version
@@ -86,8 +88,8 @@ class CrossPM:
     _ready = False
 
     def __init__(self, args=None, throw_exceptions=None, return_result=False):
-        self._config = None
-        self._output = None
+        self._config: Optional[Config] = None
+        self._output: Optional[Output] = None
         self._return_result = return_result
 
         if throw_exceptions is None:
@@ -213,7 +215,8 @@ def read_config(self):
                               _deps_path, self._args['--lock-on-success'],
                               self._args['--prefer-local'],
                               deps_content=_deps_content,
-                              deps_lock_content=_depslock_content)
+                              deps_lock_content=_depslock_content,
+                              recursive=self._args['--recursive'])
         self._output = Output(self._config.output('result', None), self._config.name_column, self._config)
 
     def exit(self, code, msg):
@@ -223,20 +226,6 @@ def exit(self, code, msg):
         else:
             return code, msg
 
-    @property
-    def recursive(self):
-        if self.command_ is Downloader:
-            if self._args['--recursive'] is None:
-                recursive = True
-            else:
-                recursive = self._args['--recursive']
-        else:
-            if self._args['--recursive'] is None:
-                recursive = False
-            else:
-                recursive = self._args['--recursive']
-        return recursive
-
     @do_run
     def check_common_args(self):
         if self._args['--output']:
@@ -378,9 +367,9 @@ def command(self, command_):
         do_load = not self._args['--list']
         # hack for Locker
         if command_ is Locker:
-            do_load = self.recursive
+            do_load = self._config.recursive
 
-        cpm_ = command_(self._config, do_load, self.recursive)
+        cpm_ = command_(self._config, do_load)
         cpm_.entrypoint()
 
         if self._return_result:

crosspm/helpers/config.py

@@ -52,7 +52,7 @@ class Config:
     windows = WINDOWS
 
     def __init__(self, config_file_name='', cmdline='', no_fails=False, deps_lock_file_path='', deps_file_path='',
-                 lock_on_success=False, prefer_local=False, deps_content=None, deps_lock_content=None):
+                 lock_on_success=False, prefer_local=False, deps_content=None, deps_lock_content=None, recursive=None):
         self._log = logging.getLogger('crosspm')
         self._config_path_env = []
         self._sources = []
@@ -73,6 +73,7 @@ def __init__(self, config_file_name='', cmdline='', no_fails=False, deps_lock_fi
         self.deps_lock_file_path = ''
         self.lock_on_success = lock_on_success
         self.prefer_local = prefer_local
+        self.recursive = recursive
         self.crosspm_cache_root = ''
         self.deps_content = deps_content
         self.deps_lock_content = deps_lock_content
@@ -551,6 +552,9 @@ def init_cpm_and_cache(self, crosspm, cmdline, cache_config):
             elif 'dependencies-lock' not in crosspm:
                 self.deps_lock_file_name = self.deps_file_name + ".lock"
 
+        if self.recursive is None:
+            self.recursive = crosspm.get('recursive', False)
+
         if not self.prefer_local:
             self.prefer_local = crosspm.get('prefer-local', False)
         if not self.lock_on_success:

crosspm/helpers/downloader.py

@@ -2,6 +2,7 @@
 import logging
 import os
 from collections import OrderedDict, defaultdict
+from typing import Optional
 
 from crosspm.helpers.config import CROSSPM_DEPENDENCY_FILENAME, CROSSPM_DEPENDENCY_LOCK_FILENAME, Config  # noqa
 from crosspm.helpers.exceptions import *
@@ -15,15 +16,15 @@ def entrypoint(self, *args, **kwargs):
 
 
 class Downloader(Command):
-    def __init__(self, config, do_load, recursive):
+    def __init__(self, config: Config, do_load: bool, recursive: Optional[bool] = None):
         self._log = logging.getLogger('crosspm')
         self._config = config  # type: Config
         self.cache = config.cache
         self.solid = config.solid
         self.common_parser = Parser('common', {}, config)
         self._root_package = Package('<root>', 0, {self._config.name_column: '<root>'}, self, None,
                                      self.common_parser)
-        self.recursive = recursive
+        self.recursive = config.recursive if recursive is None else recursive
 
         self.do_load = do_load
 
@@ -40,7 +41,7 @@ def get_dependency_packages(self, list_or_file_path=None, property_validate=True
         """
         if list_or_file_path is None:
             list_or_file_path = self._config.deps_lock_file_path
-            if not os.path.isfile(list_or_file_path):
+            if not os.path.isfile(list_or_file_path) or self._config.lock_on_success:
                 list_or_file_path = self._config.deps_file_path
 
         _packages = OrderedDict()
@@ -103,7 +104,7 @@ def download_packages(self, depslock_file_path=None):
             else self._config.deps_lock_content
         if depslock_file_path is None:
             depslock_file_path = self._config.deps_lock_file_path
-        if os.path.isfile(depslock_file_path):
+        if os.path.isfile(depslock_file_path) and not self._config.lock_on_success:
             self.search_dependencies(depslock_file_path, deps_content=deps_content)
         else:
             self.search_dependencies(self._config.deps_file_path, deps_content=deps_content)

crosspm/helpers/locker.py

@@ -3,10 +3,11 @@
 from crosspm.helpers.package import Package
 from crosspm.helpers.downloader import Downloader
 from crosspm.helpers.output import Output
+from crosspm.helpers.config import Config
 
 
 class Locker(Downloader):
-    def __init__(self, config, do_load, recursive):
+    def __init__(self, config: Config, do_load: bool, recursive: Optional[bool] = None):
         # TODO: revise logic to allow recursive search without downloading
         super(Locker, self).__init__(config, do_load, recursive)
 

crosspm/helpers/output.py

@@ -8,9 +8,12 @@
 
 from jinja2 import Environment, FileSystemLoader
 from collections import OrderedDict
+from typing import Dict
 
 from crosspm.helpers.exceptions import *
 from crosspm.helpers.parser import Parser
+from crosspm.helpers.config import Config
+from crosspm.helpers.package import Package
 
 _output_format_map = {}  # Contain map for output format, load from decorator
 
@@ -89,7 +92,7 @@ class Output:
 
     def __init__(self, data=None, name_column='', config=None):
         self._log = logging.getLogger('crosspm')
-        self._config = config
+        self._config: Config = config
         if name_column:
             self._name_column = name_column
             self._output_config['key'] = name_column
@@ -199,7 +202,7 @@ def write_to_file(self, text, out_file_path):
     def get_output_types():
         return list(_output_format_map.keys())
 
-    def write_output(self, params, packages):
+    def write_output(self, params, packages: Dict[str, Package]):
         """
         Функция вызывает определенную функцию для фиксированного out-format
         :param params:
@@ -279,7 +282,14 @@ def output_format_lock(self, packages, **kwargs):
         tmp_packages = OrderedDict()
         columns = self._config.get_columns()
         widths = {}
-        for _pkg in packages.values():
+        _packages = []
+
+        for _package in packages.values():
+            if _package:
+                _packages.append(_package)
+                _packages += _package.all_packages
+
+        for _pkg in _packages:
             _pkg_name = _pkg.package_name
             _params = _pkg.get_params(columns, merged=True, raw=False)
             if _pkg_name not in tmp_packages:

crosspm/template/gus.j2

@@ -6,4 +6,3 @@ artifacts:
     version: "{{package.get_params(merged=True)['version']}}"
     src_repo_name: "{{package.pkg.repo}}"
 {% endfor %}
-

docs/FAQ.md

@@ -23,7 +23,7 @@ Frequently Asked Questions
 ## Как запретить рекурсивное выкачивание пакетов?
 <a class="mk-toclify" id="cli"></a>
 ### При вызове crosspm в командной строке
-Используйте флаг `--recursive=False` при вызове `crosspm`. **Важно** - у команд `download` и `lock` поведение по умолчанию отличается
+Используйте флаг `--recursive=False` при вызове `crosspm`.
 ```python
 # сделать lock-файл с пакетами только первого уровня, не рекурсивно
 crosspm lock # поведение по умолчанию - только первый уровень
@@ -33,13 +33,13 @@ crosspm lock --recursive False
 crosspm lock --recursive
 crosspm lock --recursive=True
 
-# скачать все дерево пакетов вместе с зависимостями
-crosspm download # поведение по умолчанию - рекурсия
-crosspm download --recursive
-crosspm download --recursive=True
 # скачать без зависимостей, только пакеты указанные в dependencies
+crosspm download # поведение по умолчанию - только первый уровень
 crosspm download --recursive=False
 crosspm download --recursive False
+# скачать все дерево пакетов вместе с зависимостями
+crosspm download --recursive
+crosspm download --recursive=True
 ```
 
 <a class="mk-toclify" id="config"></a>
@@ -215,9 +215,9 @@ crosspm usedby -c config.yaml --dependencies-content "packagename 1.2.* master"
 ```
 
 <a class="mk-toclify" id="creds"></a>
-## Как просто и безопасно передавать учётные данные в рамках crosspm. 
+## Как просто и безопасно передавать учётные данные в рамках crosspm.
 
-Для этого нужно задать переменные в конфигурационном файле. 
+Для этого нужно задать переменные в конфигурационном файле.
 
 Подробнее можно [почитать тут](usage/USAGE-CREDS.md)
 
@@ -236,5 +236,3 @@ set CROSSPM_STDOUT=1
 crosspm download # аналогично --stdout
 ```
 В конфигурационный файл задать этот параметр нельзя, т.к. логирование происходит до прочтения конфигурационного файла.
-
-