From a1f7726bd5a9a02118e0debf072ca9f14ebba596 Mon Sep 17 00:00:00 2001 From: Anton Zolotukhin Date: Thu, 20 Oct 2022 09:04:31 +0700 Subject: [PATCH 1/9] Turn off recursion by default. Turn on recursion lock-file fallback if not found --- crosspm/adapters/artifactoryaql.py | 6 +++--- crosspm/adapters/files.py | 6 +++--- crosspm/cpm.py | 12 +++--------- 3 files changed, 9 insertions(+), 15 deletions(-) diff --git a/crosspm/adapters/artifactoryaql.py b/crosspm/adapters/artifactoryaql.py index 67e739d..86f4f64 100644 --- a/crosspm/adapters/artifactoryaql.py +++ b/crosspm/adapters/artifactoryaql.py @@ -13,6 +13,7 @@ from crosspm.adapters.common import BaseAdapter from crosspm.helpers.exceptions import * # noqa from crosspm.helpers.package import Package +from crosspm.helpers.config import CROSSPM_DEPENDENCY_LOCK_FILENAME CHUNK_SIZE = 1024 @@ -272,9 +273,8 @@ def get_packages(self, source, parser, downloader, list_or_file_path, property_v if downloader.do_load: _package.download() _deps_file = _package.get_file(self._config.deps_lock_file_name) - # TODO: turn on this fallback after testing - # if not _deps_file: - # _deps_file = _package.get_file(CROSSPM_DEPENDENCY_LOCK_FILENAME) + if not _deps_file: + _deps_file = _package.get_file(CROSSPM_DEPENDENCY_LOCK_FILENAME) if downloader.recursive: if _deps_file: _package.find_dependencies(_deps_file, property_validate=False) diff --git a/crosspm/adapters/files.py b/crosspm/adapters/files.py index 4091427..0c05aad 100644 --- a/crosspm/adapters/files.py +++ b/crosspm/adapters/files.py @@ -5,6 +5,7 @@ from crosspm.adapters.common import BaseAdapter from crosspm.helpers.exceptions import * # noqa from crosspm.helpers.package import Package +from crosspm.helpers.config import CROSSPM_DEPENDENCY_LOCK_FILENAME import os CHUNK_SIZE = 1024 @@ -209,9 +210,8 @@ def get_packages(self, source, parser, downloader, list_or_file_path): if downloader.do_load: _package.download() _deps_file = _package.get_file(self._config.deps_lock_file_name) - # TODO: turn on this fallback after testing - # if not _deps_file: - # _deps_file = _package.get_file(CROSSPM_DEPENDENCY_LOCK_FILENAME) + if not _deps_file: + _deps_file = _package.get_file(CROSSPM_DEPENDENCY_LOCK_FILENAME) if _deps_file: _package.find_dependencies(_deps_file) elif self._config.deps_file_name: diff --git a/crosspm/cpm.py b/crosspm/cpm.py index 0a7f9f9..8ec8538 100644 --- a/crosspm/cpm.py +++ b/crosspm/cpm.py @@ -225,16 +225,10 @@ def exit(self, code, msg): @property def recursive(self): - if self.command_ is Downloader: - if self._args['--recursive'] is None: - recursive = True - else: - recursive = self._args['--recursive'] + if self._args['--recursive'] is None: + recursive = False else: - if self._args['--recursive'] is None: - recursive = False - else: - recursive = self._args['--recursive'] + recursive = self._args['--recursive'] return recursive @do_run From affb0938da667a257219cf80ff7599a6b4d3739b Mon Sep 17 00:00:00 2001 From: Anton Zolotukhin Date: Thu, 20 Oct 2022 09:22:58 +0700 Subject: [PATCH 2/9] Cleanup. Pre-commit --- .pre-commit-config.yaml | 12 ++++++++++++ crosspm/adapters/artifactoryaql.py | 2 +- crosspm/adapters/files.py | 2 +- requirements-dev.txt | 3 +++ setup.cfg | 2 +- 5 files changed, 18 insertions(+), 3 deletions(-) create mode 100644 .pre-commit-config.yaml create mode 100644 requirements-dev.txt diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..b339dad --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,12 @@ +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v2.3.0 + hooks: + - id: check-yaml + - id: end-of-file-fixer + - id: trailing-whitespace + +- repo: https://github.com/PyCQA/flake8 + rev: 4.0.1 + hooks: + - id: flake8 diff --git a/crosspm/adapters/artifactoryaql.py b/crosspm/adapters/artifactoryaql.py index 86f4f64..e7f51e4 100644 --- a/crosspm/adapters/artifactoryaql.py +++ b/crosspm/adapters/artifactoryaql.py @@ -274,7 +274,7 @@ def get_packages(self, source, parser, downloader, list_or_file_path, property_v _package.download() _deps_file = _package.get_file(self._config.deps_lock_file_name) if not _deps_file: - _deps_file = _package.get_file(CROSSPM_DEPENDENCY_LOCK_FILENAME) + _deps_file = _package.get_file(CROSSPM_DEPENDENCY_LOCK_FILENAME) if downloader.recursive: if _deps_file: _package.find_dependencies(_deps_file, property_validate=False) diff --git a/crosspm/adapters/files.py b/crosspm/adapters/files.py index 0c05aad..cfa6b33 100644 --- a/crosspm/adapters/files.py +++ b/crosspm/adapters/files.py @@ -211,7 +211,7 @@ def get_packages(self, source, parser, downloader, list_or_file_path): _package.download() _deps_file = _package.get_file(self._config.deps_lock_file_name) if not _deps_file: - _deps_file = _package.get_file(CROSSPM_DEPENDENCY_LOCK_FILENAME) + _deps_file = _package.get_file(CROSSPM_DEPENDENCY_LOCK_FILENAME) if _deps_file: _package.find_dependencies(_deps_file) elif self._config.deps_file_name: diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..5c549e8 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,3 @@ +-r requirements.txt +-r requirements-test.txt +pre-commit diff --git a/setup.cfg b/setup.cfg index 9399f42..301fcf2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -3,5 +3,5 @@ test=pytest [flake8] max-line-length = 120 -exclude = tests +exclude = tests,.eggs ignore = F405,F403 From f9a007bfec9a9a3739ae5aa6867bcbc7950a5802 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 20 Oct 2022 02:31:44 +0000 Subject: [PATCH 3/9] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- LICENSE | 1 - README.md | 2 -- crosspm/template/gus.j2 | 1 - docs/FAQ.md | 6 ++--- docs/config/CONFIG.md | 32 ++++++++++++------------ docs/config/IMPORT.md | 8 +++--- docs/config/OUTPUT.md | 2 +- docs/cpmconfig.md | 2 +- docs/usage/USAGE-CMAKE.md | 1 - docs/usage/USAGE-CREDS.md | 40 +++++++++++++++--------------- docs/usage/USAGE-PYTHON.md | 5 ++-- docs/usage/USAGE-USEDBY.md | 1 - docs/usage/USAGE.md | 2 +- tests/data/config_en_cp1251.yaml | 2 +- tests/data/config_en_utf8_bom.yaml | 2 +- tests/data/config_ru_utf8_bom.yaml | 2 +- tests/test_cpm.py | 1 - tests/test_parser.py | 4 +-- 18 files changed, 52 insertions(+), 62 deletions(-) diff --git a/LICENSE b/LICENSE index 60a229f..63b9140 100644 --- a/LICENSE +++ b/LICENSE @@ -19,4 +19,3 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - diff --git a/README.md b/README.md index 55e8dbe..b247f03 100644 --- a/README.md +++ b/README.md @@ -50,5 +50,3 @@ and manifest file with the list of packages you need to download. Configuration file format is YAML, as you could see from its filename, so you free to use yaml hints and tricks, as long, as main configuration parameters remains on their levels :) - - diff --git a/crosspm/template/gus.j2 b/crosspm/template/gus.j2 index ed6a11d..67d7962 100644 --- a/crosspm/template/gus.j2 +++ b/crosspm/template/gus.j2 @@ -6,4 +6,3 @@ artifacts: version: "{{package.get_params(merged=True)['version']}}" src_repo_name: "{{package.pkg.repo}}" {% endfor %} - diff --git a/docs/FAQ.md b/docs/FAQ.md index 7b2a4b3..8fb3b8e 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -215,9 +215,9 @@ crosspm usedby -c config.yaml --dependencies-content "packagename 1.2.* master" ``` -## Как просто и безопасно передавать учётные данные в рамках crosspm. +## Как просто и безопасно передавать учётные данные в рамках crosspm. -Для этого нужно задать переменные в конфигурационном файле. +Для этого нужно задать переменные в конфигурационном файле. Подробнее можно [почитать тут](usage/USAGE-CREDS.md) @@ -236,5 +236,3 @@ set CROSSPM_STDOUT=1 crosspm download # аналогично --stdout ``` В конфигурационный файл задать этот параметр нельзя, т.к. логирование происходит до прочтения конфигурационного файла. - - diff --git a/docs/config/CONFIG.md b/docs/config/CONFIG.md index b33e142..c5ca596 100644 --- a/docs/config/CONFIG.md +++ b/docs/config/CONFIG.md @@ -44,25 +44,25 @@ Main configuration such as manifest file name and cache path. cpm: # Short description of your configuration file description: Simple example configuration - + dependencies: dependencies.txt dependencies-lock: dependencies.txt.lock lock-on-success: true - prefer-local: True + prefer-local: True ``` ## `cpm:dependencies` and `cpm:dependencies-lock` - `cpm:dependencies` - manifest file name (not path - just filename) -- `cpm:dependencies-lock` - manifest with locked dependencies (without masks and conditions) file name (not path - just filename). +- `cpm:dependencies-lock` - manifest with locked dependencies (without masks and conditions) file name (not path - just filename). Equals to "dependencies" if not set. ## `cpm:lock-on-success` -If set to `true` (or yes or 1) - dependencies lock file will be generated after successfull **download**. -Lock file will be saved near original dependencies file. +If set to `true` (or yes or 1) - dependencies lock file will be generated after successfull **download**. +Lock file will be saved near original dependencies file. ## `cpm: prefer-local` -Search in local cache before query repo. +Search in local cache before query repo. Work only with: - FIXED package version, without mask - FIXED extenstion @@ -91,7 +91,7 @@ Several way to set where crosspm stored files (archive and unpacked package) ## cache:storage Local storage setting, how `crosspm` will be stored files: - `storage:packed` - path to packed file -- `storage:unpacked` - path to unpacked file +- `storage:unpacked` - path to unpacked file ## `cache:clear` - NOT used @@ -233,19 +233,19 @@ Means that version column contains three numeric parts divided by a dot, followe ```yaml version: "{int}.{int}.{int}[.{int}][-{str}]" ``` -- `sort` - List of column names in sorting order. Used for sorting packages if more than one version found +- `sort` - List of column names in sorting order. Used for sorting packages if more than one version found for defined parameters. Asterisk can be one of values of a list representing all columns not mentioned here. - `index` - Used for picking one element from sorted list. It's just a list index as in python. - `properties` - Extra properties. i.e. object properties in Artifactory. ## `parsers:path` -Path template for searching packages in repository. Here {} is column, [|] is variation. +Path template for searching packages in repository. Here {} is column, [|] is variation. ```yaml path: "{server}/{repo}/{package}/{compiler|any}/{osname}/{package}.{version}[.zip|.tar.gz] ``` these paths will be searched: - + ```yaml https://repo.some.org/artifactory/libs-release.snapshot/boost/gcc4/linux/boost.1.60.204.zip https://repo.some.org/artifactory/libs-release.snapshot/boost/gcc4/linux/boost.1.60.204.tar.gz @@ -271,7 +271,7 @@ parsers: artifactory-nupkg: path: '{server}/{repo}/pool/*/{package}.{version}.nupkg' ``` - + # `defaults` Default values for columns not defined in "options". @@ -364,15 +364,15 @@ cpm: dependencies: dependencies.txt dependencies-lock: dependencies.txt.lock lock-on-success: true - - # search in local cache before query repo. + + # search in local cache before query repo. # Work only with: # - FIXED package version, without mask # - FIXED extenstion # - download-mode only (in lock it does not work) # - only in artifactory-aql adapters - prefer-local: True - + prefer-local: True + cache: cmdline: cache env: CROSSPM_CACHE_ROOT @@ -427,7 +427,7 @@ parsers: artifactory: path: "{server}/{repo}/{package}/{branch}/{version}/{compiler|any}/{arch|any}/{osname}/{package}.{version}[.zip|.tar.gz|.nupkg]" properties: "some.org.quality = {quality}" - + defaults: branch: master quality: stable diff --git a/docs/config/IMPORT.md b/docs/config/IMPORT.md index 102b69d..646f671 100644 --- a/docs/config/IMPORT.md +++ b/docs/config/IMPORT.md @@ -52,7 +52,7 @@ sources: # dependencies: dependencies # dependencies-lock: dependencies # prefer-local: True - + # -------------- # Тут определён файл весь конфигурации, кроме секции cpm # Так же допустимо использовать yaml-anchor для чувствительной информации @@ -63,7 +63,7 @@ sources: import: - cred.yaml - crosspm.main.yaml - + cpm: dependencies: dependencies.txt dependencies-lock: dependencies.txt.lock @@ -75,7 +75,7 @@ cpm: import: - cred.yaml - crosspm.main.yaml - + cpm: dependencies: no-dependencies.txt dependencies-lock: no-dependencies.txt.lock @@ -87,7 +87,7 @@ cpm: crosspm download ^ --options cl="vc140",arch="x86_64",os="win" ^ --config=".\crosspm.dependencies.yaml" - + crosspm download ^ --options cl="vc140",arch="x86_64",os="win" ^ --config=".\crosspm.no-dependencies.yaml" diff --git a/docs/config/OUTPUT.md b/docs/config/OUTPUT.md index 27cd1c3..c54b8bc 100644 --- a/docs/config/OUTPUT.md +++ b/docs/config/OUTPUT.md @@ -82,7 +82,7 @@ cat ./paths.sh } } ``` - + ## python [Лучшая практика использования в python](../usage/USAGE-PYTHON) diff --git a/docs/cpmconfig.md b/docs/cpmconfig.md index 2578ea7..e71d406 100644 --- a/docs/cpmconfig.md +++ b/docs/cpmconfig.md @@ -1 +1 @@ -В разработке... \ No newline at end of file +В разработке... diff --git a/docs/usage/USAGE-CMAKE.md b/docs/usage/USAGE-CMAKE.md index 78a7fbe..f50d248 100644 --- a/docs/usage/USAGE-CMAKE.md +++ b/docs/usage/USAGE-CMAKE.md @@ -50,4 +50,3 @@ target_link_libraries( ZLIB ) ``` - diff --git a/docs/usage/USAGE-CREDS.md b/docs/usage/USAGE-CREDS.md index a17ea1c..f5507c7 100644 --- a/docs/usage/USAGE-CREDS.md +++ b/docs/usage/USAGE-CREDS.md @@ -10,16 +10,16 @@ options: <... пропущен ненужный блок ...> # можно указать все блоки, а использовать или auth или user, password - auth: + auth: cmdline: auth_libs1 env: DOWNLOAD_CI_AUTH secret: true - + user: cmdline: user env: DOWNLOAD_CI_USER secret: false - + password: cmdline: password env: DOWNLOAD_CI_PASSWORD @@ -33,11 +33,11 @@ options: - `secret` - отображать или нет в лог значение переменной. Если поставить значение `true`, то параметр не будет выводиться в лог. Для примера выше значения `auth` и `password` не будут видны в логе, а значение `user` будет: ``` -boost: {'version': ['*', '*', '*', None, None], 'branch': '1.64-pm-icu', 'user': 'env_test_user', -'compiler': 'vc110', 'arch': 'x86', 'osname': 'win', 'user1': None, 'user2': None, +boost: {'version': ['*', '*', '*', None, None], 'branch': '1.64-pm-icu', 'user': 'env_test_user', +'compiler': 'vc110', 'arch': 'x86', 'osname': 'win', 'user1': None, 'user2': None, 'server': 'https://repo.example.com/artifactory'} ``` - + После того как переменные для авторизации указаны в разделе `options` можно использовать их в разделах `common` или `sourses`. Переменные задаются по имени в скобках `'{имяпеременной}'`. Варианты того как можно задать переменные: @@ -48,10 +48,10 @@ common: type: jfrog-artifactory-aql auth_type: simple auth: '{auth}' - + sources: - repo: - - some-repo.snapshot + - some-repo.snapshot ``` ```yaml @@ -60,49 +60,49 @@ common: parser: artifactory type: jfrog-artifactory-aql auth_type: simple - + sources: - repo: - another-repo.snapshot auth: '{user}:{password}' -``` - +``` + ```yaml common: server: https://repo.example.com/artifactory parser: artifactory type: jfrog-artifactory-aql auth_type: simple - + sources: - repo: - first-repo.snapshot - auth: + auth: - '{user1}' - - '{password1}' + - '{password1}' - repo: - second-repo.snapshot - auth: + auth: - '{user2}:{password2}' - auth: + auth: - 'user:{password}' - repo: - third-repo.snapshot - auth: + auth: - '{auth}' ``` ## Примеры использования. - Можно задать системные переменные, указанные в конфигурационном файле. Например: - + `export DOWNLOAD_CI_AUTH='user:password'` - + `export DOWNLOAD_CI_USER='user'` `export DOWNLOAD_CI_PASSWORD='password'` - Можно задать в командной строке. Например: - + `crosspm download -o user=user,password=password` или `crosspm download -o auth_libs1=user:password` diff --git a/docs/usage/USAGE-PYTHON.md b/docs/usage/USAGE-PYTHON.md index 26b6c4c..b8ca116 100644 --- a/docs/usage/USAGE-PYTHON.md +++ b/docs/usage/USAGE-PYTHON.md @@ -65,7 +65,7 @@ NB: version is list "osname": "win", "package": "libiconv", "repo": "libs-cpp-release", - "version": ["1", "9", "131", "feature"] + "version": ["1", "9", "131", "feature"] } """ @@ -100,7 +100,7 @@ NB: version is list "version": "1.9.131-feature", } """ - + ``` ### Logging @@ -117,4 +117,3 @@ err, crosspm_packages = cpm.run() ``` **NOTE**: If you set your logging setting and not suppressing `crosspm`-logging, you will have two logs, one in `stderr`, other in `stdout` - diff --git a/docs/usage/USAGE-USEDBY.md b/docs/usage/USAGE-USEDBY.md index 611f499..cb421aa 100644 --- a/docs/usage/USAGE-USEDBY.md +++ b/docs/usage/USAGE-USEDBY.md @@ -113,4 +113,3 @@ Dependencies tree: - package3 3.0.3 - package1 1.0.1 ``` - diff --git a/docs/usage/USAGE.md b/docs/usage/USAGE.md index 10e1b4b..cde6e50 100644 --- a/docs/usage/USAGE.md +++ b/docs/usage/USAGE.md @@ -7,7 +7,7 @@ Usage ======= -Идея CrossPM (как пакетного менеджера) примерно такая же, как и других популярных пакетных менеджеров (npm, pyenv, etc). +Идея CrossPM (как пакетного менеджера) примерно такая же, как и других популярных пакетных менеджеров (npm, pyenv, etc). Можно придерживаться примерно следующего алгоритма: 1. Создаете файл **dependencies.txt** - список пакетов, которые нужно скачать. Поддерживаются маски в версиях, свойства артефактов. diff --git a/tests/data/config_en_cp1251.yaml b/tests/data/config_en_cp1251.yaml index ee6ad09..dcf894c 100644 --- a/tests/data/config_en_cp1251.yaml +++ b/tests/data/config_en_cp1251.yaml @@ -1,2 +1,2 @@ cpm: - description: English description \ No newline at end of file + description: English description diff --git a/tests/data/config_en_utf8_bom.yaml b/tests/data/config_en_utf8_bom.yaml index 3af6205..d155d78 100644 --- a/tests/data/config_en_utf8_bom.yaml +++ b/tests/data/config_en_utf8_bom.yaml @@ -1,3 +1,3 @@ # Comment cpm: - description: My config for english test \ No newline at end of file + description: My config for english test diff --git a/tests/data/config_ru_utf8_bom.yaml b/tests/data/config_ru_utf8_bom.yaml index a1caae0..f8f9dfd 100644 --- a/tests/data/config_ru_utf8_bom.yaml +++ b/tests/data/config_ru_utf8_bom.yaml @@ -1,3 +1,3 @@ # Комментарий cpm: - description: Описание на русском \ No newline at end of file + description: Описание на русском diff --git a/tests/test_cpm.py b/tests/test_cpm.py index 66ac4e3..8c1d773 100644 --- a/tests/test_cpm.py +++ b/tests/test_cpm.py @@ -133,4 +133,3 @@ def test_stdout_flag(): os.environ['CROSSPM_STDOUT'] = '1' cpm = CrossPM("download") assert cpm.stdout is True - diff --git a/tests/test_parser.py b/tests/test_parser.py index 96f2636..31b8e7f 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -40,7 +40,7 @@ class TestParser(BaseParserTest): - version - '*' index: -1 - + artifactory: path: "{server}/{repo}/{package}/{branch}/{version}/{compiler|any}/{arch|any}/{osname}/{package}.{version}[.zip|.tar.gz|.nupkg]" @@ -56,7 +56,7 @@ class TestParser(BaseParserTest): "deb.name": "package" "deb.version": "version" "qaverdict": "qaverdict" - + path-parser: 'https://repo.example.com/artifactory/libs-cpp-release.snapshot/(?P.*?)/(?P.*?)/(?P.*?)/(?P.*?)/(?P.*?)/(?P.*?)/.*.tar.gz' """ From 0fd3cb721e359a98b291b11b12aae2e57e428a48 Mon Sep 17 00:00:00 2001 From: Anton Zolotukhin Date: Thu, 20 Oct 2022 17:05:18 +0700 Subject: [PATCH 4/9] Add `recursive` config param. Save found by recursion packages in lock file. Tests for config `deps*_path` and `recursive` logic --- crosspm/cpm.py | 21 +++++------- crosspm/helpers/config.py | 6 +++- crosspm/helpers/downloader.py | 9 +++--- crosspm/helpers/locker.py | 3 +- crosspm/helpers/output.py | 16 ++++++++-- docs/FAQ.md | 10 +++--- docs/config/CONFIG.md | 4 +++ tests/data/config_deps.yaml | 16 ++++++++++ tests/data/config_deps_depslock.yaml | 17 ++++++++++ tests/data/config_depslock.yaml | 16 ++++++++++ tests/data/config_recursive_off.yaml | 16 ++++++++++ tests/data/config_recursive_on.yaml | 16 ++++++++++ tests/data/config_stub.yaml | 16 ++++++++++ tests/test_config.py | 40 +++++++++++++++++++++++ tests/test_cpm.py | 48 ---------------------------- 15 files changed, 179 insertions(+), 75 deletions(-) create mode 100644 tests/data/config_deps.yaml create mode 100644 tests/data/config_deps_depslock.yaml create mode 100644 tests/data/config_depslock.yaml create mode 100644 tests/data/config_recursive_off.yaml create mode 100644 tests/data/config_recursive_on.yaml create mode 100644 tests/data/config_stub.yaml diff --git a/crosspm/cpm.py b/crosspm/cpm.py index 8ec8538..d9c8d08 100644 --- a/crosspm/cpm.py +++ b/crosspm/cpm.py @@ -43,6 +43,8 @@ import sys import time +from typing import Optional + from docopt import docopt from crosspm import version @@ -86,8 +88,8 @@ class CrossPM: _ready = False def __init__(self, args=None, throw_exceptions=None, return_result=False): - self._config = None - self._output = None + self._config: Optional[Config] = None + self._output: Optional[Output] = None self._return_result = return_result if throw_exceptions is None: @@ -213,7 +215,8 @@ def read_config(self): _deps_path, self._args['--lock-on-success'], self._args['--prefer-local'], deps_content=_deps_content, - deps_lock_content=_depslock_content) + deps_lock_content=_depslock_content, + recursive=self._args['--recursive']) self._output = Output(self._config.output('result', None), self._config.name_column, self._config) def exit(self, code, msg): @@ -223,14 +226,6 @@ def exit(self, code, msg): else: return code, msg - @property - def recursive(self): - if self._args['--recursive'] is None: - recursive = False - else: - recursive = self._args['--recursive'] - return recursive - @do_run def check_common_args(self): if self._args['--output']: @@ -372,9 +367,9 @@ def command(self, command_): do_load = not self._args['--list'] # hack for Locker if command_ is Locker: - do_load = self.recursive + do_load = self._config.recursive - cpm_ = command_(self._config, do_load, self.recursive) + cpm_ = command_(self._config, do_load) cpm_.entrypoint() if self._return_result: diff --git a/crosspm/helpers/config.py b/crosspm/helpers/config.py index 55e0e43..8e5150f 100644 --- a/crosspm/helpers/config.py +++ b/crosspm/helpers/config.py @@ -52,7 +52,7 @@ class Config: windows = WINDOWS def __init__(self, config_file_name='', cmdline='', no_fails=False, deps_lock_file_path='', deps_file_path='', - lock_on_success=False, prefer_local=False, deps_content=None, deps_lock_content=None): + lock_on_success=False, prefer_local=False, deps_content=None, deps_lock_content=None, recursive=None): self._log = logging.getLogger('crosspm') self._config_path_env = [] self._sources = [] @@ -73,6 +73,7 @@ def __init__(self, config_file_name='', cmdline='', no_fails=False, deps_lock_fi self.deps_lock_file_path = '' self.lock_on_success = lock_on_success self.prefer_local = prefer_local + self.recursive = recursive self.crosspm_cache_root = '' self.deps_content = deps_content self.deps_lock_content = deps_lock_content @@ -551,6 +552,9 @@ def init_cpm_and_cache(self, crosspm, cmdline, cache_config): elif 'dependencies-lock' not in crosspm: self.deps_lock_file_name = self.deps_file_name + ".lock" + if self.recursive is None: + self.recursive = crosspm.get('recursive', False) + if not self.prefer_local: self.prefer_local = crosspm.get('prefer-local', False) if not self.lock_on_success: diff --git a/crosspm/helpers/downloader.py b/crosspm/helpers/downloader.py index f022631..ca7d5a7 100644 --- a/crosspm/helpers/downloader.py +++ b/crosspm/helpers/downloader.py @@ -2,6 +2,7 @@ import logging import os from collections import OrderedDict, defaultdict +from typing import Optional from crosspm.helpers.config import CROSSPM_DEPENDENCY_FILENAME, CROSSPM_DEPENDENCY_LOCK_FILENAME, Config # noqa from crosspm.helpers.exceptions import * @@ -15,7 +16,7 @@ def entrypoint(self, *args, **kwargs): class Downloader(Command): - def __init__(self, config, do_load, recursive): + def __init__(self, config: Config, do_load: bool, recursive: Optional[bool] = None): self._log = logging.getLogger('crosspm') self._config = config # type: Config self.cache = config.cache @@ -23,7 +24,7 @@ def __init__(self, config, do_load, recursive): self.common_parser = Parser('common', {}, config) self._root_package = Package('', 0, {self._config.name_column: ''}, self, None, self.common_parser) - self.recursive = recursive + self.recursive = config.recursive if recursive is None else recursive self.do_load = do_load @@ -40,7 +41,7 @@ def get_dependency_packages(self, list_or_file_path=None, property_validate=True """ if list_or_file_path is None: list_or_file_path = self._config.deps_lock_file_path - if not os.path.isfile(list_or_file_path): + if not os.path.isfile(list_or_file_path) or self._config.lock_on_success: list_or_file_path = self._config.deps_file_path _packages = OrderedDict() @@ -103,7 +104,7 @@ def download_packages(self, depslock_file_path=None): else self._config.deps_lock_content if depslock_file_path is None: depslock_file_path = self._config.deps_lock_file_path - if os.path.isfile(depslock_file_path): + if os.path.isfile(depslock_file_path) and not self._config.lock_on_success: self.search_dependencies(depslock_file_path, deps_content=deps_content) else: self.search_dependencies(self._config.deps_file_path, deps_content=deps_content) diff --git a/crosspm/helpers/locker.py b/crosspm/helpers/locker.py index a2af696..ede54c2 100644 --- a/crosspm/helpers/locker.py +++ b/crosspm/helpers/locker.py @@ -3,10 +3,11 @@ from crosspm.helpers.package import Package from crosspm.helpers.downloader import Downloader from crosspm.helpers.output import Output +from crosspm.helpers.config import Config class Locker(Downloader): - def __init__(self, config, do_load, recursive): + def __init__(self, config: Config, do_load: bool, recursive: Optional[bool] = None): # TODO: revise logic to allow recursive search without downloading super(Locker, self).__init__(config, do_load, recursive) diff --git a/crosspm/helpers/output.py b/crosspm/helpers/output.py index fb18165..09e0ab0 100644 --- a/crosspm/helpers/output.py +++ b/crosspm/helpers/output.py @@ -8,9 +8,12 @@ from jinja2 import Environment, FileSystemLoader from collections import OrderedDict +from typing import Dict from crosspm.helpers.exceptions import * from crosspm.helpers.parser import Parser +from crosspm.helpers.config import Config +from crosspm.helpers.package import Package _output_format_map = {} # Contain map for output format, load from decorator @@ -89,7 +92,7 @@ class Output: def __init__(self, data=None, name_column='', config=None): self._log = logging.getLogger('crosspm') - self._config = config + self._config: Config = config if name_column: self._name_column = name_column self._output_config['key'] = name_column @@ -199,7 +202,7 @@ def write_to_file(self, text, out_file_path): def get_output_types(): return list(_output_format_map.keys()) - def write_output(self, params, packages): + def write_output(self, params, packages: Dict[str, Package]): """ Функция вызывает определенную функцию для фиксированного out-format :param params: @@ -279,7 +282,14 @@ def output_format_lock(self, packages, **kwargs): tmp_packages = OrderedDict() columns = self._config.get_columns() widths = {} - for _pkg in packages.values(): + _packages = [] + + for _package in packages.values(): + if _package: + _packages.append(_package) + _packages += _package.all_packages + + for _pkg in _packages: _pkg_name = _pkg.package_name _params = _pkg.get_params(columns, merged=True, raw=False) if _pkg_name not in tmp_packages: diff --git a/docs/FAQ.md b/docs/FAQ.md index 8fb3b8e..3d66a13 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -23,7 +23,7 @@ Frequently Asked Questions ## Как запретить рекурсивное выкачивание пакетов? ### При вызове crosspm в командной строке -Используйте флаг `--recursive=False` при вызове `crosspm`. **Важно** - у команд `download` и `lock` поведение по умолчанию отличается +Используйте флаг `--recursive=False` при вызове `crosspm`. ```python # сделать lock-файл с пакетами только первого уровня, не рекурсивно crosspm lock # поведение по умолчанию - только первый уровень @@ -34,12 +34,12 @@ crosspm lock --recursive crosspm lock --recursive=True # скачать все дерево пакетов вместе с зависимостями -crosspm download # поведение по умолчанию - рекурсия -crosspm download --recursive -crosspm download --recursive=True -# скачать без зависимостей, только пакеты указанные в dependencies +crosspm download # поведение по умолчанию - только первый уровень crosspm download --recursive=False crosspm download --recursive False +# скачать без зависимостей, только пакеты указанные в dependencies +crosspm download --recursive +crosspm download --recursive=True ``` diff --git a/docs/config/CONFIG.md b/docs/config/CONFIG.md index c5ca596..d54fdc6 100644 --- a/docs/config/CONFIG.md +++ b/docs/config/CONFIG.md @@ -8,6 +8,7 @@ CrossPM Config * [cpm:dependencies and cpm:dependencies-lock](#cpmdependencies-and-cpmdependencies-lock) * [cpm:lock-on-success](#cpmlock-on-success) * [cpm: prefer-local](#cpm-prefer-local) + * [cpm: recursive](#cpm-recursive) * [cache](#cache) * [cache:storage](#cachestorage) * [cache:clear - NOT used](#cacheclear---not-used) @@ -69,6 +70,9 @@ Work only with: - download-mode only (in lock it does not work) - only in artifactory-aql adapters +## `cpm: recursive` +If set to `true` (or yes or 1) - crosspm will process all packages recursively to find and lock all dependencies. + # `cache` Parameters for cache handling diff --git a/tests/data/config_deps.yaml b/tests/data/config_deps.yaml new file mode 100644 index 0000000..720f1b1 --- /dev/null +++ b/tests/data/config_deps.yaml @@ -0,0 +1,16 @@ +cpm: + dependencies: test1_dependencies.txt + +common: + server: https://stub/artifactory + parser: artifactory + type: jfrog-artifactory-aql + +parsers: + artifactory: + path: '{server}/{repo}/{package}.{version}[.tar.gz|.zip]' + properties: '' + +sources: + - repo: + - stub diff --git a/tests/data/config_deps_depslock.yaml b/tests/data/config_deps_depslock.yaml new file mode 100644 index 0000000..12a3515 --- /dev/null +++ b/tests/data/config_deps_depslock.yaml @@ -0,0 +1,17 @@ +cpm: + dependencies: test3_dependencies.txt + dependencies-lock: test4_dependencies.txt.lock + +common: + server: https://stub/artifactory + parser: artifactory + type: jfrog-artifactory-aql + +parsers: + artifactory: + path: '{server}/{repo}/{package}.{version}[.tar.gz|.zip]' + properties: '' + +sources: + - repo: + - stub diff --git a/tests/data/config_depslock.yaml b/tests/data/config_depslock.yaml new file mode 100644 index 0000000..d917a8d --- /dev/null +++ b/tests/data/config_depslock.yaml @@ -0,0 +1,16 @@ +cpm: + dependencies-lock: test2_dependencies.txt.lock + +common: + server: https://stub/artifactory + parser: artifactory + type: jfrog-artifactory-aql + +parsers: + artifactory: + path: '{server}/{repo}/{package}.{version}[.tar.gz|.zip]' + properties: '' + +sources: + - repo: + - stub diff --git a/tests/data/config_recursive_off.yaml b/tests/data/config_recursive_off.yaml new file mode 100644 index 0000000..2c4adbe --- /dev/null +++ b/tests/data/config_recursive_off.yaml @@ -0,0 +1,16 @@ +cpm: + recursive: false + +common: + server: https://stub/artifactory + parser: artifactory + type: jfrog-artifactory-aql + +parsers: + artifactory: + path: '{server}/{repo}/{package}.{version}[.tar.gz|.zip]' + properties: '' + +sources: + - repo: + - stub diff --git a/tests/data/config_recursive_on.yaml b/tests/data/config_recursive_on.yaml new file mode 100644 index 0000000..2e6b58c --- /dev/null +++ b/tests/data/config_recursive_on.yaml @@ -0,0 +1,16 @@ +cpm: + recursive: true + +common: + server: https://stub/artifactory + parser: artifactory + type: jfrog-artifactory-aql + +parsers: + artifactory: + path: '{server}/{repo}/{package}.{version}[.tar.gz|.zip]' + properties: '' + +sources: + - repo: + - stub diff --git a/tests/data/config_stub.yaml b/tests/data/config_stub.yaml new file mode 100644 index 0000000..73ed076 --- /dev/null +++ b/tests/data/config_stub.yaml @@ -0,0 +1,16 @@ +cpm: + description: test stub + +common: + server: https://stub/artifactory + parser: artifactory + type: jfrog-artifactory-aql + +parsers: + artifactory: + path: '{server}/{repo}/{package}.{version}[.tar.gz|.zip]' + properties: '' + +sources: + - repo: + - stub diff --git a/tests/test_config.py b/tests/test_config.py index 3892513..e7668bb 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -27,3 +27,43 @@ def test_config_comment_before_import_handle(filename, monkeypatch): monkeypatch.chdir('{}/data'.format(Path(__file__).parent)) result = Config.load_yaml(str(DATA / '{}.yaml'.format(filename))) assert '.aliases' in result + + +@pytest.mark.parametrize("filename, cli_param, result", [ + ('config_recursive_on', None, True), + ('config_recursive_off', None, False), + ('config_recursive_on', False, False), + ('config_recursive_off', False, False), + ('config_recursive_on', True, True), + ('config_recursive_off', True, True), + ('config_stub', None, False), + ('config_stub', True, True), + ('config_stub', False, False), +]) +def test_config_recursive(filename, cli_param, result): + config = Config(config_file_name=str(DATA / f'{filename}.yaml'), recursive=cli_param) + assert config.recursive == result + + +@pytest.mark.parametrize("filename, deps, deps_lock, result_deps, result_deps_lock", [ + ('config_deps', "", "", "test1_dependencies.txt", "test1_dependencies.txt.lock"), + ('config_depslock', "", "", "dependencies.txt", "test2_dependencies.txt.lock"), + ('config_deps_depslock', "", "", "test3_dependencies.txt", "test4_dependencies.txt.lock"), + ('config_stub', "", "", "dependencies.txt", "dependencies.txt.lock"), + ('config_deps', "derps.txt", "", "derps.txt", "derps.txt.lock"), + ('config_depslock', "derps.txt", "", "derps.txt", "derps.txt.lock"), + ('config_deps_depslock', "derps.txt", "", "derps.txt", "derps.txt.lock"), + ('config_stub', "derps.txt", "", "derps.txt", "derps.txt.lock"), + ('config_deps', "", "derps.txt.lock", "test1_dependencies.txt", "derps.txt.lock"), + ('config_depslock', "", "derps.txt.lock", "dependencies.txt", "derps.txt.lock"), + ('config_deps_depslock', "", "derps.txt.lock", "test3_dependencies.txt", "derps.txt.lock"), + ('config_stub', "derps.txt", "derps.txt.lock", "derps.txt", "derps.txt.lock"), + ('config_deps', "derps.txt", "derps.txt.lock", "derps.txt", "derps.txt.lock"), + ('config_depslock', "derps.txt", "derps.txt.lock", "derps.txt", "derps.txt.lock"), + ('config_deps_depslock', "derps.txt", "derps.txt.lock", "derps.txt", "derps.txt.lock"), + ('config_stub', "derps.txt", "derps.txt.lock", "derps.txt", "derps.txt.lock"), +]) +def test_config_deps_depslock(filename, deps, deps_lock, result_deps, result_deps_lock): + config = Config(config_file_name=str(DATA / f'{filename}.yaml'), deps_file_path=deps, deps_lock_file_path=deps_lock) + assert config.deps_file_name == result_deps + assert config.deps_lock_file_name == result_deps_lock diff --git a/tests/test_cpm.py b/tests/test_cpm.py index 8c1d773..80ceb52 100644 --- a/tests/test_cpm.py +++ b/tests/test_cpm.py @@ -66,54 +66,6 @@ def test_prepare_args_linux(expect, raw): assert CrossPM.prepare_args(raw, windows=False) == expect, "Error when pass as string, on Linux-system" -# -# DOWNLOAD -# - -def test_download_recursive_default(): - cpm = CrossPM("download") - assert cpm.recursive is True - - -def test_download_recursive(): - cpm = CrossPM("download --recursive") - assert cpm.recursive is True - - -def test_download_recursive_false(): - cpm = CrossPM("download --recursive False") - assert cpm.recursive is False - - -def test_download_recursive_true(): - cpm = CrossPM("download --recursive True") - assert cpm.recursive is True - - -# -# LOCK -# - -def test_lock_recursive_default(): - cpm = CrossPM("lock") - assert cpm.recursive is False - - -def test_lock_recursive(): - cpm = CrossPM("lock --recursive") - assert cpm.recursive is True - - -def test_lock_recursive_true(): - cpm = CrossPM("lock --recursive True") - assert cpm.recursive is True - - -def test_lock_recursive_false(): - cpm = CrossPM("lock --recursive False") - assert cpm.recursive is False - - def test_lock_recursive_unknown(): try: _ = CrossPM("lock --recursive Trololo") From 802fdc9643b37f5f69a91fb228e624604bae6504 Mon Sep 17 00:00:00 2001 From: Anton Zolotukhin Date: Thu, 20 Oct 2022 17:16:15 +0700 Subject: [PATCH 5/9] revert pre-commit codepage changes --- .pre-commit-config.yaml | 1 + tests/data/config_en_cp1251.yaml | 2 +- tests/data/config_en_utf8_bom.yaml | 2 +- tests/data/config_ru_utf8_bom.yaml | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b339dad..87d79d9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,3 +1,4 @@ +exclude: ^tests/data repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v2.3.0 diff --git a/tests/data/config_en_cp1251.yaml b/tests/data/config_en_cp1251.yaml index dcf894c..ee6ad09 100644 --- a/tests/data/config_en_cp1251.yaml +++ b/tests/data/config_en_cp1251.yaml @@ -1,2 +1,2 @@ cpm: - description: English description + description: English description \ No newline at end of file diff --git a/tests/data/config_en_utf8_bom.yaml b/tests/data/config_en_utf8_bom.yaml index d155d78..3af6205 100644 --- a/tests/data/config_en_utf8_bom.yaml +++ b/tests/data/config_en_utf8_bom.yaml @@ -1,3 +1,3 @@ # Comment cpm: - description: My config for english test + description: My config for english test \ No newline at end of file diff --git a/tests/data/config_ru_utf8_bom.yaml b/tests/data/config_ru_utf8_bom.yaml index f8f9dfd..a1caae0 100644 --- a/tests/data/config_ru_utf8_bom.yaml +++ b/tests/data/config_ru_utf8_bom.yaml @@ -1,3 +1,3 @@ # Комментарий cpm: - description: Описание на русском + description: Описание на русском \ No newline at end of file From 6533f444d657e55b971d463c79709e708728f30d Mon Sep 17 00:00:00 2001 From: Anton Zolotukhin Date: Thu, 20 Oct 2022 17:22:38 +0700 Subject: [PATCH 6/9] pre-commit tuning --- .pre-commit-config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 87d79d9..325837d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,3 +11,4 @@ repos: rev: 4.0.1 hooks: - id: flake8 + args: ['crosspm'] From d8bfb73906579d19ef86cd2d71d815b6775f5e12 Mon Sep 17 00:00:00 2001 From: Anton Zolotukhin Date: Thu, 20 Oct 2022 17:23:54 +0700 Subject: [PATCH 7/9] Bump ver --- crosspm/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crosspm/config.py b/crosspm/config.py index bf66ffc..cc08f08 100644 --- a/crosspm/config.py +++ b/crosspm/config.py @@ -1 +1 @@ -__version__ = '1.0.13' +__version__ = '1.0.14' From 9126061008ca0035d4ff1ca27d167ee233986222 Mon Sep 17 00:00:00 2001 From: Anton Zolotukhin Date: Thu, 20 Oct 2022 17:32:17 +0700 Subject: [PATCH 8/9] pre-commit exclude tests --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 325837d..6310a4f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,4 +11,4 @@ repos: rev: 4.0.1 hooks: - id: flake8 - args: ['crosspm'] + exclude: ^tests/ From 3c1a08745c7b79085965f951cf921d369e7775c7 Mon Sep 17 00:00:00 2001 From: Anton Zolotukhin Date: Thu, 20 Oct 2022 19:02:40 +0700 Subject: [PATCH 9/9] fix faq --- docs/FAQ.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/FAQ.md b/docs/FAQ.md index 3d66a13..c94ca2e 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -33,11 +33,11 @@ crosspm lock --recursive False crosspm lock --recursive crosspm lock --recursive=True -# скачать все дерево пакетов вместе с зависимостями +# скачать без зависимостей, только пакеты указанные в dependencies crosspm download # поведение по умолчанию - только первый уровень crosspm download --recursive=False crosspm download --recursive False -# скачать без зависимостей, только пакеты указанные в dependencies +# скачать все дерево пакетов вместе с зависимостями crosspm download --recursive crosspm download --recursive=True ```