From 5caa7e02e710830f71ec0d320ce3f8539d9f8adb Mon Sep 17 00:00:00 2001 From: Sergei Silnov Date: Wed, 7 Aug 2024 18:29:19 +0200 Subject: [PATCH 1/6] chore: Split core tests into multiple files --- tests/core/test_add_dependency.py | 65 +++ .../core/test_create_project_from_example.py | 62 +++ tests/core/test_pack_component.py | 214 +++++++++ tests/core/test_upload_component.py | 83 ++++ tests/core/test_yank_version.py | 23 + tests/test_component_manager.py | 421 ------------------ 6 files changed, 447 insertions(+), 421 deletions(-) create mode 100644 tests/core/test_add_dependency.py create mode 100644 tests/core/test_create_project_from_example.py create mode 100644 tests/core/test_pack_component.py create mode 100644 tests/core/test_upload_component.py create mode 100644 tests/core/test_yank_version.py delete mode 100644 tests/test_component_manager.py diff --git a/tests/core/test_add_dependency.py b/tests/core/test_add_dependency.py new file mode 100644 index 0000000..c5d65ce --- /dev/null +++ b/tests/core/test_add_dependency.py @@ -0,0 +1,65 @@ +# SPDX-FileCopyrightText: 2022-2024 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 +"""Test Core commands""" + +import pytest +import vcr + +from idf_component_manager.core import ComponentManager +from idf_component_tools.constants import MANIFEST_FILENAME +from idf_component_tools.errors import FatalError +from idf_component_tools.manager import ManifestManager + + +@vcr.use_cassette('tests/fixtures/vcr_cassettes/test_init_project.yaml') +def test_init_project(mock_registry, tmp_path): + (tmp_path / 'main').mkdir() + (tmp_path / 'components' / 'foo').mkdir(parents=True) + main_manifest_path = tmp_path / 'main' / MANIFEST_FILENAME + foo_manifest_path = tmp_path / 'components' / 'foo' / MANIFEST_FILENAME + + manager = ComponentManager(path=str(tmp_path)) + manager.create_manifest() + manager.create_manifest(component='foo') + + for filepath in [main_manifest_path, foo_manifest_path]: + assert filepath.read_text().startswith('## IDF Component Manager') + + manager.add_dependency('cmp==4.0.3') + manifest_manager = ManifestManager(str(main_manifest_path), 'main') + assert manifest_manager.manifest_tree['dependencies']['espressif/cmp'] == '==4.0.3' + + manager.add_dependency('espressif/cmp==4.0.3', component='foo') + manifest_manager = ManifestManager(str(foo_manifest_path), 'foo') + assert manifest_manager.manifest_tree['dependencies']['espressif/cmp'] == '==4.0.3' + + +@vcr.use_cassette('tests/fixtures/vcr_cassettes/test_init_project_with_path.yaml') +def test_init_project_with_path(mock_registry, tmp_path): + src_path = tmp_path / 'src' + src_path.mkdir(parents=True, exist_ok=True) + src_manifest_path = src_path / MANIFEST_FILENAME + + outside_project_path = tmp_path.parent + outside_project_path_error_match = 'Directory ".*" is not under project directory!' + component_and_path_error_match = 'Cannot determine manifest directory.' + + manager = ComponentManager(path=str(tmp_path)) + manager.create_manifest(path=str(src_path)) + + with pytest.raises(FatalError, match=outside_project_path_error_match): + manager.create_manifest(path=str(outside_project_path)) + + with pytest.raises(FatalError, match=component_and_path_error_match): + manager.create_manifest(component='src', path=str(src_path)) + + manager.add_dependency('espressif/cmp==4.0.3', path=str(src_path)) + manifest_manager = ManifestManager(str(src_manifest_path), 'src') + + assert manifest_manager.manifest_tree['dependencies']['espressif/cmp'] == '==4.0.3' + + with pytest.raises(FatalError, match=outside_project_path_error_match): + manager.create_manifest(path=str(outside_project_path)) + + with pytest.raises(FatalError, match=component_and_path_error_match): + manager.add_dependency('espressif/cmp==4.0.3', component='src', path=str(src_path)) diff --git a/tests/core/test_create_project_from_example.py b/tests/core/test_create_project_from_example.py new file mode 100644 index 0000000..184f163 --- /dev/null +++ b/tests/core/test_create_project_from_example.py @@ -0,0 +1,62 @@ +# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 +import vcr +from pytest import raises + +from idf_component_manager.core import ComponentManager +from idf_component_tools.errors import FatalError + + +def test_create_example_project_path_not_a_directory(tmp_path): + existing_file = tmp_path / 'example' + existing_file.write_text('test') + + manager = ComponentManager(path=str(tmp_path)) + + with raises(FatalError, match='Your target path is not a directory*'): + manager.create_project_from_example('test:example') + + +def test_create_example_project_path_not_empty(tmp_path): + example_dir = tmp_path / 'example' + example_dir.mkdir() + existing_file = example_dir / 'test' + existing_file.write_text('test') + + manager = ComponentManager(path=str(tmp_path)) + + with raises(FatalError, match='To create an example you must*'): + manager.create_project_from_example('test:example') + + +@vcr.use_cassette('tests/fixtures/vcr_cassettes/test_create_example_component_not_exist.yaml') +def test_create_example_component_not_exist(tmp_path): + manager = ComponentManager(path=str(tmp_path)) + with raises(FatalError, match='Component "espressif/test" not found'): + manager.create_project_from_example('test:example') + + +@vcr.use_cassette('tests/fixtures/vcr_cassettes/test_create_example_not_exist.yaml') +def test_create_example_version_not_exist(mock_registry, tmp_path): + manager = ComponentManager(path=str(tmp_path)) + with raises( + FatalError, + match='Version of the component "test/cmp" satisfying the spec "=2.0.0" was not found.', + ): + manager.create_project_from_example('test/cmp=2.0.0:example') + + +@vcr.use_cassette('tests/fixtures/vcr_cassettes/test_create_example_not_exist.yaml') +def test_create_example_not_exist(mock_registry, tmp_path): + manager = ComponentManager(path=str(tmp_path)) + with raises( + FatalError, + match='Cannot find example "example" for "test/cmp" version "=1.0.1"', + ): + manager.create_project_from_example('test/cmp=1.0.1:example') + + +@vcr.use_cassette('tests/fixtures/vcr_cassettes/test_create_example_success.yaml') +def test_create_example_success(mock_registry, tmp_path): + manager = ComponentManager(path=str(tmp_path)) + manager.create_project_from_example('test/cmp>=1.0.0:sample_project') diff --git a/tests/core/test_pack_component.py b/tests/core/test_pack_component.py new file mode 100644 index 0000000..8f0f50c --- /dev/null +++ b/tests/core/test_pack_component.py @@ -0,0 +1,214 @@ +# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 +import os +import shutil +import tempfile +from pathlib import Path + +import pytest +import yaml + +from idf_component_manager.core import ComponentManager +from idf_component_tools.archive_tools import unpack_archive +from idf_component_tools.constants import MANIFEST_FILENAME +from idf_component_tools.errors import FatalError, ManifestError +from idf_component_tools.git_client import GitClient +from idf_component_tools.manager import ManifestManager +from idf_component_tools.semver import Version + + +def list_dir(folder): + res = [] + for root, _, files in os.walk(folder): + for file in files: + res.append(os.path.join(root, file)) + return res + + +def copy_into(src, dest): + for item in os.listdir(src): + s = os.path.join(src, item) + d = os.path.join(dest, item) + if os.path.isdir(s): + shutil.copytree(s, d) + else: + shutil.copy2(s, d) + + +def remove_version_line(path): + with open(os.path.join(str(path), MANIFEST_FILENAME), 'r+') as f: + lines = f.readlines() + f.seek(0) + f.writelines(lines[1:]) + f.truncate() + + +def test_pack_component_version_from_CLI_and_not_in_manifest(tmp_path, release_component_path): + copy_into(release_component_path, str(tmp_path)) + component_manager = ComponentManager(path=str(tmp_path)) + + # remove the first version line + remove_version_line(tmp_path) + + component_manager.pack_component('cmp', '2.3.4') + + tempdir = os.path.join(tempfile.tempdir, 'cmp') + unpack_archive(os.path.join(component_manager.dist_path, 'cmp_2.3.4.tgz'), tempdir) + manifest = ManifestManager(tempdir, 'cmp').load() + assert manifest.version == '2.3.4' + + +def test_pack_component_no_version_provided(tmp_path, release_component_path): + copy_into(release_component_path, str(tmp_path)) + component_manager = ComponentManager(path=str(tmp_path)) + + remove_version_line(tmp_path) + + with pytest.raises(ManifestError, match='Manifest is not valid'): + component_manager.pack_component('cmp', None) + + +def test_pack_component_version_from_git(monkeypatch, tmp_path, pre_release_component_path): + copy_into(pre_release_component_path, str(tmp_path)) + component_manager = ComponentManager(path=str(tmp_path)) + + # remove the first version line + remove_version_line(tmp_path) + + def mock_git_tag(self, cwd=None): + return Version('3.0.0') + + monkeypatch.setattr(GitClient, 'get_tag_version', mock_git_tag) + + component_manager.pack_component('pre', 'git') + + tempdir = os.path.join(tempfile.tempdir, 'cmp_pre') + unpack_archive(os.path.join(component_manager.dist_path, 'pre_3.0.0.tgz'), tempdir) + manifest = ManifestManager(tempdir, 'pre').load() + assert manifest.version == '3.0.0' + assert set(list_dir(tempdir)) == set( + os.path.join(tempdir, file) + for file in [ + 'idf_component.yml', + 'cmp.c', + 'CMakeLists.txt', + 'LICENSE', + os.path.join('include', 'cmp.h'), + ] + ) + + +@pytest.mark.parametrize( + 'version, expected_version', + [ + ('2.3.4', '2.3.4'), + ('2.3.4.1', '2.3.4~1'), + ('2.3.4~1', '2.3.4~1'), + ], +) +def test_pack_component_with_dest_dir(version, expected_version, tmp_path, release_component_path): + copy_into(release_component_path, str(tmp_path)) + component_manager = ComponentManager(path=str(tmp_path)) + + dest_path = tmp_path / 'dest_dir' + os.mkdir(str(dest_path)) + + # remove the first version line + remove_version_line(tmp_path) + + component_manager.pack_component('cmp', version, 'dest_dir') + + tempdir = os.path.join(tempfile.tempdir, 'cmp') + unpack_archive(os.path.join(str(dest_path), 'cmp_{}.tgz'.format(expected_version)), tempdir) + manifest = ManifestManager(tempdir, 'cmp').load() + assert manifest.version == expected_version + + +def test_pack_component_with_replacing_manifest_params(tmp_path, release_component_path): + copy_into(release_component_path, str(tmp_path)) + component_manager = ComponentManager(path=str(tmp_path)) + + repository_url = 'https://github.com/kumekay/test_multiple_comp' + commit_id = '252f10c83610ebca1a059c0bae8255eba2f95be4d1d7bcfa89d7248a82d9f111' + + component_manager.pack_component( + 'cmp', '2.3.5', repository=repository_url, commit_sha=commit_id + ) + + tempdir = os.path.join(tempfile.tempdir, 'cmp') + unpack_archive(os.path.join(component_manager.dist_path, 'cmp_2.3.5.tgz'), tempdir) + manifest = ManifestManager(tempdir, 'cmp').load() + + assert manifest.version == '2.3.5' + assert manifest.links.repository == repository_url + assert manifest.repository_info.commit_sha == commit_id + + +def test_pack_component_with_examples(tmp_path, example_component_path): + project_path = tmp_path / 'cmp' + shutil.copytree(example_component_path, str(project_path)) + component_manager = ComponentManager(path=str(project_path)) + + component_manager.pack_component('cmp', '2.3.4') + + unpack_archive( + str(Path(component_manager.dist_path, 'cmp_2.3.4.tgz')), + str(tmp_path / 'unpack'), + ) + + assert (tmp_path / 'unpack' / 'examples' / 'cmp_ex').is_dir() + assert ( + 'cmake_minimum_required(VERSION 3.16)' + in (tmp_path / 'unpack' / 'examples' / 'cmp_ex' / 'CMakeLists.txt').read_text() + ) + + +def test_pack_component_with_rules_if( + tmp_path, release_component_path, valid_optional_dependency_manifest_with_idf +): + project_path = tmp_path / 'cmp' + shutil.copytree(release_component_path, str(project_path)) + with open(str(project_path / MANIFEST_FILENAME), 'w') as fw: + yaml.dump(valid_optional_dependency_manifest_with_idf, fw) + + component_manager = ComponentManager(path=str(project_path)) + component_manager.pack_component('cmp', '2.3.4') + + +@pytest.mark.parametrize( + 'examples, message', + [ + ( + [ + {'path': './custom_example_path/cmp_ex'}, + {'path': './custom_example_path_2/cmp_ex'}, + ], + 'Examples from "./custom_example_path/cmp_ex" and "./custom_example_path_2/cmp_ex" ' + 'have the same name: cmp_ex.', + ), + ( + [{'path': './custom_example_path'}, {'path': './custom_example_path'}], + 'Some paths in the `examples` block in the manifest are listed multiple times: ' + './custom_example_path', + ), + ([{'path': './unknown_path'}], "Example directory doesn't exist:*"), + ], +) +def test_pack_component_with_examples_errors(tmp_path, example_component_path, examples, message): + project_path = tmp_path / 'cmp' + shutil.copytree(example_component_path, str(project_path)) + if len(examples) == 2 and examples[0] != examples[1]: # Add second example + shutil.copytree( + str(Path(example_component_path, 'custom_example_path')), + str(project_path / 'custom_example_path_2'), + ) + + component_manager = ComponentManager(path=str(project_path)) + + # Add folder with the same name of the example + manifest_manager = ManifestManager(str(project_path), 'cmp') + manifest_manager.manifest.examples = examples + manifest_manager.dump(str(project_path)) + + with pytest.raises(FatalError, match=message): + component_manager.pack_component('cmp', '2.3.4') diff --git a/tests/core/test_upload_component.py b/tests/core/test_upload_component.py new file mode 100644 index 0000000..9667e8c --- /dev/null +++ b/tests/core/test_upload_component.py @@ -0,0 +1,83 @@ +# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 +import shutil + +import pytest +import requests_mock +import vcr + +from idf_component_manager.core import ComponentManager +from idf_component_tools.errors import FatalError, NothingToDoError + + +@vcr.use_cassette('tests/fixtures/vcr_cassettes/test_upload_component.yaml') +def test_upload_component(mock_registry, pre_release_component_path, capsys): + manager = ComponentManager(path=pre_release_component_path) + + manager.upload_component('cmp') + captured = capsys.readouterr() + + assert 'WARNING: A homepage URL has not been provided in the manifest file.' in captured.err + + +def test_upload_component_http_error(mock_registry, pre_release_component_path): + with requests_mock.Mocker() as m: + # Mock the HTTP request to return a 502 error + m.get( + 'http://localhost:5000/api/components/espressif/cmp', + status_code=502, + json={'error': 'Err', 'messages': ['Some error messages']}, + ) + + with pytest.raises( + FatalError, + match='Internal server error happened while processing request.\n' + 'URL: http://localhost:5000/api/components/espressif/cmp\n' + 'Status code: 502 Bad Gateway', + ): + manager = ComponentManager(path=pre_release_component_path) + manager.upload_component('cmp') + + +@vcr.use_cassette('tests/fixtures/vcr_cassettes/test_check_only_component.yaml') +def test_check_only_upload_component(mock_registry, pre_release_component_path): + manager = ComponentManager(path=pre_release_component_path) + + manager.upload_component( + 'cmp', + check_only=True, + ) + + +@vcr.use_cassette('tests/fixtures/vcr_cassettes/test_allow_existing_component.yaml') +def test_allow_existing_component(mock_registry, release_component_path, tmp_path): + shutil.copytree(release_component_path, str(tmp_path / 'cmp')) + manager = ComponentManager(path=str(tmp_path / 'cmp')) + + manager.upload_component( + 'cmp', + allow_existing=True, + ) + + +@vcr.use_cassette('tests/fixtures/vcr_cassettes/test_validate_component.yaml') +def test_validate_component(mock_registry_without_token, pre_release_component_path): + manager = ComponentManager(path=pre_release_component_path) + + manager.upload_component( + 'cmp', + dry_run=True, + ) + + +@vcr.use_cassette('tests/fixtures/vcr_cassettes/test_upload_component_skip_pre.yaml') +def test_upload_component_skip_pre(mock_registry, pre_release_component_path): + manager = ComponentManager(path=pre_release_component_path) + + with pytest.raises(NothingToDoError) as e: + manager.upload_component( + 'cmp', + skip_pre_release=True, + ) + + assert str(e.value).startswith('Skipping pre-release') diff --git a/tests/core/test_yank_version.py b/tests/core/test_yank_version.py new file mode 100644 index 0000000..8f9cb6c --- /dev/null +++ b/tests/core/test_yank_version.py @@ -0,0 +1,23 @@ +# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 +import vcr +from pytest import raises + +from idf_component_manager.core import ComponentManager +from idf_component_tools.errors import FatalError + + +@vcr.use_cassette('tests/fixtures/vcr_cassettes/test_yank_version_success.yaml') +def test_yank_component_version(mock_registry, tmp_path): + manager = ComponentManager(path=str(tmp_path)) + manager.yank_version('cmp', '1.1.0', 'critical test', namespace='test') + + +@vcr.use_cassette('tests/fixtures/vcr_cassettes/test_yank_version_success.yaml') +def test_yank_component_version_not_exists(mock_registry, tmp_path): + manager = ComponentManager(path=str(tmp_path)) + with raises( + FatalError, + match='Version 1.2.0 of the component "test/cmp" is not on the registry', + ): + manager.yank_version('cmp', '1.2.0', 'critical test', namespace='test') diff --git a/tests/test_component_manager.py b/tests/test_component_manager.py deleted file mode 100644 index e0f12e3..0000000 --- a/tests/test_component_manager.py +++ /dev/null @@ -1,421 +0,0 @@ -# SPDX-FileCopyrightText: 2022-2024 Espressif Systems (Shanghai) CO LTD -# SPDX-License-Identifier: Apache-2.0 -"""Test Core commands""" - -import os -import shutil -import tempfile -from io import open -from pathlib import Path - -import pytest -import requests_mock -import vcr -import yaml -from pytest import raises - -from idf_component_manager.core import ComponentManager -from idf_component_tools.archive_tools import unpack_archive -from idf_component_tools.constants import MANIFEST_FILENAME -from idf_component_tools.errors import FatalError, ManifestError, NothingToDoError -from idf_component_tools.git_client import GitClient -from idf_component_tools.manager import ManifestManager -from idf_component_tools.semver import Version - - -def list_dir(folder): - res = [] - for root, _, files in os.walk(folder): - for file in files: - res.append(os.path.join(root, file)) - return res - - -def copy_into(src, dest): - for item in os.listdir(src): - s = os.path.join(src, item) - d = os.path.join(dest, item) - if os.path.isdir(s): - shutil.copytree(s, d) - else: - shutil.copy2(s, d) - - -@vcr.use_cassette('tests/fixtures/vcr_cassettes/test_init_project.yaml') -def test_init_project(mock_registry, tmp_path): - tempdir = str(tmp_path) - try: - os.makedirs(os.path.join(tempdir, 'main')) - os.makedirs(os.path.join(tempdir, 'components', 'foo')) - main_manifest_path = os.path.join(tempdir, 'main', MANIFEST_FILENAME) - foo_manifest_path = os.path.join(tempdir, 'components', 'foo', MANIFEST_FILENAME) - - manager = ComponentManager(path=tempdir) - manager.create_manifest() - manager.create_manifest(component='foo') - - for filepath in [main_manifest_path, foo_manifest_path]: - with open(filepath, mode='r') as file: - assert file.readline().startswith('## IDF Component Manager') - - manager.add_dependency('cmp==4.0.3') - manifest_manager = ManifestManager(main_manifest_path, 'main') - assert manifest_manager.manifest_tree['dependencies']['espressif/cmp'] == '==4.0.3' - - manager.add_dependency('espressif/cmp==4.0.3', component='foo') - manifest_manager = ManifestManager(foo_manifest_path, 'foo') - assert manifest_manager.manifest_tree['dependencies']['espressif/cmp'] == '==4.0.3' - - finally: - shutil.rmtree(tempdir) - - -@vcr.use_cassette('tests/fixtures/vcr_cassettes/test_init_project_with_path.yaml') -def test_init_project_with_path(mock_registry, tmp_path): - tempdir = str(tmp_path) - try: - os.makedirs(os.path.join(tempdir, 'src')) - src_path = os.path.join(tempdir, 'src') - src_manifest_path = os.path.join(src_path, MANIFEST_FILENAME) - - outside_project_path = str(Path(tempdir).parent) - outside_project_path_error_match = 'Directory ".*" is not under project directory!' - component_and_path_error_match = 'Cannot determine manifest directory.' - - manager = ComponentManager(path=tempdir) - manager.create_manifest(path=src_path) - - with pytest.raises(FatalError, match=outside_project_path_error_match): - manager.create_manifest(path=outside_project_path) - - with pytest.raises(FatalError, match=component_and_path_error_match): - manager.create_manifest(component='src', path=src_path) - - manager.add_dependency('espressif/cmp==4.0.3', path=src_path) - manifest_manager = ManifestManager(src_manifest_path, 'src') - - assert manifest_manager.manifest_tree['dependencies']['espressif/cmp'] == '==4.0.3' - - with pytest.raises(FatalError, match=outside_project_path_error_match): - manager.create_manifest(path=outside_project_path) - - with pytest.raises(FatalError, match=component_and_path_error_match): - manager.add_dependency('espressif/cmp==4.0.3', component='src', path=src_path) - - finally: - shutil.rmtree(tempdir) - - -@vcr.use_cassette('tests/fixtures/vcr_cassettes/test_upload_component.yaml') -def test_upload_component(mock_registry, pre_release_component_path, capsys): - manager = ComponentManager(path=pre_release_component_path) - - manager.upload_component('cmp') - captured = capsys.readouterr() - - assert 'WARNING: A homepage URL has not been provided in the manifest file.' in captured.err - - -def test_upload_component_http_error(mock_registry, pre_release_component_path): - with requests_mock.Mocker() as m: - # Mock the HTTP request to return a 502 error - m.get( - 'http://localhost:5000/api/components/espressif/cmp', - status_code=502, - json={'error': 'Err', 'messages': ['Some error messages']}, - ) - - with pytest.raises( - FatalError, - match='Internal server error happened while processing request.\n' - 'URL: http://localhost:5000/api/components/espressif/cmp\n' - 'Status code: 502 Bad Gateway', - ): - manager = ComponentManager(path=pre_release_component_path) - manager.upload_component('cmp') - - -@vcr.use_cassette('tests/fixtures/vcr_cassettes/test_check_only_component.yaml') -def test_check_only_upload_component(mock_registry, pre_release_component_path): - manager = ComponentManager(path=pre_release_component_path) - - manager.upload_component( - 'cmp', - check_only=True, - ) - - -@vcr.use_cassette('tests/fixtures/vcr_cassettes/test_allow_existing_component.yaml') -def test_allow_existing_component(mock_registry, release_component_path, tmp_path): - shutil.copytree(release_component_path, str(tmp_path / 'cmp')) - manager = ComponentManager(path=str(tmp_path / 'cmp')) - - manager.upload_component( - 'cmp', - allow_existing=True, - ) - - -@vcr.use_cassette('tests/fixtures/vcr_cassettes/test_validate_component.yaml') -def test_validate_component(mock_registry_without_token, pre_release_component_path): - manager = ComponentManager(path=pre_release_component_path) - - manager.upload_component( - 'cmp', - dry_run=True, - ) - - -@vcr.use_cassette('tests/fixtures/vcr_cassettes/test_upload_component_skip_pre.yaml') -def test_upload_component_skip_pre(mock_registry, pre_release_component_path): - manager = ComponentManager(path=pre_release_component_path) - - with pytest.raises(NothingToDoError) as e: - manager.upload_component( - 'cmp', - skip_pre_release=True, - ) - - assert str(e.value).startswith('Skipping pre-release') - - -def remove_version_line(path): - with open(os.path.join(str(path), MANIFEST_FILENAME), 'r+') as f: - lines = f.readlines() - f.seek(0) - f.writelines(lines[1:]) - f.truncate() - - -def test_pack_component_version_from_CLI_and_not_in_manifest(tmp_path, release_component_path): - copy_into(release_component_path, str(tmp_path)) - component_manager = ComponentManager(path=str(tmp_path)) - - # remove the first version line - remove_version_line(tmp_path) - - component_manager.pack_component('cmp', '2.3.4') - - tempdir = os.path.join(tempfile.tempdir, 'cmp') - unpack_archive(os.path.join(component_manager.dist_path, 'cmp_2.3.4.tgz'), tempdir) - manifest = ManifestManager(tempdir, 'cmp').load() - assert manifest.version == '2.3.4' - - -def test_pack_component_no_version_provided(tmp_path, release_component_path): - copy_into(release_component_path, str(tmp_path)) - component_manager = ComponentManager(path=str(tmp_path)) - - remove_version_line(tmp_path) - - with pytest.raises(ManifestError, match='Manifest is not valid'): - component_manager.pack_component('cmp', None) - - -def test_pack_component_version_from_git(monkeypatch, tmp_path, pre_release_component_path): - copy_into(pre_release_component_path, str(tmp_path)) - component_manager = ComponentManager(path=str(tmp_path)) - - # remove the first version line - remove_version_line(tmp_path) - - def mock_git_tag(self, cwd=None): - return Version('3.0.0') - - monkeypatch.setattr(GitClient, 'get_tag_version', mock_git_tag) - - component_manager.pack_component('pre', 'git') - - tempdir = os.path.join(tempfile.tempdir, 'cmp_pre') - unpack_archive(os.path.join(component_manager.dist_path, 'pre_3.0.0.tgz'), tempdir) - manifest = ManifestManager(tempdir, 'pre').load() - assert manifest.version == '3.0.0' - assert set(list_dir(tempdir)) == set( - os.path.join(tempdir, file) - for file in [ - 'idf_component.yml', - 'cmp.c', - 'CMakeLists.txt', - 'LICENSE', - os.path.join('include', 'cmp.h'), - ] - ) - - -@pytest.mark.parametrize( - 'version, expected_version', - [ - ('2.3.4', '2.3.4'), - ('2.3.4.1', '2.3.4~1'), - ('2.3.4~1', '2.3.4~1'), - ], -) -def test_pack_component_with_dest_dir(version, expected_version, tmp_path, release_component_path): - copy_into(release_component_path, str(tmp_path)) - component_manager = ComponentManager(path=str(tmp_path)) - - dest_path = tmp_path / 'dest_dir' - os.mkdir(str(dest_path)) - - # remove the first version line - remove_version_line(tmp_path) - - component_manager.pack_component('cmp', version, 'dest_dir') - - tempdir = os.path.join(tempfile.tempdir, 'cmp') - unpack_archive(os.path.join(str(dest_path), 'cmp_{}.tgz'.format(expected_version)), tempdir) - manifest = ManifestManager(tempdir, 'cmp').load() - assert manifest.version == expected_version - - -def test_pack_component_with_replacing_manifest_params(tmp_path, release_component_path): - copy_into(release_component_path, str(tmp_path)) - component_manager = ComponentManager(path=str(tmp_path)) - - repository_url = 'https://github.com/kumekay/test_multiple_comp' - commit_id = '252f10c83610ebca1a059c0bae8255eba2f95be4d1d7bcfa89d7248a82d9f111' - - component_manager.pack_component( - 'cmp', '2.3.5', repository=repository_url, commit_sha=commit_id - ) - - tempdir = os.path.join(tempfile.tempdir, 'cmp') - unpack_archive(os.path.join(component_manager.dist_path, 'cmp_2.3.5.tgz'), tempdir) - manifest = ManifestManager(tempdir, 'cmp').load() - - assert manifest.version == '2.3.5' - assert manifest.links.repository == repository_url - assert manifest.repository_info.commit_sha == commit_id - - -def test_pack_component_with_examples(tmp_path, example_component_path): - project_path = tmp_path / 'cmp' - shutil.copytree(example_component_path, str(project_path)) - component_manager = ComponentManager(path=str(project_path)) - - component_manager.pack_component('cmp', '2.3.4') - - unpack_archive( - str(Path(component_manager.dist_path, 'cmp_2.3.4.tgz')), str(tmp_path / 'unpack') - ) - - assert (tmp_path / 'unpack' / 'examples' / 'cmp_ex').is_dir() - assert ( - 'cmake_minimum_required(VERSION 3.16)' - in (tmp_path / 'unpack' / 'examples' / 'cmp_ex' / 'CMakeLists.txt').read_text() - ) - - -def test_pack_component_with_rules_if( - tmp_path, release_component_path, valid_optional_dependency_manifest_with_idf -): - project_path = tmp_path / 'cmp' - shutil.copytree(release_component_path, str(project_path)) - with open(str(project_path / MANIFEST_FILENAME), 'w') as fw: - yaml.dump(valid_optional_dependency_manifest_with_idf, fw) - - component_manager = ComponentManager(path=str(project_path)) - component_manager.pack_component('cmp', '2.3.4') - - -@pytest.mark.parametrize( - 'examples, message', - [ - ( - [{'path': './custom_example_path/cmp_ex'}, {'path': './custom_example_path_2/cmp_ex'}], - 'Examples from "./custom_example_path/cmp_ex" and "./custom_example_path_2/cmp_ex" ' - 'have the same name: cmp_ex.', - ), - ( - [{'path': './custom_example_path'}, {'path': './custom_example_path'}], - 'Some paths in the `examples` block in the manifest are listed multiple times: ' - './custom_example_path', - ), - ([{'path': './unknown_path'}], "Example directory doesn't exist:*"), - ], -) -def test_pack_component_with_examples_errors(tmp_path, example_component_path, examples, message): - project_path = tmp_path / 'cmp' - shutil.copytree(example_component_path, str(project_path)) - if len(examples) == 2 and examples[0] != examples[1]: # Add second example - shutil.copytree( - str(Path(example_component_path, 'custom_example_path')), - str(project_path / 'custom_example_path_2'), - ) - - component_manager = ComponentManager(path=str(project_path)) - - # Add folder with the same name of the example - manifest_manager = ManifestManager(str(project_path), 'cmp') - manifest_manager.manifest.examples = examples - manifest_manager.dump(str(project_path)) - - with pytest.raises(FatalError, match=message): - component_manager.pack_component('cmp', '2.3.4') - - -def test_create_example_project_path_not_a_directory(tmp_path): - existing_file = tmp_path / 'example' - existing_file.write_text('test') - - manager = ComponentManager(path=str(tmp_path)) - - with raises(FatalError, match='Your target path is not a directory*'): - manager.create_project_from_example('test:example') - - -def test_create_example_project_path_not_empty(tmp_path): - example_dir = tmp_path / 'example' - example_dir.mkdir() - existing_file = example_dir / 'test' - existing_file.write_text('test') - - manager = ComponentManager(path=str(tmp_path)) - - with raises(FatalError, match='To create an example you must*'): - manager.create_project_from_example('test:example') - - -@vcr.use_cassette('tests/fixtures/vcr_cassettes/test_create_example_component_not_exist.yaml') -def test_create_example_component_not_exist(tmp_path): - manager = ComponentManager(path=str(tmp_path)) - with raises(FatalError, match='Component "espressif/test" not found'): - manager.create_project_from_example('test:example') - - -@vcr.use_cassette('tests/fixtures/vcr_cassettes/test_create_example_not_exist.yaml') -def test_create_example_version_not_exist(mock_registry, tmp_path): - manager = ComponentManager(path=str(tmp_path)) - with raises( - FatalError, - match='Version of the component "test/cmp" satisfying the spec "=2.0.0" was not found.', - ): - manager.create_project_from_example('test/cmp=2.0.0:example') - - -@vcr.use_cassette('tests/fixtures/vcr_cassettes/test_create_example_not_exist.yaml') -def test_create_example_not_exist(mock_registry, tmp_path): - manager = ComponentManager(path=str(tmp_path)) - with raises(FatalError, match='Cannot find example "example" for "test/cmp" version "=1.0.1"'): - manager.create_project_from_example('test/cmp=1.0.1:example') - - -@vcr.use_cassette('tests/fixtures/vcr_cassettes/test_create_example_success.yaml') -def test_create_example_success(mock_registry, tmp_path): - manager = ComponentManager(path=str(tmp_path)) - manager.create_project_from_example('test/cmp>=1.0.0:sample_project') - - -@vcr.use_cassette('tests/fixtures/vcr_cassettes/test_yank_version_success.yaml') -def test_yank_component_version(mock_registry, tmp_path): - manager = ComponentManager(path=str(tmp_path)) - manager.yank_version('cmp', '1.1.0', 'critical test', namespace='test') - - -@vcr.use_cassette('tests/fixtures/vcr_cassettes/test_yank_version_success.yaml') -def test_yank_component_version_not_exists(mock_registry, tmp_path): - manager = ComponentManager(path=str(tmp_path)) - with raises( - FatalError, match='Version 1.2.0 of the component "test/cmp" is not on the registry' - ): - manager.yank_version('cmp', '1.2.0', 'critical test', namespace='test') From 41d8010b40d15d6bc05b0f7b122076ed91aa6105 Mon Sep 17 00:00:00 2001 From: Sergei Silnov Date: Thu, 8 Aug 2024 11:27:08 +0200 Subject: [PATCH 2/6] fix: Fix login command with non-existing directory --- idf_component_tools/config.py | 19 +++++++++++-------- tests/cli/test_compote.py | 9 +++++---- tests/test_config.py | 8 +++++++- 3 files changed, 23 insertions(+), 13 deletions(-) diff --git a/idf_component_tools/config.py b/idf_component_tools/config.py index 4237ff3..7fcb878 100644 --- a/idf_component_tools/config.py +++ b/idf_component_tools/config.py @@ -5,6 +5,7 @@ import os import typing as t +from pathlib import Path import yaml from pydantic import ( @@ -26,8 +27,6 @@ from .build_system_tools import get_idf_version -DEFAULT_CONFIG_DIR = os.path.join('~', '.espressif') - RegistryUrlField = t.Union[ Literal['default'], UrlField, @@ -81,12 +80,12 @@ class Config(BaseModel): profiles: t.Dict[str, t.Optional[ProfileItem]] = {} -def config_dir(): - return os.environ.get('IDF_TOOLS_PATH') or os.path.expanduser(DEFAULT_CONFIG_DIR) +def config_dir() -> Path: + return Path(os.environ.get('IDF_TOOLS_PATH') or Path.home() / '.espressif') -def root_managed_components_dir(): - return os.path.join(config_dir(), 'root_managed_components', f'idf{get_idf_version()}') +def root_managed_components_dir() -> Path: + return config_dir() / 'root_managed_components' / f'idf{get_idf_version()}' class ConfigError(FatalError): @@ -95,11 +94,11 @@ class ConfigError(FatalError): class ConfigManager: def __init__(self, path=None): - self.config_path = path or os.path.join(config_dir(), 'idf_component_manager.yml') + self.config_path = Path(path) if path else (config_dir() / 'idf_component_manager.yml') def load(self) -> Config: """Loads config from disk""" - if not os.path.isfile(self.config_path): + if not self.config_path.is_file(): return Config() with open(self.config_path, encoding='utf-8') as f: @@ -120,5 +119,9 @@ def validate(cls, data: t.Any) -> Config: def dump(self, config: Config) -> None: """Writes config to disk""" + + # Make sure that directory exists + self.config_path.parent.mkdir(parents=True, exist_ok=True) + with open(self.config_path, mode='w', encoding='utf-8') as f: yaml.dump(data=config.model_dump(), stream=f, encoding='utf-8', allow_unicode=True) diff --git a/tests/cli/test_compote.py b/tests/cli/test_compote.py index 7a63b7b..dbab836 100644 --- a/tests/cli/test_compote.py +++ b/tests/cli/test_compote.py @@ -47,16 +47,17 @@ def test_raise_exception_on_warnings(monkeypatch): ) -def test_login_to_registry(monkeypatch, tmp_path, mock_registry, mock_token_information): - monkeypatch.setenv('IDF_TOOLS_PATH', str(tmp_path)) - +def test_login_to_registry(tmp_path, mock_registry, mock_token_information): runner = CliRunner() cli = initialize_cli() output = runner.invoke( cli, ['registry', 'login', '--no-browser'], input='test_token', - env={'IDF_TOOLS_PATH': str(tmp_path)}, + env={ + # non-existing path is to check PACMAN-961 + 'IDF_TOOLS_PATH': str(tmp_path / 'non-existing-path') + }, ) assert output.exit_code == 0 diff --git a/tests/test_config.py b/tests/test_config.py index 4f2f5b1..dabf589 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -95,7 +95,7 @@ def test_load_config(tmp_path): def test_config_dump(tmp_path): - config_path = str(tmp_path / 'idf_component_manager.yml') + config_path = tmp_path / 'idf_component_manager.yml' config = ConfigManager.validate({ 'profiles': { 'default': { @@ -111,6 +111,12 @@ def test_config_dump(tmp_path): assert loaded_config.profiles['in_office'].registry_url == 'http://api.localserver.local:5000/' +def test_dump_non_existing_dir(tmp_path): + config_path = tmp_path / 'non_existing_dir' / 'idf_component_manager.yml' + config = ConfigManager.validate({}) + ConfigManager(path=config_path).dump(config) + + def test_component_registry_url_storage_env(monkeypatch): monkeypatch.setenv('IDF_COMPONENT_STORAGE_URL', 'https://storage.com/') assert ['https://storage.com/'] == get_storage_urls() From 15b17037aa335b4bf1094da74ee49b34a96295ab Mon Sep 17 00:00:00 2001 From: Sergei Silnov Date: Thu, 8 Aug 2024 17:30:35 +0200 Subject: [PATCH 3/6] fix: pack and upload components without manifests --- idf_component_tools/manager.py | 53 ++++++++++--------- idf_component_tools/manifest/models.py | 7 ++- tests/core/test_pack_component.py | 11 ++++ .../manifest/test_manager.py | 5 +- 4 files changed, 47 insertions(+), 29 deletions(-) diff --git a/idf_component_tools/manager.py b/idf_component_tools/manager.py index 43eeb81..dbd2d62 100644 --- a/idf_component_tools/manager.py +++ b/idf_component_tools/manager.py @@ -3,6 +3,7 @@ import os import typing as t +from pathlib import Path import yaml @@ -30,7 +31,9 @@ def __init__( commit_sha: t.Optional[str] = None, repository_path: t.Optional[str] = None, ) -> None: - self.path = os.path.join(path, MANIFEST_FILENAME) if os.path.isdir(path) else path + source_path = Path(path) + self.path: Path = source_path / MANIFEST_FILENAME if source_path.is_dir() else source_path + self.name = name self._manifest: 'Manifest' = None # type: ignore @@ -48,30 +51,32 @@ def __init__( self._validation_errors: t.List[str] = None # type: ignore def validate(self) -> 'ManifestManager': - from .manifest.models import Manifest, RepositoryInfoField # avoid circular dependency + from .manifest.models import ( + Manifest, + RepositoryInfoField, + ) + + # avoid circular dependency from .utils import ComponentVersion if self._manifest: return self - if not os.path.isfile(self.path): - self._validation_errors = [] - self._manifest = Manifest(name=self.name, manifest_manager=self) - return self - + if not self.path.exists(): + manifest_dict: t.Dict[str, t.Any] = {} # validate manifest - try: - with open(self.path, 'r') as f: - d = yaml.safe_load(f) or {} - except yaml.YAMLError: - self._validation_errors = [ - 'Cannot parse the manifest file. Please check that\n' - '\t{}\n' - 'is a valid YAML file\n'.format(self.path) - ] - return self - - if not isinstance(d, dict): + else: + try: + manifest_dict = yaml.safe_load(self.path.read_text()) or {} + except yaml.YAMLError: + self._validation_errors = [ + 'Cannot parse the manifest file. Please check that\n' + '\t{}\n' + 'is a valid YAML file\n'.format(self.path) + ] + return self + + if not isinstance(manifest_dict, dict): self._validation_errors = [ 'Manifest file should be a dictionary. Please check that\n' '\t{}\n' @@ -80,15 +85,15 @@ def validate(self) -> 'ManifestManager': return self if self.name: - d['name'] = self.name + manifest_dict['name'] = self.name if self._version: - d['version'] = self._version + manifest_dict['version'] = self._version - d['manifest_manager'] = self + manifest_dict['manifest_manager'] = self self._validation_errors, self._manifest = Manifest.validate_manifest( # type: ignore - d, + manifest_dict, upload_mode=self.upload_mode, return_with_object=True, ) @@ -150,7 +155,7 @@ def load(self) -> 'Manifest': def dump( self, - path: t.Optional[str] = None, + path: t.Optional[t.Union[str, Path]] = None, ) -> None: if path is None: path = self.path diff --git a/idf_component_tools/manifest/models.py b/idf_component_tools/manifest/models.py index 40b9582..41326df 100644 --- a/idf_component_tools/manifest/models.py +++ b/idf_component_tools/manifest/models.py @@ -18,7 +18,10 @@ from pydantic_core.core_schema import SerializerFunctionWrapHandler from pyparsing import ParseException -from idf_component_tools.build_system_tools import build_name, build_name_to_namespace_name +from idf_component_tools.build_system_tools import ( + build_name, + build_name_to_namespace_name, +) from idf_component_tools.constants import ( COMMIT_ID_RE, COMPILED_GIT_URL_RE, @@ -566,7 +569,7 @@ def real_name(self) -> str: @property def path(self) -> str: - return self._manifest_manager.path if self._manifest_manager else '' + return str(self._manifest_manager.path) if self._manifest_manager else '' class SolvedComponent(BaseModel): diff --git a/tests/core/test_pack_component.py b/tests/core/test_pack_component.py index 8f0f50c..28a4265 100644 --- a/tests/core/test_pack_component.py +++ b/tests/core/test_pack_component.py @@ -68,6 +68,17 @@ def test_pack_component_no_version_provided(tmp_path, release_component_path): component_manager.pack_component('cmp', None) +def test_pack_component_no_version_provided_nor_manifest(tmp_path, release_component_path): + copy_into(release_component_path, tmp_path) + + component_manager = ComponentManager(path=tmp_path) + + (tmp_path / MANIFEST_FILENAME).unlink() + + with pytest.raises(ManifestError, match='Manifest is not valid'): + component_manager.pack_component('cmp', None) + + def test_pack_component_version_from_git(monkeypatch, tmp_path, pre_release_component_path): copy_into(pre_release_component_path, str(tmp_path)) component_manager = ComponentManager(path=str(tmp_path)) diff --git a/tests/idf_comonent_tools_tests/manifest/test_manager.py b/tests/idf_comonent_tools_tests/manifest/test_manager.py index 98bac72..8098836 100644 --- a/tests/idf_comonent_tools_tests/manifest/test_manager.py +++ b/tests/idf_comonent_tools_tests/manifest/test_manager.py @@ -11,10 +11,9 @@ def test_check_filename(tmp_path): - path = tmp_path.as_posix() - parser = ManifestManager(path, name='test') + parser = ManifestManager(tmp_path, name='test') - assert parser.path == os.path.join(path, 'idf_component.yml') + assert parser.path == tmp_path / 'idf_component.yml' def test_parse_invalid_yaml(fixtures_path): From c0393e4eafe5ff35a16f487836cbff9680aebccf Mon Sep 17 00:00:00 2001 From: Sergei Silnov Date: Thu, 8 Aug 2024 18:13:03 +0200 Subject: [PATCH 4/6] fix!: remove redundant option --namespace from component pack CLI --- idf_component_manager/cli/component.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/idf_component_manager/cli/component.py b/idf_component_manager/cli/component.py index b2b31f3..885fc36 100644 --- a/idf_component_manager/cli/component.py +++ b/idf_component_manager/cli/component.py @@ -5,6 +5,7 @@ from .constants import ( get_dest_dir_option, + get_name_option, get_namespace_name_options, get_project_dir_option, get_project_options, @@ -15,6 +16,7 @@ def init_component(): PROJECT_DIR_OPTION = get_project_dir_option() PROJECT_OPTIONS = get_project_options() + NAME_OPTION = get_name_option() NAMESPACE_NAME_OPTIONS = get_namespace_name_options() DEST_DIR_OPTION = get_dest_dir_option() @@ -55,14 +57,13 @@ def component(): @component.command() @add_options( PROJECT_DIR_OPTION - + NAMESPACE_NAME_OPTIONS + + NAME_OPTION + COMPONENT_VERSION_OPTION + DEST_DIR_OPTION + COMMIT_SHA_REPO_OPTION ) def pack( manager, - namespace, name, version, dest_dir, From c9e55befec73ce86a2704cbc68e29cf7440b7242 Mon Sep 17 00:00:00 2001 From: Sergei Silnov Date: Thu, 8 Aug 2024 18:38:33 +0200 Subject: [PATCH 5/6] fix: manifest dump always adds empty fields --- idf_component_tools/manifest/models.py | 2 +- .../manifest/test_manager.py | 17 ++++++++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/idf_component_tools/manifest/models.py b/idf_component_tools/manifest/models.py index 41326df..39c95e8 100644 --- a/idf_component_tools/manifest/models.py +++ b/idf_component_tools/manifest/models.py @@ -389,7 +389,7 @@ def model_dump( self, **kwargs, ) -> t.Dict[str, t.Any]: - return super().model_dump(exclude=['name']) + return super().model_dump(exclude=['name'], exclude_unset=True) @field_validator('version') @classmethod diff --git a/tests/idf_comonent_tools_tests/manifest/test_manager.py b/tests/idf_comonent_tools_tests/manifest/test_manager.py index 8098836..3e5dba6 100644 --- a/tests/idf_comonent_tools_tests/manifest/test_manager.py +++ b/tests/idf_comonent_tools_tests/manifest/test_manager.py @@ -63,7 +63,7 @@ def test_validate_env_not_expanded(valid_manifest, tmp_path): valid_manifest['dependencies']['test']['rules'] = [{'if': 'target == $CURRENT_TARGET'}] valid_manifest['dependencies']['test']['matches'] = [{'if': 'idf_version == $CURRENT_IDF'}] - manifest_path = os.path.join(str(tmp_path), 'idf_component.yml') + manifest_path = tmp_path / 'idf_component.yml' with open(manifest_path, 'w') as fw: yaml.dump(valid_manifest, fw) @@ -79,3 +79,18 @@ def test_validate_env_not_expanded(valid_manifest, tmp_path): # Check that file is not modified assert filecmp.cmp(manifest_path, test_dump_path / 'idf_component.yml') + + +def test_dump_does_not_add_fields(tmp_path): + manifest_path = tmp_path / 'idf_component.yml' + manifest_content = 'version: 1.0.0\n' + manifest_path.write_text(manifest_content) + manager = ManifestManager(manifest_path, name='tst') + + test_dump_path = tmp_path / 'test' + test_dump_path.mkdir() + manager.dump(test_dump_path) + + dumped_manifiest_content = (test_dump_path / 'idf_component.yml').read_text() + + assert manifest_content == dumped_manifiest_content From f0af77fe54cd6ce1d8753567f9b894f1a7c8478b Mon Sep 17 00:00:00 2001 From: Sergei Silnov Date: Thu, 8 Aug 2024 18:47:08 +0200 Subject: [PATCH 6/6] test: Add test to get namespace from profile --- tests/test_service_details.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/test_service_details.py b/tests/test_service_details.py index a659059..2c0279e 100644 --- a/tests/test_service_details.py +++ b/tests/test_service_details.py @@ -61,10 +61,17 @@ def test_get_namespace_with_namespace(): assert get_api_client(namespace='example').default_namespace == 'example' -def test_get_namespace_default(service_config): +def test_get_namespace_default(): assert get_storage_client(namespace=None).default_namespace == DEFAULT_NAMESPACE +def test_get_namespace_from_profile( + config_path, +): + api_client = get_api_client(profile_name='test', config_path=config_path) + assert api_client.default_namespace == 'test' + + def test_get_token_env(monkeypatch): monkeypatch.setenv('IDF_COMPONENT_API_TOKEN', 'some_token') @@ -81,7 +88,8 @@ def test_empty_env_profile(monkeypatch): def test_get_token_profile(config_path, monkeypatch): - assert get_api_client(profile_name='test', config_path=config_path).api_token == 'token' + api_client = get_api_client(profile_name='test', config_path=config_path) + assert api_client.api_token == 'token' def test_get_profile_success(config_path):