diff --git a/.gitlab/ci/.gitlab-ci.on-push.yml b/.gitlab/ci/.gitlab-ci.on-push.yml index f1e6a3634a43..bc818d96446a 100644 --- a/.gitlab/ci/.gitlab-ci.on-push.yml +++ b/.gitlab/ci/.gitlab-ci.on-push.yml @@ -136,6 +136,7 @@ validate-content-conf: policy: pull-push variables: KUBERNETES_CPU_REQUEST: 2000m + EXTRACT_PRIVATE_TESTDATA: "true" needs: [] stage: prepare-testing-bucket script: @@ -587,6 +588,14 @@ slack-notify-on-merge: - ./Tests/scripts/test_modeling_rules.sh || EXIT_CODE=$? - section_end "Test Modeling Rules" + - section_start "Packs re-installation test" + - | + if [[ -z "${NIGHTLY}" && -s $ARTIFACTS_FOLDER/packs_reinstall_to_test.txt ]]; then + echo "Running the packs re-installation test." + ./Tests/scripts/reinstall_packs_on_cloud_instances.sh || EXIT_CODE=$? + fi + - section_end "Packs re-installation test" + - !reference [.cloud-machine-information] - job-done - exit $EXIT_CODE @@ -594,18 +603,6 @@ slack-notify-on-merge: - source .gitlab/helper_functions.sh - !reference [.unlock-machine] -#xsiam_server_dev: -# extends: -# - .test_content_on_xsiam_server_instances_base -# rules: -# - if: '$CI_PIPELINE_SOURCE =~ /^(push|contrib)$/' -# - if: '$NIGHTLY' -# when: always -# variables: -# INSTANCE_ROLE: "XSIAM Master" -# PRODUCT_TYPE: "XSIAM" -# GCS_QUEUE_FILE: "queue-master" -# TEST_MACHINES_LIST: "test-machines-master" xsiam_server_ga: extends: diff --git a/Tests/Marketplace/Tests/search_and_unistall_pack_test.py b/Tests/Marketplace/Tests/search_and_unistall_pack_test.py index d4d9726d15cb..995a10913257 100644 --- a/Tests/Marketplace/Tests/search_and_unistall_pack_test.py +++ b/Tests/Marketplace/Tests/search_and_unistall_pack_test.py @@ -1,3 +1,7 @@ +import json +import tempfile +from pathlib import Path + import demisto_client import pytest @@ -168,3 +172,47 @@ def success_handler(_): assert res == ret_value assert request_method.call_count == num_of_retries + + +@pytest.mark.parametrize('status_code, expected_res', [ + (200, True), + (599, True), + (400, False), +]) +def test_handle_delete_datasets_by_testdata(requests_mock, status_code, expected_res): + """ + Given + - Modeling rules with test data. + When + - Cleaning XSIAM machine + Then + - Verify the dataset deletion was results as expected. + """ + test_data = {"data": [ + { + "test_data_event_id": "some_uuid", + "vendor": "vendor", + "product": "product", + "dataset": "product_vendor_raw", + "event_data": { + "test": "test" + }, + "expected_values": { + "test": "test" + } + }] + } + with tempfile.TemporaryDirectory() as temp_dir: + test_data_file = Path("Packs/MyPack/ModelingRules/MR/MR_testdata.json") + test_data_file.parent.mkdir(parents=True, exist_ok=True) + test_data_file.write_text(json.dumps(test_data)) + + with open(f'{temp_dir}/modeling_rules_to_test.txt', 'w') as mr_to_test: + mr_to_test.write("MyPack/ModelingRules/MR") + + dataset_names = script.get_datasets_to_delete(modeling_rules_file=f'{temp_dir}/modeling_rules_to_test.txt') + requests_mock.post(f'{BASE_URL}/public_api/v1/xql/delete_dataset', status_code=status_code) + res = script.delete_datasets_by_testdata(base_url=BASE_URL, api_key=API_KEY, auth_id="1", + dataset_names=dataset_names) + test_data_file.unlink() + assert res == expected_res diff --git a/Tests/Marketplace/common.py b/Tests/Marketplace/common.py index 4c9e5b7e9d04..d4aa6125320e 100644 --- a/Tests/Marketplace/common.py +++ b/Tests/Marketplace/common.py @@ -2,8 +2,9 @@ import time from typing import Any from collections.abc import Callable - +from urllib.parse import urljoin import demisto_client +import requests from demisto_client.demisto_api.rest import ApiException from urllib3.exceptions import HTTPError, HTTPWarning @@ -163,3 +164,104 @@ def wait_until_not_updating(client: demisto_client, return False logging.info(f"Exiting after exhausting the allowed time:{maximum_time_to_wait} seconds") return False + + +def send_api_request_with_retries( + base_url: str, + retries_message: str, + exception_message: str, + success_message: str, + prior_message: str, + endpoint: str, + method: str, + params: dict | None = None, + headers: dict | None = None, + request_timeout: int | None = None, + accept: str = 'application/json', + body: Any | None = None, + attempts_count: int = 5, + sleep_interval: int = 60, + should_try_handler: Callable[[Any], bool] | None = None, + success_handler: Callable[[Any], Any] | None = None, + api_exception_handler: Callable[[ApiException, int], Any] | None = None, + http_exception_handler: Callable[[HTTPError | HTTPWarning], Any] | None = None +): + """ + Args: + body: request body. + base_url: base url to send request + headers: headers for the request. + success_message: message to print after success response. + retries_message: message to print after failure when we have more attempts. + exception_message: message to print when we get and exception that is not API or HTTP exception. + prior_message: message to print when a new retry is made. + endpoint: endpoint to send request to. + method: HTTP method to use. + params: api request params. + request_timeout: request param. + accept: request param. + attempts_count: number of total attempts made. + sleep_interval: sleep interval between attempts. + should_try_handler: a method to determine if we should send the next request. + success_handler: a method to run in case of successful request (according to the response status). + api_exception_handler: a method to run in case of api exception. + http_exception_handler: a method to run in case of http exception + + Returns: True if the request succeeded + + """ + headers = headers if headers else {} + headers['Accept'] = accept + response = None + url_path = urljoin(base_url, endpoint) + try: + for attempts_left in range(attempts_count - 1, -1, -1): + try: + if should_try_handler and not should_try_handler(response): + # if the method exist, and we should not try again. + return True + + logging.info(f"{prior_message}, attempt: {attempts_count - attempts_left}/{attempts_count}") + response = requests.request( + method=method, + url=url_path, + verify=False, + params=params, + data=body, + headers=headers, + timeout=request_timeout, + ) + if 200 <= response.status_code < 300 and response.status_code != 204: + logging.debug(f"Got successful response: {response.status_code=}, {response.content=}.") + logging.info(success_message) + if success_handler: + return success_handler(response) + + return True + + else: + err = f"Got {response.status_code=}, {response.headers=}, {response.content=}" + + if not attempts_left: + raise Exception(err) + + logging.warning(f"Got error: {err}") + + except ApiException as ex: + if api_exception_handler: + api_exception_handler(ex, attempts_left) + if not attempts_left: + raise Exception(f"Got status {ex.status} from server, message: {ex.body}, headers: {ex.headers}") from ex + logging.debug(f"Process failed, got error {ex}") + except (HTTPError, HTTPWarning) as http_ex: + if http_exception_handler: + http_exception_handler(http_ex) + if not attempts_left: + raise Exception("Failed to perform http request to the server") from http_ex + logging.debug(f"Process failed, got error {http_ex}") + + logging.debug(f"{retries_message}, sleeping for {sleep_interval} seconds.") + time.sleep(sleep_interval) + except Exception as e: + logging.exception(f'{exception_message}. Additional info: {str(e)}') + return False diff --git a/Tests/Marketplace/reinstall_packs.py b/Tests/Marketplace/reinstall_packs.py new file mode 100644 index 000000000000..4d4b69424541 --- /dev/null +++ b/Tests/Marketplace/reinstall_packs.py @@ -0,0 +1,106 @@ +import argparse +import os +import sys +from concurrent.futures import ThreadPoolExecutor, as_completed +from pathlib import Path +import demisto_client +from Tests.Marketplace.configure_and_install_packs import search_and_install_packs_and_their_dependencies +from Tests.Marketplace.search_and_uninstall_pack import uninstall_pack +from Tests.configure_and_test_integration_instances import CloudBuild, get_custom_user_agent +from Tests.scripts.utils import logging_wrapper as logging +from Tests.scripts.utils.log_util import install_logging + + +def options_handler() -> argparse.Namespace: + """ + Returns: options parsed from input arguments. + """ + parser = argparse.ArgumentParser(description='Utility for testing re-installation XSIAM packs.') + parser.add_argument('--cloud_machine', help='cloud machine to use, if it is cloud build.') + parser.add_argument('--cloud_servers_path', help='Path to secret cloud server metadata file.') + parser.add_argument('--cloud_servers_api_keys', help='Path to the file with cloud Servers api keys.') + parser.add_argument('--non-removable-packs', help='List of packs that cant be removed.') + parser.add_argument('--build-number', help='CI job number where the instances were created', required=True) + parser.add_argument('--packs_to_reinstall', help='List of packs to check its re-installation.', required=True) + + options = parser.parse_args() + + return options + + +def reinstall_packs(options: argparse.Namespace, cloud_machine: str) -> bool: + """ + Packs re-installation test. Uninstall and then install packs from options.packs_to_reinstall list. + + Args: + options: Script arguments. + cloud_machine (str): Cloud machine name to test on. + + Returns: + Boolean - If the operation succeeded. + """ + success = True + api_key, _, base_url, xdr_auth_id = CloudBuild.get_cloud_configuration(cloud_machine, + options.cloud_servers_path, + options.cloud_servers_api_keys) + + client = demisto_client.configure(base_url=base_url, + verify_ssl=False, + api_key=api_key, + auth_id=xdr_auth_id) + + client.api_client.user_agent = get_custom_user_agent(options.build_number) + host = client.api_client.configuration.host.replace('https://api-', 'https://') # disable-secrets-detection + + logging.debug(f'Setting user agent on client to: {client.api_client.user_agent}') + + non_removable_packs = options.non_removable_packs.split(',') + packs_to_reinstall_path = Path(options.packs_to_reinstall) + packs_to_reinstall = packs_to_reinstall_path.read_text().split('\n') + + for pack in packs_to_reinstall: + if pack in non_removable_packs: + continue + successful_uninstall, _ = uninstall_pack(client, pack) + _, successful_install = search_and_install_packs_and_their_dependencies( + pack_ids=[pack], + client=client, + hostname=host, + multithreading=False, + production_bucket=False + ) + success &= successful_uninstall & successful_install + + return success + + +def main(): + install_logging('reinstall_packs_check.log', logger=logging) + + # In Cloud, We don't use demisto username + os.environ.pop('DEMISTO_USERNAME', None) + + options = options_handler() + logging.info(f'Starting reinstall test for CLOUD servers:{options.cloud_machine}.') + cloud_machines: list[str] = list(filter(None, options.cloud_machine.split(','))) + success = True + with ThreadPoolExecutor(max_workers=len(cloud_machines), thread_name_prefix='clean-machine') as executor: + futures = [ + executor.submit(reinstall_packs, options, cloud_machine) + for cloud_machine in cloud_machines + ] + for future in as_completed(futures): + try: + success &= future.result() + except Exception as ex: + logging.exception(f'Failed to run reinstall packs test. Additional info: {str(ex)}') + success = False + + if not success: + logging.error('Failed to reinstall packs.') + sys.exit(2) + logging.info('Finished reinstall packs test successfully.') + + +if __name__ == '__main__': + main() diff --git a/Tests/Marketplace/search_and_uninstall_pack.py b/Tests/Marketplace/search_and_uninstall_pack.py index dc7b3a457513..672c8ccffe66 100644 --- a/Tests/Marketplace/search_and_uninstall_pack.py +++ b/Tests/Marketplace/search_and_uninstall_pack.py @@ -1,22 +1,29 @@ import argparse import ast +import json import math import os import sys from concurrent.futures import ThreadPoolExecutor, as_completed from datetime import datetime +from pathlib import Path from time import sleep from typing import Any import demisto_client -from Tests.Marketplace.common import generic_request_with_retries, wait_until_not_updating, ALREADY_IN_PROGRESS +from Tests.Marketplace.common import generic_request_with_retries, wait_until_not_updating, ALREADY_IN_PROGRESS, \ + send_api_request_with_retries from Tests.Marketplace.configure_and_install_packs import search_and_install_packs_and_their_dependencies from Tests.configure_and_test_integration_instances import CloudBuild, get_custom_user_agent from Tests.scripts.utils import logging_wrapper as logging from Tests.scripts.utils.log_util import install_logging +TEST_DATA_PATTERN = '*_testdata.json' +DATASET_NOT_FOUND_ERROR_CODE = 599 + + def check_if_pack_still_installed(client: demisto_client, pack_id: str, attempts_count: int = 3, @@ -334,19 +341,104 @@ def api_exception_handler(api_ex, _) -> Any: return success +def delete_datasets(dataset_names, base_url, api_key, auth_id): + """ + Return dataset names from testdata files. + Args: + dataset_names (set):dataset names to delete + base_url (str): The base url of the machine. + api_key (str): API key of the machine. + auth_id (str): authentication parameter for the machine. + Returns: + Boolean - If the operation succeeded. + """ + def should_try_handler(response) -> Any: + if response is not None and response.status_code == DATASET_NOT_FOUND_ERROR_CODE: + logging.info("Failed to delete dataset, probably it is not exist on the machine.") + return False + return True + + success = True + for dataset in dataset_names: + headers = { + "x-xdr-auth-id": str(auth_id), + "Authorization": api_key, + "Content-Type": "application/json", + } + body = {'dataset_name': dataset} + success &= send_api_request_with_retries( + base_url=base_url, + retries_message='Retrying to delete dataset', + success_message=f'Successfully deleted dataset: "{dataset}".', + exception_message=f'Failed to delete dataset: "{dataset}"', + prior_message=f'Trying to delete dataset: "{dataset}"', + endpoint='/public_api/v1/xql/delete_dataset', + method='POST', + headers=headers, + accept='application/json', + body=json.dumps(body), + should_try_handler=should_try_handler, + ) + return success + + +def get_datasets_to_delete(modeling_rules_file: str): + """ + Given a path to a file containing a list of modeling rules paths, + returns a list of their corresponding datasets that should be deleted. + Args: + modeling_rules_file (str): A path to a file holding the list of modeling rules collected for testing in this build. + Returns: + Set - datasets to delete. + """ + datasets_to_delete = set() + with open(modeling_rules_file) as f: + for modeling_rule_to_test in f.readlines(): + modeling_rule_path = Path(f'Packs/{modeling_rule_to_test.strip()}') + test_data_matches = list(modeling_rule_path.glob(TEST_DATA_PATTERN)) + if test_data_matches: + modeling_rule_testdata_path = test_data_matches[0] + test_data = json.loads(modeling_rule_testdata_path.read_text()) + for data in test_data.get('data', []): + dataset_name = data.get('dataset') + if dataset_name: + datasets_to_delete.add(dataset_name) + return datasets_to_delete + + +def delete_datasets_by_testdata(base_url, api_key, auth_id, dataset_names): + """ + Delete all datasets that the build will test in this job. + + Args: + base_url (str): The base url of the machine. + api_key (str): API key of the machine. + auth_id (str): authentication parameter for the machine. + dataset_names (set): datasets to delete + + Returns: + Boolean - If the operation succeeded. + """ + logging.info("Starting to handle delete datasets from cloud instance.") + logging.debug(f'Collected datasets to delete {dataset_names=}.') + success = delete_datasets(dataset_names=dataset_names, base_url=base_url, api_key=api_key, auth_id=auth_id) + return success + + def options_handler() -> argparse.Namespace: """ Returns: options parsed from input arguments. """ - parser = argparse.ArgumentParser(description='Utility for instantiating and testing integration instances') + parser = argparse.ArgumentParser(description='Utility for cleaning Cloud machines.') parser.add_argument('--cloud_machine', help='cloud machine to use, if it is cloud build.') parser.add_argument('--cloud_servers_path', help='Path to secret cloud server metadata file.') parser.add_argument('--cloud_servers_api_keys', help='Path to the file with cloud Servers api keys.') parser.add_argument('--non-removable-packs', help='List of packs that cant be removed.') parser.add_argument('--one-by-one', help='Uninstall pack one pack at a time.', action='store_true') parser.add_argument('--build-number', help='CI job number where the instances were created', required=True) + parser.add_argument('--modeling_rules_to_test_files', help='List of modeling rules test data to check.', required=True) options = parser.parse_args() @@ -377,6 +469,12 @@ def clean_machine(options: argparse.Namespace, cloud_machine: str) -> bool: success = uninstall_all_packs(client, cloud_machine, non_removable_packs) and \ wait_for_uninstallation_to_complete(client, non_removable_packs) success &= sync_marketplace(client=client) + success &= delete_datasets_by_testdata(base_url=base_url, + api_key=api_key, + auth_id=xdr_auth_id, + dataset_names=get_datasets_to_delete( + modeling_rules_file=options.modeling_rules_to_test_files) + ) return success diff --git a/Tests/scripts/collect_tests/collect_tests.py b/Tests/scripts/collect_tests/collect_tests.py index 4f8eda4d83a4..c57ad3e53e98 100644 --- a/Tests/scripts/collect_tests/collect_tests.py +++ b/Tests/scripts/collect_tests/collect_tests.py @@ -16,7 +16,8 @@ DEFAULT_MARKETPLACE_WHEN_MISSING, IGNORED_FILE_TYPES, NON_CONTENT_FOLDERS, ONLY_INSTALL_PACK_FILE_TYPES, SANITY_TEST_TO_PACK, ONLY_UPLOAD_PACK_FILE_TYPES, SKIPPED_CONTENT_ITEMS__NOT_UNDER_PACK, XSOAR_SANITY_TEST_NAMES, - ALWAYS_INSTALLED_PACKS_MAPPING, MODELING_RULE_COMPONENT_FILES, XSIAM_COMPONENT_FILES) + ALWAYS_INSTALLED_PACKS_MAPPING, MODELING_RULE_COMPONENT_FILES, XSIAM_COMPONENT_FILES, + TEST_DATA_PATTERN) from Tests.scripts.collect_tests.exceptions import ( DeprecatedPackException, IncompatibleMarketplaceException, InvalidTestException, NonDictException, NonXsoarSupportedPackException, @@ -89,6 +90,7 @@ def __init__( skip_support_level_compatibility: bool = False, only_to_install: bool = False, only_to_upload: bool = False, + pack_to_reinstall: str | None = None, ): """ Collected test playbook, and/or a pack to install. @@ -113,6 +115,7 @@ def __init__( This is used when collecting a pack containing a content item, when their marketplace values differ. :param only_to_install: whether to collect the pack only to install it without upload to the bucket. :param only_to_upload: whether to collect the pack only to upload it to the bucket without install. + :param pack_to_reinstall: pack name to collect for reinstall test """ self.tests: set[str] = set() self.modeling_rules_to_test: set[str | Path] = set() @@ -120,6 +123,7 @@ def __init__( self.packs_to_upload: set[str] = set() self.version_range = None if version_range and version_range.is_default else version_range self.machines: tuple[Machine, ...] | None = None + self.packs_to_reinstall: set[str] = set() try: # raises if invalid @@ -179,6 +183,10 @@ def __init__( self.modeling_rules_to_test = {modeling_rule_to_test} logger.info(f'collected {modeling_rule_to_test=}, {reason} ({reason_description}, {version_range=})') + if pack_to_reinstall: + self.packs_to_reinstall = {pack_to_reinstall} + logger.info(f'collected {pack_to_reinstall=}, {reason} ({reason_description}, {version_range=})') + @staticmethod def _validate_collection( pack: str | None, @@ -268,6 +276,7 @@ def __add__(self, other: Optional['CollectionResult']) -> 'CollectionResult': result.packs_to_install = self.packs_to_install | other.packs_to_install # type: ignore[operator] result.packs_to_upload = self.packs_to_upload | other.packs_to_upload result.version_range = self.version_range | other.version_range if self.version_range else other.version_range + result.packs_to_reinstall = self.packs_to_reinstall | other.packs_to_reinstall return result @staticmethod @@ -602,6 +611,7 @@ def _collect_pack_for_modeling_rule( raise NothingToCollectException(changed_file_path, 'packs for Modeling Rules are only collected for XSIAM') pack = PACK_MANAGER.get_pack_metadata(pack_id) + pack_to_reinstall = None version_range = content_item_range \ if pack.version_range.is_default \ @@ -622,6 +632,10 @@ def _collect_pack_for_modeling_rule( # the modeling rule to test will be the containing directory of the modeling rule's component files relative_path_of_mr = PACK_MANAGER.relative_to_packs(changed_file_path) modeling_rule_to_test = relative_path_of_mr.parent + test_data_file_for_mr = list(changed_file_path.parent.glob(TEST_DATA_PATTERN)) + if test_data_file_for_mr: + pack_to_reinstall = pack_id + return CollectionResult( test=None, modeling_rule_to_test=modeling_rule_to_test, @@ -631,7 +645,8 @@ def _collect_pack_for_modeling_rule( reason_description=reason_description, conf=self.conf, id_set=self.id_set, - is_nightly=is_nightly + is_nightly=is_nightly, + pack_to_reinstall=pack_to_reinstall, ) def _collect_pack_for_xsiam_component( @@ -1357,12 +1372,14 @@ def output(result: CollectionResult | None): ) if result else () modeling_rules_to_test = (x.as_posix() if isinstance(x, Path) else str(x) for x in modeling_rules_to_test) machines = result.machines if result and result.machines else () + packs_to_reinstall_test = sorted(result.packs_to_reinstall, key=lambda x: x.lower()) if result else () test_str = '\n'.join(tests) packs_to_install_str = '\n'.join(packs_to_install) packs_to_upload_str = '\n'.join(packs_to_upload) modeling_rules_to_test_str = '\n'.join(modeling_rules_to_test) machine_str = ', '.join(sorted(map(str, machines))) + packs_to_reinstall_test_str = '\n'.join(packs_to_reinstall_test) logger.info(f'collected {len(tests)} test playbooks:\n{test_str}') logger.info(f'collected {len(packs_to_install)} packs to install:\n{packs_to_install_str}') @@ -1370,12 +1387,14 @@ def output(result: CollectionResult | None): num_of_modeling_rules = len(modeling_rules_to_test_str.split("\n")) logger.info(f'collected {num_of_modeling_rules} modeling rules to test:\n{modeling_rules_to_test_str}') logger.info(f'collected {len(machines)} machines: {machine_str}') + logger.info(f'collected {len(packs_to_reinstall_test)} packs to reinstall to test:\n{packs_to_reinstall_test_str}') PATHS.output_tests_file.write_text(test_str) PATHS.output_packs_file.write_text(packs_to_install_str) PATHS.output_packs_to_upload_file.write_text(packs_to_upload_str) PATHS.output_modeling_rules_to_test_file.write_text(modeling_rules_to_test_str) PATHS.output_machines_file.write_text(json.dumps({str(machine): (machine in machines) for machine in Machine})) + PATHS.output_packs_to_reinstall_test_file.write_text(packs_to_reinstall_test_str) class XPANSENightlyTestCollector(NightlyTestCollector): diff --git a/Tests/scripts/collect_tests/constants.py b/Tests/scripts/collect_tests/constants.py index 25c7e6a60513..f8c40d5adeb9 100644 --- a/Tests/scripts/collect_tests/constants.py +++ b/Tests/scripts/collect_tests/constants.py @@ -144,3 +144,5 @@ FileType.TRIGGER, FileType.CORRELATION_RULE, } + +TEST_DATA_PATTERN = '*_testdata.json' diff --git a/Tests/scripts/collect_tests/path_manager.py b/Tests/scripts/collect_tests/path_manager.py index bf6610edeb38..ff6fdb88f270 100644 --- a/Tests/scripts/collect_tests/path_manager.py +++ b/Tests/scripts/collect_tests/path_manager.py @@ -1,7 +1,7 @@ import logging import os from pathlib import Path -from typing import Iterable, Union +from collections.abc import Iterable from git import InvalidGitRepositoryError, Repo @@ -53,6 +53,7 @@ def __init__(self, content_path: Path): self.output_packs_file = PathManager.ARTIFACTS_PATH / 'content_packs_to_install.txt' self.output_packs_to_upload_file = PathManager.ARTIFACTS_PATH / 'content_packs_to_upload.txt' self.output_machines_file = PathManager.ARTIFACTS_PATH / 'filter_envs.json' + self.output_packs_to_reinstall_test_file = PathManager.ARTIFACTS_PATH / 'packs_reinstall_to_test.txt' def _glob_single(self, relative_path: str) -> set[Path]: """ @@ -67,16 +68,16 @@ def _glob_single(self, relative_path: str) -> set[Path]: if not path.exists(): logging.error(f'could not find {path} for calculating excluded paths') elif path.is_dir(): - result.update((_ for _ in path.rglob('*') if _.is_file())) + result.update(_ for _ in path.rglob('*') if _.is_file()) elif '*' in path.name: - result.update((_ for _ in path.rglob(path.name) if _.is_file())) + result.update(_ for _ in path.rglob(path.name) if _.is_file()) elif path.is_file() and '*' not in path.name: result.add(path) else: logging.error(f'could not glob {path} - unexpected case') return set(result) - def _glob(self, paths: Iterable[Union[str, Path]]) -> set[Path]: + def _glob(self, paths: Iterable[str | Path]) -> set[Path]: """ :param paths: to glob :return: set of all results diff --git a/Tests/scripts/reinstall_packs_on_cloud_instances.sh b/Tests/scripts/reinstall_packs_on_cloud_instances.sh new file mode 100644 index 000000000000..12cb31057411 --- /dev/null +++ b/Tests/scripts/reinstall_packs_on_cloud_instances.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +function exit_on_error { + if [ "${1}" -ne 0 ]; then + echo "ERROR: ${2}, exiting with code ${1}" 1>&2 + exit "${1}" + fi +} + +if [ -n "${CLOUD_SERVERS_FILE}" ]; then + CLOUD_SERVERS_PATH=$(cat "${CLOUD_SERVERS_FILE}") + echo "CLOUD_SERVERS_PATH is set to: ${CLOUD_SERVERS_PATH}" +else + exit_on_error 1 "CLOUD_SERVERS_FILE is not set" +fi + +if [ -n "${CLOUD_API_KEYS}" ]; then + echo "${CLOUD_API_KEYS}" > "cloud_api_keys.json" +fi + + + +python3 ./Tests/Marketplace/reinstall_packs.py --cloud_machine "${CLOUD_CHOSEN_MACHINE_IDS}" \ + --cloud_servers_path "${CLOUD_SERVERS_PATH}" --cloud_servers_api_keys "cloud_api_keys.json" \ + --non-removable-packs "${NON_REMOVABLE_PACKS}" --build-number "${CI_PIPELINE_ID}" \ + --packs_to_reinstall "$ARTIFACTS_FOLDER/packs_reinstall_to_test.txt" +exit_on_error $? "Failed to re-install packs for cloud machines:${CLOUD_CHOSEN_MACHINE_IDS}" + +echo "Successfully finished uninstalling packs from cloud machines" +exit 0 + diff --git a/Tests/scripts/uninstall_packs_and_reset_bucket_cloud.sh b/Tests/scripts/uninstall_packs_and_reset_bucket_cloud.sh index cf2fce9c0d07..ec8571b6c162 100644 --- a/Tests/scripts/uninstall_packs_and_reset_bucket_cloud.sh +++ b/Tests/scripts/uninstall_packs_and_reset_bucket_cloud.sh @@ -40,7 +40,8 @@ else python3 ./Tests/Marketplace/search_and_uninstall_pack.py --cloud_machine "${CLOUD_CHOSEN_MACHINE_IDS}" \ --cloud_servers_path "${CLOUD_SERVERS_PATH}" --cloud_servers_api_keys "cloud_api_keys.json" \ - --non-removable-packs "${NON_REMOVABLE_PACKS}" --one-by-one --build-number "${CI_PIPELINE_ID}" + --non-removable-packs "${NON_REMOVABLE_PACKS}" --one-by-one --build-number "${CI_PIPELINE_ID}" \ + --modeling_rules_to_test_files "${ARTIFACTS_FOLDER}/modeling_rules_to_test.txt" exit_on_error $? "Failed to uninstall packs from cloud machines:${CLOUD_CHOSEN_MACHINE_IDS}" echo "Successfully finished uninstalling packs from cloud machines"