Install xsaim packs with and without dataset (#30301)

* adding delete dataset call for cleaning; changing Qualys for testing; adding reinstall packs list to collect tests;

* fix pre-commit

* auth id should be str

* adding logs

* adding global varible to download conf

* adding more logging

* delete logs

* add reinstall script

* fix path

* test without un-installation

* test without un-installation

* uninstallatio is mandatory

* adding got here logging

* adding more logs

* remove some logs to not wait

* save more logs, ucomment

* fixes to trigger nightly

* revert unnessasary changes

* delete logs

* testing

* revert changes

* ruining modeling rules before installation

* save second check

* check nightly

* revert changes

* save changed from cr

* save changed from cr

* revert qualys changes

* pre-commit fix

.gitlab/ci/.gitlab-ci.on-push.yml

@@ -136,6 +136,7 @@ validate-content-conf:
     policy: pull-push
   variables:
     KUBERNETES_CPU_REQUEST: 2000m
+    EXTRACT_PRIVATE_TESTDATA: "true"
   needs: []
   stage: prepare-testing-bucket
   script:
@@ -587,25 +588,21 @@ slack-notify-on-merge:
     - ./Tests/scripts/test_modeling_rules.sh || EXIT_CODE=$?
     - section_end "Test Modeling Rules"
 
+    - section_start "Packs re-installation test"
+    - |
+      if [[ -z "${NIGHTLY}" && -s $ARTIFACTS_FOLDER/packs_reinstall_to_test.txt ]]; then
+        echo "Running the packs re-installation test."
+        ./Tests/scripts/reinstall_packs_on_cloud_instances.sh || EXIT_CODE=$?
+      fi
+    - section_end "Packs re-installation test"
+
     - !reference [.cloud-machine-information]
     - job-done
     - exit $EXIT_CODE
   after_script:
     - source .gitlab/helper_functions.sh
     - !reference [.unlock-machine]
 
-#xsiam_server_dev:
-#  extends:
-#    - .test_content_on_xsiam_server_instances_base
-#  rules:
-#    - if: '$CI_PIPELINE_SOURCE =~ /^(push|contrib)$/'
-#    - if: '$NIGHTLY'
-#      when: always
-#  variables:
-#    INSTANCE_ROLE: "XSIAM Master"
-#    PRODUCT_TYPE: "XSIAM"
-#    GCS_QUEUE_FILE: "queue-master"
-#    TEST_MACHINES_LIST: "test-machines-master"
 
 xsiam_server_ga:
   extends:

Tests/Marketplace/Tests/search_and_unistall_pack_test.py

@@ -1,3 +1,7 @@
+import json
+import tempfile
+from pathlib import Path
+
 import demisto_client
 import pytest
 
@@ -168,3 +172,47 @@ def success_handler(_):
 
     assert res == ret_value
     assert request_method.call_count == num_of_retries
+
+
+@pytest.mark.parametrize('status_code, expected_res', [
+    (200, True),
+    (599, True),
+    (400, False),
+])
+def test_handle_delete_datasets_by_testdata(requests_mock, status_code, expected_res):
+    """
+    Given
+        - Modeling rules with test data.
+    When
+        - Cleaning XSIAM machine
+    Then
+        - Verify the dataset deletion was results as expected.
+    """
+    test_data = {"data": [
+        {
+            "test_data_event_id": "some_uuid",
+            "vendor": "vendor",
+            "product": "product",
+            "dataset": "product_vendor_raw",
+            "event_data": {
+                "test": "test"
+            },
+            "expected_values": {
+                "test": "test"
+            }
+        }]
+    }
+    with tempfile.TemporaryDirectory() as temp_dir:
+        test_data_file = Path("Packs/MyPack/ModelingRules/MR/MR_testdata.json")
+        test_data_file.parent.mkdir(parents=True, exist_ok=True)
+        test_data_file.write_text(json.dumps(test_data))
+
+        with open(f'{temp_dir}/modeling_rules_to_test.txt', 'w') as mr_to_test:
+            mr_to_test.write("MyPack/ModelingRules/MR")
+
+        dataset_names = script.get_datasets_to_delete(modeling_rules_file=f'{temp_dir}/modeling_rules_to_test.txt')
+        requests_mock.post(f'{BASE_URL}/public_api/v1/xql/delete_dataset', status_code=status_code)
+        res = script.delete_datasets_by_testdata(base_url=BASE_URL, api_key=API_KEY, auth_id="1",
+                                                 dataset_names=dataset_names)
+        test_data_file.unlink()
+    assert res == expected_res

Tests/Marketplace/common.py

@@ -2,8 +2,9 @@
 import time
 from typing import Any
 from collections.abc import Callable
-
+from urllib.parse import urljoin
 import demisto_client
+import requests
 from demisto_client.demisto_api.rest import ApiException
 from urllib3.exceptions import HTTPError, HTTPWarning
 
@@ -163,3 +164,104 @@ def wait_until_not_updating(client: demisto_client,
                 return False
     logging.info(f"Exiting after exhausting the allowed time:{maximum_time_to_wait} seconds")
     return False
+
+
+def send_api_request_with_retries(
+    base_url: str,
+    retries_message: str,
+    exception_message: str,
+    success_message: str,
+    prior_message: str,
+    endpoint: str,
+    method: str,
+    params: dict | None = None,
+    headers: dict | None = None,
+    request_timeout: int | None = None,
+    accept: str = 'application/json',
+    body: Any | None = None,
+    attempts_count: int = 5,
+    sleep_interval: int = 60,
+    should_try_handler: Callable[[Any], bool] | None = None,
+    success_handler: Callable[[Any], Any] | None = None,
+    api_exception_handler: Callable[[ApiException, int], Any] | None = None,
+    http_exception_handler: Callable[[HTTPError | HTTPWarning], Any] | None = None
+):
+    """
+    Args:
+        body: request body.
+        base_url: base url to send request
+        headers: headers for the request.
+        success_message:  message to print after success response.
+        retries_message: message to print after failure when we have more attempts.
+        exception_message: message to print when we get and exception that is not API or HTTP exception.
+        prior_message: message to print when a new retry is made.
+        endpoint: endpoint to send request to.
+        method: HTTP method to use.
+        params: api request params.
+        request_timeout: request param.
+        accept: request param.
+        attempts_count: number of total attempts made.
+        sleep_interval: sleep interval between attempts.
+        should_try_handler: a method to determine if we should send the next request.
+        success_handler: a method to run in case of successful request (according to the response status).
+        api_exception_handler: a method to run in case of api exception.
+        http_exception_handler: a method to run in case of http exception
+
+    Returns: True if the request succeeded
+
+    """
+    headers = headers if headers else {}
+    headers['Accept'] = accept
+    response = None
+    url_path = urljoin(base_url, endpoint)
+    try:
+        for attempts_left in range(attempts_count - 1, -1, -1):
+            try:
+                if should_try_handler and not should_try_handler(response):
+                    # if the method exist, and we should not try again.
+                    return True
+
+                logging.info(f"{prior_message}, attempt: {attempts_count - attempts_left}/{attempts_count}")
+                response = requests.request(
+                    method=method,
+                    url=url_path,
+                    verify=False,
+                    params=params,
+                    data=body,
+                    headers=headers,
+                    timeout=request_timeout,
+                )
+                if 200 <= response.status_code < 300 and response.status_code != 204:
+                    logging.debug(f"Got successful response: {response.status_code=},  {response.content=}.")
+                    logging.info(success_message)
+                    if success_handler:
+                        return success_handler(response)
+
+                    return True
+
+                else:
+                    err = f"Got {response.status_code=}, {response.headers=}, {response.content=}"
+
+                if not attempts_left:
+                    raise Exception(err)
+
+                logging.warning(f"Got error: {err}")
+
+            except ApiException as ex:
+                if api_exception_handler:
+                    api_exception_handler(ex, attempts_left)
+                if not attempts_left:
+                    raise Exception(f"Got status {ex.status} from server, message: {ex.body}, headers: {ex.headers}") from ex
+                logging.debug(f"Process failed, got error {ex}")
+            except (HTTPError, HTTPWarning) as http_ex:
+                if http_exception_handler:
+                    http_exception_handler(http_ex)
+                if not attempts_left:
+                    raise Exception("Failed to perform http request to the server") from http_ex
+                logging.debug(f"Process failed, got error {http_ex}")
+
+            logging.debug(f"{retries_message}, sleeping for {sleep_interval} seconds.")
+            time.sleep(sleep_interval)
+    except Exception as e:
+        logging.exception(f'{exception_message}. Additional info: {str(e)}')
+    return False

Tests/Marketplace/reinstall_packs.py

@@ -0,0 +1,106 @@
+import argparse
+import os
+import sys
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from pathlib import Path
+import demisto_client
+from Tests.Marketplace.configure_and_install_packs import search_and_install_packs_and_their_dependencies
+from Tests.Marketplace.search_and_uninstall_pack import uninstall_pack
+from Tests.configure_and_test_integration_instances import CloudBuild, get_custom_user_agent
+from Tests.scripts.utils import logging_wrapper as logging
+from Tests.scripts.utils.log_util import install_logging
+
+
+def options_handler() -> argparse.Namespace:
+    """
+    Returns: options parsed from input arguments.
+    """
+    parser = argparse.ArgumentParser(description='Utility for testing re-installation XSIAM packs.')
+    parser.add_argument('--cloud_machine', help='cloud machine to use, if it is cloud build.')
+    parser.add_argument('--cloud_servers_path', help='Path to secret cloud server metadata file.')
+    parser.add_argument('--cloud_servers_api_keys', help='Path to the file with cloud Servers api keys.')
+    parser.add_argument('--non-removable-packs', help='List of packs that cant be removed.')
+    parser.add_argument('--build-number', help='CI job number where the instances were created', required=True)
+    parser.add_argument('--packs_to_reinstall', help='List of packs to check its re-installation.', required=True)
+
+    options = parser.parse_args()
+
+    return options
+
+
+def reinstall_packs(options: argparse.Namespace, cloud_machine: str) -> bool:
+    """
+    Packs re-installation test. Uninstall and then install packs from options.packs_to_reinstall list.
+
+    Args:
+        options: Script arguments.
+        cloud_machine (str): Cloud machine name to test on.
+
+    Returns:
+        Boolean - If the operation succeeded.
+    """
+    success = True
+    api_key, _, base_url, xdr_auth_id = CloudBuild.get_cloud_configuration(cloud_machine,
+                                                                           options.cloud_servers_path,
+                                                                           options.cloud_servers_api_keys)
+
+    client = demisto_client.configure(base_url=base_url,
+                                      verify_ssl=False,
+                                      api_key=api_key,
+                                      auth_id=xdr_auth_id)
+
+    client.api_client.user_agent = get_custom_user_agent(options.build_number)
+    host = client.api_client.configuration.host.replace('https://api-', 'https://')  # disable-secrets-detection
+
+    logging.debug(f'Setting user agent on client to: {client.api_client.user_agent}')
+
+    non_removable_packs = options.non_removable_packs.split(',')
+    packs_to_reinstall_path = Path(options.packs_to_reinstall)
+    packs_to_reinstall = packs_to_reinstall_path.read_text().split('\n')
+
+    for pack in packs_to_reinstall:
+        if pack in non_removable_packs:
+            continue
+        successful_uninstall, _ = uninstall_pack(client, pack)
+        _, successful_install = search_and_install_packs_and_their_dependencies(
+            pack_ids=[pack],
+            client=client,
+            hostname=host,
+            multithreading=False,
+            production_bucket=False
+        )
+        success &= successful_uninstall & successful_install
+
+    return success
+
+
+def main():
+    install_logging('reinstall_packs_check.log', logger=logging)
+
+    # In Cloud, We don't use demisto username
+    os.environ.pop('DEMISTO_USERNAME', None)
+
+    options = options_handler()
+    logging.info(f'Starting reinstall test for CLOUD servers:{options.cloud_machine}.')
+    cloud_machines: list[str] = list(filter(None, options.cloud_machine.split(',')))
+    success = True
+    with ThreadPoolExecutor(max_workers=len(cloud_machines), thread_name_prefix='clean-machine') as executor:
+        futures = [
+            executor.submit(reinstall_packs, options, cloud_machine)
+            for cloud_machine in cloud_machines
+        ]
+        for future in as_completed(futures):
+            try:
+                success &= future.result()
+            except Exception as ex:
+                logging.exception(f'Failed to run reinstall packs test. Additional info: {str(ex)}')
+                success = False
+
+    if not success:
+        logging.error('Failed to reinstall packs.')
+        sys.exit(2)
+    logging.info('Finished reinstall packs test successfully.')
+
+
+if __name__ == '__main__':
+    main()