Fix python test

Signed-off-by: Zelin Hao <[email protected]>

src/test_workflow/integ_test/integ_test_suite.py

@@ -16,7 +16,6 @@
 from test_workflow.test_recorder.log_recorder import LogRecorder
 from test_workflow.test_recorder.test_recorder import TestRecorder
 from test_workflow.test_result.test_component_results import TestComponentResults
-from test_workflow.test_recorder.test_result_data import TestResultData
 
 
 class IntegTestSuite(abc.ABC):
@@ -87,5 +86,6 @@ def test_artifact_files(self) -> Dict[str, str]:
     def result_data(self) -> list:
         return self.test_result_data
 
+
 class InvalidTestConfigError(Exception):
     pass

src/test_workflow/test_recorder/test_recorder.py

@@ -9,12 +9,12 @@
 import os
 import shutil
 from typing import Any
+from urllib.parse import urljoin
 
 import yaml
 
 from test_workflow.test_recorder.log_recorder import LogRecorder
 from test_workflow.test_recorder.test_result_data import TestResultData
-from urllib.parse import urljoin
 
 
 class TestRecorder:
@@ -25,7 +25,7 @@ class TestRecorder:
     remote_cluster_logs: Any
     test_results_logs: Any
 
-    def __init__(self, test_run_id: str, test_type: str, tests_dir: str, base_path: str = "") -> None:
+    def __init__(self, test_run_id: str, test_type: str, tests_dir: str, base_path: str = None) -> None:
         self.test_run_id = test_run_id
         self.test_type = test_type
         self.location = os.path.join(tests_dir, str(self.test_run_id), self.test_type)
@@ -37,7 +37,7 @@ def __init__(self, test_run_id: str, test_type: str, tests_dir: str, base_path:
         self.base_path = base_path
 
     def _get_file_path(self, base_path: str, component_name: str, component_test_config: str) -> str:
-        if base_path.startswith("https"):
+        if base_path.startswith("https://"):
             file_path = urljoin(base_path, "/".join(["test-results", str(self.test_run_id), self.test_type, component_name, component_test_config]))
         else:
             file_path = self._create_base_folder_structure(component_name, component_test_config)
@@ -66,7 +66,7 @@ def _generate_yml(self, test_result_data: TestResultData, output_path: str) -> s
 
         opensearch_service_logs_path = os.path.join(local_cluster_logs_path, "opensearch-service-logs")
         opensearch_service_logs = self._get_list_files(opensearch_service_logs_path)
-        opensearch_service_logs = self._update_absolute_file_paths(opensearch_service_logs, base_file_path , os.path.join("local-cluster-logs", "opensearch-service-logs"))
+        opensearch_service_logs = self._update_absolute_file_paths(opensearch_service_logs, base_file_path, os.path.join("local-cluster-logs", "opensearch-service-logs"))
 
         test_result_file = components_files + [
             {
@@ -92,7 +92,7 @@ def _update_absolute_file_paths(self, files: list, base_path: str, relative_path
 
     # get a list of files within directory without folders.
     def _get_list_files(self, dir: str) -> list:
-        return [f for f in os.listdir(dir) if os.path.isfile(dir+'/'+f)]
+        return [f for f in os.listdir(dir) if os.path.isfile(dir + '/' + f)]
 
     def _copy_log_files(self, log_files: dict, dest_directory: str) -> None:
         if log_files:

tests/tests_test_workflow/test_integ_workflow/integ_test/test_integ_test_runner_opensearch.py

@@ -59,13 +59,16 @@ def test_with_integ_test(self, mock_temp: Mock, mock_test_recorder: Mock, mock_s
         mock_test_recorder_object = MagicMock()
         mock_test_recorder.return_value = mock_test_recorder_object
 
+        mock_suite_object.result_data.__iter__.return_value = [MagicMock(), MagicMock()]
+
         runner = IntegTestRunnerOpenSearch(self.args, self.test_manifest)
 
         # call the test target
         results = runner.run()
 
         self.assertEqual(results["sql"], mock_test_results)
 
+        mock_suite_object.result_data.__iter__.assert_called()
         mock_test_recorder_object.test_results_logs.save_test_result_data.assert_called()
 
         mock_suite.assert_called_once_with(

tests/tests_test_workflow/test_integ_workflow/integ_test/test_integ_test_runner_opensearch_dashboards.py

@@ -66,13 +66,16 @@ def test_with_integ_test(self, mock_temp: Mock, mock_test_recorder: Mock, mock_s
         mock_test_recorder_object = MagicMock()
         mock_test_recorder.return_value = mock_test_recorder_object
 
+        mock_suite_object.result_data.__iter__.return_value = [MagicMock(), MagicMock()]
+
         runner = IntegTestRunnerOpenSearchDashboards(self.args, self.test_manifest)
 
         # call the test target
         results = runner.run()
 
         self.assertEqual(results["sql"], mock_test_results)
 
+        mock_suite_object.result_data.__iter__.assert_called()
         mock_test_recorder_object.test_results_logs.save_test_result_data.assert_called()
 
         mock_suite.assert_called_once_with(

tests/tests_test_workflow/test_recorder/test_test_recorder.py

@@ -8,9 +8,11 @@
 
 import os
 import unittest
-from unittest.mock import MagicMock, Mock, patch, call, mock_open
-from test_workflow.test_recorder.test_recorder import TestRecorder, LocalClusterLogs
 from typing import Any
+from unittest.mock import MagicMock, Mock, call, mock_open, patch
+
+from test_workflow.test_recorder.test_recorder import LocalClusterLogs, TestRecorder
+
 
 @patch("os.makedirs")
 @patch("os.chdir")
@@ -66,25 +68,24 @@ def test_generate_yml(self, mock_local_cluster_logs: Mock, mock_remote_cluster_l
 
         mock_file_path = MagicMock()
         mock_file_path.return_value = "working-directory"
-        test_recorder._get_file_path = mock_file_path
+        test_recorder._get_file_path = mock_file_path   # type: ignore
 
         mock_list_files = MagicMock()
         mock_list_files.return_value = ["file1", "file2"]
-        test_recorder._get_list_files = mock_list_files
+        test_recorder._get_list_files = mock_list_files  # type: ignore
 
         mock_update_path = MagicMock()
         mock_update_path.return_value = ["working-directory/file1", "working-directory/file2"]
-        test_recorder._update_absolute_file_paths = mock_update_path
+        test_recorder._update_absolute_file_paths = mock_update_path    # type: ignore
 
         component_yml = test_recorder._generate_yml(mock_test_result_data, "working-directory")
 
-        mock_open.assert_has_calls([call("working-directory/sql.yml", 'w')])
-        mock_file_path.assert_called_once_with("", "sql", "with-security")
+        mock_open.assert_has_calls([call(os.path.join("working-directory", "sql.yml"), 'w')])
+        mock_file_path.assert_called_once_with(None, "sql", "with-security")
         mock_list_files.assert_called()
         mock_update_path.assert_called()
         self.assertEqual(component_yml, os.path.realpath("sql.yml"))
 
-
     @patch("test_workflow.test_recorder.test_recorder.TestResultsLogs")
     @patch("test_workflow.test_recorder.test_recorder.RemoteClusterLogs")
     @patch("test_workflow.test_recorder.test_recorder.LocalClusterLogs")
@@ -95,9 +96,10 @@ def test_update_absolute_file_paths(self, mock_local_cluster_logs: Mock, mock_re
             "working-directory",
             "https://ci.opensearch.org/ci/dbc/integ-test/"
         )
-        file_path = test_recorder._update_absolute_file_paths(["file1","file2"], "working-directory", "sub-directory")
+        file_path = test_recorder._update_absolute_file_paths(["file1", "file2"], "working-directory", "sub-directory")
         self.assertEqual(file_path, [os.path.join("working-directory", "sub-directory", "file1"), os.path.join("working-directory", "sub-directory", "file2")])
 
+
 class TestLocalClusterLogs(unittest.TestCase):
 
     def test(self) -> None: