diff --git a/scripts/tests/chiptest/yamltest_with_chip_repl_tester.py b/scripts/tests/chiptest/yamltest_with_chip_repl_tester.py
index 740c797493f97d..ce1db0330d18d8 100644
--- a/scripts/tests/chiptest/yamltest_with_chip_repl_tester.py
+++ b/scripts/tests/chiptest/yamltest_with_chip_repl_tester.py
@@ -72,7 +72,7 @@ def main(setup_code, yaml_path, node_id):
         chip.native.Init()
         chip_stack = ChipStack(chip_stack_storage.name)
         certificate_authority_manager = chip.CertificateAuthority.CertificateAuthorityManager(
-            chip_stack, chipStack.GetStorageManager())
+            chip_stack, chip_stack.GetStorageManager())
         certificate_authority_manager.LoadAuthoritiesFromStorage()
 
         if len(certificate_authority_manager.activeCaList) == 0:
@@ -96,7 +96,7 @@ def main(setup_code, yaml_path, node_id):
 
         # Parsing YAML test and setting up chip-repl yamltests runner.
         yaml = TestParser(yaml_path, None, clusters_definitions)
-        runner = ReplTestRunner(clusters_definitions, certificate_authority_manager)
+        runner = ReplTestRunner(clusters_definitions, certificate_authority_manager, dev_ctrl)
 
         # Executing and validating test
         for test_step in yaml.tests:
@@ -107,8 +107,9 @@ def main(setup_code, yaml_path, node_id):
                 decoded_response = runner.decode(response)
                 post_processing_result = test_step.post_process_response(decoded_response)
                 if not post_processing_result.is_success():
-                    # TODO figure out how we error out here
-                    pass
+                    exit(-2)
+            else:
+                exit(-2)
 
         runner.shutdown()
         # Tearing down chip stack. If not done in the correct order test will fail.
diff --git a/src/controller/python/chip/yaml/runner.py b/src/controller/python/chip/yaml/runner.py
index 1fa9d5b90c3feb..7bb59178ad7789 100644
--- a/src/controller/python/chip/yaml/runner.py
+++ b/src/controller/python/chip/yaml/runner.py
@@ -470,15 +470,13 @@ class ReplTestRunner:
     Uses ChipDeviceCtrl from chip-repl to execute parsed YAML TestSteps.
     '''
 
-    def __init__(self, test_spec_definition, certificate_authority_manager):
+    def __init__(self, test_spec_definition, certificate_authority_manager, alpha_dev_ctrl):
         self._test_spec_definition = test_spec_definition
         self._context = _ExecutionContext(data_model_lookup=PreDefinedDataModelLookup())
         self._certificate_authority_manager = certificate_authority_manager
         self._dev_ctrls = {}
 
-        ca_list = certificate_authority_manager.activeCaList
-        dev_ctrl = ca_list[0].adminList[0].NewController()
-        self._dev_ctrls['alpha'] = dev_ctrl
+        self._dev_ctrls['alpha'] = alpha_dev_ctrl
 
     def _invoke_action_factory(self, test_step, cluster: str):
         '''Creates cluster invoke action command from TestStep.