Revert "Refactoring of support test scripts to enhance matter testing…

… infrast…" (#36016)

This reverts commit b1cd9fd.

docs/testing/python.md

@@ -25,7 +25,7 @@ Python tests located in src/python_testing
         section should include various parameters and their respective values,
         which will guide the test runner on how to execute the tests.
 -   All test classes inherit from `MatterBaseTest` in
-    [matter_testing.py](https://github.com/project-chip/connectedhomeip/blob/master/src/python_testing/matter_testing_infrastructure/chip/testing/matter_testing.py)
+    [matter_testing_support.py](https://github.com/project-chip/connectedhomeip/blob/master/src/python_testing/matter_testing_support.py)
     -   Support for commissioning using the python controller
     -   Default controller (`self.default_controller`) of type `ChipDeviceCtrl`
     -   `MatterBaseTest` inherits from the Mobly BaseTestClass
@@ -38,7 +38,7 @@ Python tests located in src/python_testing
         decorated with the @async_test_body decorator
 -   Use `ChipDeviceCtrl` to interact with the DUT
     -   Controller API is in `ChipDeviceCtrl.py` (see API doc in file)
-    -   Some support methods in `matter_testing.py`
+    -   Some support methods in `matter_testing_support.py`
 -   Use Mobly assertions for failing tests
 -   `self.step()` along with a `steps_*` method to mark test plan steps for cert
     tests
@@ -379,7 +379,7 @@ pai = await dev_ctrl.SendCommand(nodeid, 0, Clusters.OperationalCredentials.Comm
 ## Mobly helpers
 
 The test system is based on Mobly, and the
-[matter_testing.py](https://github.com/project-chip/connectedhomeip/blob/master/src/python_testing/matter_testing_infrastructure/chip/testing/matter_testing.py)
+[matter_testing_support.py](https://github.com/project-chip/connectedhomeip/blob/master/src/python_testing/matter_testing_support.py)
 class provides some helpers for Mobly integration.
 
 -   `default_matter_test_main`
@@ -561,11 +561,11 @@ these steps to set this up:
 
 ## Other support utilities
 
--   `basic_composition`
+-   `basic_composition_support`
     -   wildcard read, whole device analysis
 -   `CommissioningFlowBlocks`
     -   various commissioning support for core tests
--   `spec_parsing`
+-   `spec_parsing_support`
     -   parsing data model XML into python readable format
 
 # Running tests locally

scripts/spec_xml/generate_spec_xml.py

@@ -24,17 +24,26 @@
 from pathlib import Path
 
 import click
-from paths import Branch, get_chip_root, get_data_model_path, get_documentation_file_path, get_in_progress_defines
 
-# Use the get_in_progress_defines() function to fetch the in-progress defines
-CURRENT_IN_PROGRESS_DEFINES = get_in_progress_defines()
-
-# Replace hardcoded paths with dynamic paths using paths.py functions
-DEFAULT_CHIP_ROOT = get_chip_root()
-DEFAULT_OUTPUT_DIR_1_3 = get_data_model_path(Branch.V1_3)
-DEFAULT_OUTPUT_DIR_IN_PROGRESS = get_data_model_path(Branch.IN_PROGRESS)
-DEFAULT_OUTPUT_DIR_TOT = get_data_model_path(Branch.MASTER)
-DEFAULT_DOCUMENTATION_FILE = get_documentation_file_path()
+DEFAULT_CHIP_ROOT = os.path.abspath(
+    os.path.join(os.path.dirname(__file__), '..', '..'))
+DEFAULT_OUTPUT_DIR_1_3 = os.path.abspath(
+    os.path.join(DEFAULT_CHIP_ROOT, 'data_model', '1.3'))
+DEFAULT_OUTPUT_DIR_IN_PROGRESS = os.path.abspath(
+    os.path.join(DEFAULT_CHIP_ROOT, 'data_model', 'in_progress'))
+DEFAULT_OUTPUT_DIR_TOT = os.path.abspath(
+    os.path.join(DEFAULT_CHIP_ROOT, 'data_model', 'master'))
+DEFAULT_DOCUMENTATION_FILE = os.path.abspath(
+    os.path.join(DEFAULT_CHIP_ROOT, 'docs', 'spec_clusters.md'))
+
+# questions
+# is energy-calendar still in?
+# is heat-pump out? wasn't in 0.7
+# location-cluster - is this define gone now?
+# queuedpreset - is this define gone now?
+CURRENT_IN_PROGRESS_DEFINES = ['aliro', 'atomicwrites', 'battery-storage', 'device-location', 'e2e-jf', 'energy-calendar', 'energy-drlc',
+                               'energy-management', 'heat-pump', 'hrap-1', 'hvac', 'matter-fabric-synchronization', 'metering', 'secondary-net',
+                               'service-area-cluster', 'solar-power', 'tcp', 'water-heater', 'wifiSetup']
 
 
 def get_xml_path(filename, output_dir):
@@ -81,6 +90,7 @@ def make_asciidoc(target: str, include_in_progress: str, spec_dir: str, dry_run:
     '--include-in-progress',
     type=click.Choice(['All', 'None', 'Current']), default='All')
 def main(scraper, spec_root, output_dir, dry_run, include_in_progress):
+    # Clusters need to be scraped first because the cluster directory is passed to the device type directory
     if not output_dir:
         output_dir_map = {'All': DEFAULT_OUTPUT_DIR_TOT, 'None': DEFAULT_OUTPUT_DIR_1_3, 'Current': DEFAULT_OUTPUT_DIR_IN_PROGRESS}
         output_dir = output_dir_map[include_in_progress]
@@ -93,28 +103,30 @@ def main(scraper, spec_root, output_dir, dry_run, include_in_progress):
 
 def scrape_clusters(scraper, spec_root, output_dir, dry_run, include_in_progress):
     src_dir = os.path.abspath(os.path.join(spec_root, 'src'))
-    sdm_clusters_dir = os.path.abspath(os.path.join(src_dir, 'service_device_management'))
+    sdm_clusters_dir = os.path.abspath(
+        os.path.join(src_dir, 'service_device_management'))
     app_clusters_dir = os.path.abspath(os.path.join(src_dir, 'app_clusters'))
     dm_clusters_dir = os.path.abspath(os.path.join(src_dir, 'data_model'))
-    media_clusters_dir = os.path.abspath(os.path.join(app_clusters_dir, 'media'))
-
-    clusters_output_dir = os.path.join(output_dir, 'clusters')
+    media_clusters_dir = os.path.abspath(
+        os.path.join(app_clusters_dir, 'media'))
+    clusters_output_dir = os.path.abspath(os.path.join(output_dir, 'clusters'))
 
     if not os.path.exists(clusters_output_dir):
         os.makedirs(clusters_output_dir)
 
-    print('Generating main spec to get file include list - this may take a few minutes')
+    print('Generating main spec to get file include list - this make take a few minutes')
     main_out = make_asciidoc('pdf', include_in_progress, spec_root, dry_run)
-    print('Generating cluster spec to get file include list - this may take a few minutes')
+    print('Generating cluster spec to get file include list - this make take a few minutes')
     cluster_out = make_asciidoc('pdf-appclusters-book', include_in_progress, spec_root, dry_run)
 
     def scrape_cluster(filename: str) -> None:
         base = Path(filename).stem
         if base not in main_out and base not in cluster_out:
-            print(f'Skipping file: {base} as it is not compiled into the asciidoc')
+            print(f'skipping file: {base} as it is not compiled into the asciidoc')
             return
         xml_path = get_xml_path(filename, clusters_output_dir)
-        cmd = [scraper, 'cluster', '-i', filename, '-o', xml_path, '-nd']
+        cmd = [scraper, 'cluster', '-i', filename, '-o',
+               xml_path, '-nd']
         if include_in_progress == 'All':
             cmd.extend(['--define', 'in-progress'])
         elif include_in_progress == 'Current':
@@ -138,29 +150,33 @@ def scrape_all_clusters(dir: str, exclude_list: list[str] = []) -> None:
         tree = ElementTree.parse(f'{xml_path}')
         root = tree.getroot()
         cluster = next(root.iter('cluster'))
+        # If there's no cluster ID table, this isn't a cluster
         try:
             next(cluster.iter('clusterIds'))
         except StopIteration:
+            # If there's no cluster ID table, this isn't a cluster just some kind of intro adoc
             print(f'Removing file {xml_path} as it does not include any cluster definitions')
             os.remove(xml_path)
             continue
 
 
 def scrape_device_types(scraper, spec_root, output_dir, dry_run, include_in_progress):
-    device_type_dir = os.path.abspath(os.path.join(spec_root, 'src', 'device_types'))
-    device_types_output_dir = os.path.abspath(os.path.join(output_dir, 'device_types'))
+    device_type_dir = os.path.abspath(
+        os.path.join(spec_root, 'src', 'device_types'))
+    device_types_output_dir = os.path.abspath(
+        os.path.join(output_dir, 'device_types'))
     clusters_output_dir = os.path.abspath(os.path.join(output_dir, 'clusters'))
 
     if not os.path.exists(device_types_output_dir):
         os.makedirs(device_types_output_dir)
 
-    print('Generating device type library to get file include list - this may take a few minutes')
+    print('Generating device type library to get file include list - this make take a few minutes')
     device_type_output = make_asciidoc('pdf-devicelibrary-book', include_in_progress, spec_root, dry_run)
 
     def scrape_device_type(filename: str) -> None:
         base = Path(filename).stem
         if base not in device_type_output:
-            print(f'Skipping file: {filename} as it is not compiled into the asciidoc')
+            print(f'skipping file: {filename} as it is not compiled into the asciidoc')
             return
         xml_path = get_xml_path(filename, device_types_output_dir)
         cmd = [scraper, 'devicetype', '-c', '-cls', clusters_output_dir,

src/python_testing/MinimalRepresentation.py

@@ -17,10 +17,10 @@
 
 from dataclasses import dataclass, field
 
-from chip.testing.conformance import ConformanceDecision
-from chip.testing.global_attribute_ids import GlobalAttributeIds
-from chip.testing.matter_testing import MatterBaseTest, async_test_body, default_matter_test_main
 from chip.tlv import uint
+from conformance_support import ConformanceDecision
+from global_attribute_ids import GlobalAttributeIds
+from matter_testing_support import MatterBaseTest, async_test_body, default_matter_test_main
 from TC_DeviceConformance import DeviceConformanceTests
 
 

src/python_testing/TCP_Tests.py

@@ -33,7 +33,7 @@
 import chip.clusters as Clusters
 from chip import ChipDeviceCtrl
 from chip.interaction_model import InteractionModelError
-from chip.testing.matter_testing import MatterBaseTest, async_test_body, default_matter_test_main
+from matter_testing_support import MatterBaseTest, async_test_body, default_matter_test_main
 from mobly import asserts
 
 

src/python_testing/TC_ACE_1_2.py

@@ -42,7 +42,7 @@
 from chip.clusters.Attribute import EventReadResult, SubscriptionTransaction, TypedAttributePath
 from chip.exceptions import ChipStackError
 from chip.interaction_model import Status
-from chip.testing.matter_testing import MatterBaseTest, async_test_body, default_matter_test_main
+from matter_testing_support import MatterBaseTest, async_test_body, default_matter_test_main
 from mobly import asserts
 
 

src/python_testing/TC_ACE_1_3.py

@@ -38,7 +38,7 @@
 
 import chip.clusters as Clusters
 from chip.interaction_model import Status
-from chip.testing.matter_testing import MatterBaseTest, TestStep, async_test_body, default_matter_test_main
+from matter_testing_support import MatterBaseTest, TestStep, async_test_body, default_matter_test_main
 from mobly import asserts
 
 

src/python_testing/TC_ACE_1_4.py

@@ -42,7 +42,7 @@
 
 import chip.clusters as Clusters
 from chip.interaction_model import Status
-from chip.testing.matter_testing import MatterBaseTest, async_test_body, default_matter_test_main
+from matter_testing_support import MatterBaseTest, async_test_body, default_matter_test_main
 from mobly import asserts
 
 # This test requires several additional command line arguments

src/python_testing/TC_ACE_1_5.py

@@ -40,7 +40,7 @@
 import chip.clusters as Clusters
 from chip import ChipDeviceCtrl
 from chip.interaction_model import Status
-from chip.testing.matter_testing import MatterBaseTest, async_test_body, default_matter_test_main
+from matter_testing_support import MatterBaseTest, async_test_body, default_matter_test_main
 from mobly import asserts
 
 

src/python_testing/TC_ACL_2_11.py

@@ -42,13 +42,13 @@
 import queue
 
 import chip.clusters as Clusters
+from basic_composition_support import arls_populated
 from chip.clusters.Attribute import EventReadResult, SubscriptionTransaction, ValueDecodeFailure
 from chip.clusters.ClusterObjects import ALL_ACCEPTED_COMMANDS, ALL_ATTRIBUTES, ALL_CLUSTERS, ClusterEvent
 from chip.clusters.Objects import AccessControl
 from chip.clusters.Types import NullValue
 from chip.interaction_model import InteractionModelError, Status
-from chip.testing.basic_composition import arls_populated
-from chip.testing.matter_testing import MatterBaseTest, TestStep, async_test_body, default_matter_test_main
+from matter_testing_support import MatterBaseTest, TestStep, async_test_body, default_matter_test_main
 from mobly import asserts
 
 

src/python_testing/TC_ACL_2_2.py

@@ -32,7 +32,7 @@
 # === END CI TEST ARGUMENTS ===
 
 import chip.clusters as Clusters
-from chip.testing.matter_testing import MatterBaseTest, TestStep, async_test_body, default_matter_test_main
+from matter_testing_support import MatterBaseTest, TestStep, async_test_body, default_matter_test_main
 from mobly import asserts
 
 

src/python_testing/TC_AccessChecker.py

@@ -23,13 +23,13 @@
 from typing import Optional
 
 import chip.clusters as Clusters
+from basic_composition_support import BasicCompositionTests
 from chip.interaction_model import Status
-from chip.testing.basic_composition import BasicCompositionTests
-from chip.testing.global_attribute_ids import GlobalAttributeIds
-from chip.testing.matter_testing import (AttributePathLocation, ClusterPathLocation, MatterBaseTest, TestStep, async_test_body,
-                                         default_matter_test_main)
-from chip.testing.spec_parsing import XmlCluster, build_xml_clusters
 from chip.tlv import uint
+from global_attribute_ids import GlobalAttributeIds
+from matter_testing_support import (AttributePathLocation, ClusterPathLocation, MatterBaseTest, TestStep, async_test_body,
+                                    default_matter_test_main)
+from spec_parsing_support import XmlCluster, build_xml_clusters
 
 
 class AccessTestType(Enum):

src/python_testing/TC_BOOLCFG_2_1.py

@@ -36,7 +36,7 @@
 from operator import ior
 
 import chip.clusters as Clusters
-from chip.testing.matter_testing import MatterBaseTest, TestStep, async_test_body, default_matter_test_main
+from matter_testing_support import MatterBaseTest, TestStep, async_test_body, default_matter_test_main
 from mobly import asserts
 
 

src/python_testing/TC_BOOLCFG_3_1.py

@@ -36,7 +36,7 @@
 
 import chip.clusters as Clusters
 from chip.interaction_model import Status
-from chip.testing.matter_testing import MatterBaseTest, TestStep, async_test_body, default_matter_test_main
+from matter_testing_support import MatterBaseTest, TestStep, async_test_body, default_matter_test_main
 from mobly import asserts