Initial automation of integration tests

.gitignore

@@ -6,7 +6,7 @@ venv/
 
 *.whl
 
-test_apps/build
-test_apps/install
+integration_tests/build
+integration_tests/install
 
 build/

example_apps/call_method_tcp.py

@@ -1,6 +1,7 @@
 import asyncio
 import ipaddress
 import logging
+import sys
 
 from someipy import TransportLayerProtocol
 from someipy.client_service_instance import (
@@ -14,7 +15,7 @@
 
 SD_MULTICAST_GROUP = "224.224.224.245"
 SD_PORT = 30490
-INTERFACE_IP = "127.0.0.1"
+DEFAULT_INTERFACE_IP = "127.0.0.1"  # Default IP if not provided
 
 SAMPLE_SERVICE_ID = 0x1234
 SAMPLE_INSTANCE_ID = 0x5678
@@ -26,11 +27,19 @@ async def main():
     # It's possible to configure the logging level of the someipy library, e.g. logging.INFO, logging.DEBUG, logging.WARN, ..
     set_someipy_log_level(logging.DEBUG)
 
+    # Get interface ip to use from command line argument (--interface_ip) or use default
+    interface_ip = DEFAULT_INTERFACE_IP
+    for i, arg in enumerate(sys.argv):
+        if arg == "--interface_ip":
+            if i + 1 < len(sys.argv):
+                interface_ip = sys.argv[i + 1]
+                break
+
     # Since the construction of the class ServiceDiscoveryProtocol is not trivial and would require an async __init__ function
     # use the construct_service_discovery function
     # The local interface IP address needs to be passed so that the src-address of all SD UDP packets is correctly set
     service_discovery = await construct_service_discovery(
-        SD_MULTICAST_GROUP, SD_PORT, INTERFACE_IP
+        SD_MULTICAST_GROUP, SD_PORT, interface_ip
     )
 
     addition_service = (
@@ -44,7 +53,7 @@ async def main():
     client_instance_addition = await construct_client_service_instance(
         service=addition_service,
         instance_id=SAMPLE_INSTANCE_ID,
-        endpoint=(ipaddress.IPv4Address(INTERFACE_IP), 3002),
+        endpoint=(ipaddress.IPv4Address(interface_ip), 3002),
         ttl=5,
         sd_sender=service_discovery,
         protocol=TransportLayerProtocol.TCP,

example_apps/call_method_udp.py

@@ -1,6 +1,8 @@
 import asyncio
+import datetime
 import ipaddress
 import logging
+import sys
 
 from someipy import TransportLayerProtocol
 from someipy.client_service_instance import (
@@ -14,7 +16,7 @@
 
 SD_MULTICAST_GROUP = "224.224.224.245"
 SD_PORT = 30490
-INTERFACE_IP = "127.0.0.1"
+DEFAULT_INTERFACE_IP = "127.0.0.1"  # Default IP if not provided
 
 SAMPLE_SERVICE_ID = 0x1234
 SAMPLE_INSTANCE_ID = 0x5678
@@ -26,11 +28,21 @@ async def main():
     # It's possible to configure the logging level of the someipy library, e.g. logging.INFO, logging.DEBUG, logging.WARN, ..
     set_someipy_log_level(logging.DEBUG)
 
+    # Get interface ip to use from command line argument (--interface_ip) or use default
+    interface_ip = DEFAULT_INTERFACE_IP
+    for i, arg in enumerate(sys.argv):
+        if arg == "--interface_ip":
+            if i + 1 < len(sys.argv):
+                interface_ip = sys.argv[i + 1]
+                break
+
+    print(interface_ip)
+
     # Since the construction of the class ServiceDiscoveryProtocol is not trivial and would require an async __init__ function
     # use the construct_service_discovery function
     # The local interface IP address needs to be passed so that the src-address of all SD UDP packets is correctly set
     service_discovery = await construct_service_discovery(
-        SD_MULTICAST_GROUP, SD_PORT, INTERFACE_IP
+        SD_MULTICAST_GROUP, SD_PORT, interface_ip
     )
 
     addition_service = (
@@ -44,7 +56,7 @@ async def main():
     client_instance_addition = await construct_client_service_instance(
         service=addition_service,
         instance_id=SAMPLE_INSTANCE_ID,
-        endpoint=(ipaddress.IPv4Address(INTERFACE_IP), 3002),
+        endpoint=(ipaddress.IPv4Address(interface_ip), 3002),
         ttl=5,
         sd_sender=service_discovery,
         protocol=TransportLayerProtocol.UDP,
@@ -56,7 +68,6 @@ async def main():
 
     try:
         while True:
-
             method_parameter = Addends(addend1=1, addend2=2)
             method_success, method_result = await client_instance_addition.call_method(
                 SAMPLE_METHOD_ID, method_parameter.serialize()
@@ -77,7 +88,7 @@ async def main():
             elif method_success == MethodResult.SERVICE_NOT_FOUND:
                 print("Service not yet available..")
 
-            await asyncio.sleep(2)
+            await asyncio.sleep(1)
     except asyncio.CancelledError:
         print("Shutdown..")
     finally:

example_apps/offer_method_tcp.py

@@ -1,6 +1,7 @@
 import asyncio
 import ipaddress
 import logging
+import sys
 from typing import Tuple
 
 from someipy import TransportLayerProtocol
@@ -13,7 +14,7 @@
 
 SD_MULTICAST_GROUP = "224.224.224.245"
 SD_PORT = 30490
-INTERFACE_IP = "127.0.0.1"
+DEFAULT_INTERFACE_IP = "127.0.0.1"  # Default IP if not provided
 
 SAMPLE_SERVICE_ID = 0x1234
 SAMPLE_INSTANCE_ID = 0x5678
@@ -50,11 +51,19 @@ async def main():
     # It's possible to configure the logging level of the someipy library, e.g. logging.INFO, logging.DEBUG, logging.WARN, ..
     set_someipy_log_level(logging.DEBUG)
 
+    # Get interface ip to use from command line argument (--interface_ip) or use default
+    interface_ip = DEFAULT_INTERFACE_IP
+    for i, arg in enumerate(sys.argv):
+        if arg == "--interface_ip":
+            if i + 1 < len(sys.argv):
+                interface_ip = sys.argv[i + 1]
+                break
+
     # Since the construction of the class ServiceDiscoveryProtocol is not trivial and would require an async __init__ function
     # use the construct_service_discovery function
     # The local interface IP address needs to be passed so that the src-address of all SD UDP packets is correctly set
     service_discovery = await construct_service_discovery(
-        SD_MULTICAST_GROUP, SD_PORT, INTERFACE_IP
+        SD_MULTICAST_GROUP, SD_PORT, interface_ip
     )
 
     addition_method = Method(id=SAMPLE_METHOD_ID, method_handler=add_method_handler)
@@ -72,7 +81,7 @@ async def main():
         addition_service,
         instance_id=SAMPLE_INSTANCE_ID,
         endpoint=(
-            ipaddress.IPv4Address(INTERFACE_IP),
+            ipaddress.IPv4Address(interface_ip),
             3000,
         ),  # src IP and port of the service
         ttl=5,

example_apps/offer_method_udp.py

@@ -1,6 +1,7 @@
 import asyncio
 import ipaddress
 import logging
+import sys
 from typing import Tuple
 
 from someipy import TransportLayerProtocol
@@ -13,7 +14,7 @@
 
 SD_MULTICAST_GROUP = "224.224.224.245"
 SD_PORT = 30490
-INTERFACE_IP = "127.0.0.1"
+DEFAULT_INTERFACE_IP = "127.0.0.1"  # Default IP if not provided
 
 SAMPLE_SERVICE_ID = 0x1234
 SAMPLE_INSTANCE_ID = 0x5678
@@ -50,11 +51,19 @@ async def main():
     # It's possible to configure the logging level of the someipy library, e.g. logging.INFO, logging.DEBUG, logging.WARN, ..
     set_someipy_log_level(logging.DEBUG)
 
+    # Get interface ip to use from command line argument (--interface_ip) or use default
+    interface_ip = DEFAULT_INTERFACE_IP
+    for i, arg in enumerate(sys.argv):
+        if arg == "--interface_ip":
+            if i + 1 < len(sys.argv):
+                interface_ip = sys.argv[i + 1]
+                break
+
     # Since the construction of the class ServiceDiscoveryProtocol is not trivial and would require an async __init__ function
     # use the construct_service_discovery function
     # The local interface IP address needs to be passed so that the src-address of all SD UDP packets is correctly set
     service_discovery = await construct_service_discovery(
-        SD_MULTICAST_GROUP, SD_PORT, INTERFACE_IP
+        SD_MULTICAST_GROUP, SD_PORT, interface_ip
     )
 
     addition_method = Method(id=SAMPLE_METHOD_ID, method_handler=add_method_handler)
@@ -72,7 +81,7 @@ async def main():
         addition_service,
         instance_id=SAMPLE_INSTANCE_ID,
         endpoint=(
-            ipaddress.IPv4Address(INTERFACE_IP),
+            ipaddress.IPv4Address(interface_ip),
             3000,
         ),  # src IP and port of the service
         ttl=5,

example_apps/receive_events_tcp.py

@@ -1,6 +1,7 @@
 import asyncio
 import ipaddress
 import logging
+import sys
 
 from someipy import ServiceBuilder, EventGroup, TransportLayerProtocol, SomeIpMessage
 from someipy.service_discovery import construct_service_discovery
@@ -10,7 +11,9 @@
 
 SD_MULTICAST_GROUP = "224.224.224.245"
 SD_PORT = 30490
-INTERFACE_IP = "127.0.0.1"
+DEFAULT_INTERFACE_IP = (
+    "127.0.0.1"  # default interface ip if not passed to the application
+)
 
 SAMPLE_SERVICE_ID = 0x1234
 SAMPLE_INSTANCE_ID = 0x5678
@@ -40,11 +43,19 @@ async def main():
     # It's possible to configure the logging level of the someipy library, e.g. logging.INFO, logging.DEBUG, logging.WARN, ..
     set_someipy_log_level(logging.DEBUG)
 
+    # Get interface ip to use from command line argument (--interface_ip) or use default
+    interface_ip = DEFAULT_INTERFACE_IP
+    for i, arg in enumerate(sys.argv):
+        if arg == "--interface_ip":
+            if i + 1 < len(sys.argv):
+                interface_ip = sys.argv[i + 1]
+                break
+
     # Since the construction of the class ServiceDiscoveryProtocol is not trivial and would require an async __init__ function
     # use the construct_service_discovery function
     # The local interface IP address needs to be passed so that the src-address of all SD UDP packets is correctly set
     service_discovery = await construct_service_discovery(
-        SD_MULTICAST_GROUP, SD_PORT, INTERFACE_IP
+        SD_MULTICAST_GROUP, SD_PORT, interface_ip
     )
 
     # 1. For receiving events use a ClientServiceInstance. Since the construction of the class ClientServiceInstance is not
@@ -67,7 +78,7 @@ async def main():
     service_instance_temperature = await construct_client_service_instance(
         service=temperature_service,
         instance_id=SAMPLE_INSTANCE_ID,
-        endpoint=(ipaddress.IPv4Address(INTERFACE_IP), 3002),
+        endpoint=(ipaddress.IPv4Address(interface_ip), 3002),
         ttl=5,
         sd_sender=service_discovery,
         protocol=TransportLayerProtocol.TCP,

integration_tests/README.md

@@ -0,0 +1,24 @@
+# Integration Test Applications
+
+The integration test applications use the vsomeip library in order to stimulate the someipy applications.
+
+Each test application has a corresponding someipy example application, which are started together. The test applications can be either started manually or run automatically using the scripts `run_all.py` in `automated_tests`. The script will launch each pair of applications (a someipy app and a vsomeip app). After running both applications, the script will evaluate the logfiles.
+
+## Build Test Applications using CMake
+
+```bash
+rm -rf build install && mkdir -p build && cd build && cmake .. && make && make install  && cd ..
+```
+
+## Setup Python Pip Package from Source
+
+```bash
+python3 -m pip install -e .
+```
+
+### Network Setup Linux
+
+```bash
+sudo ip addr add 127.0.0.2/8 dev lo
+sudo ip addr add 224.224.224.245 dev lo autojoin
+```

integration_tests/automated_tests/call_method_tcp.py

@@ -0,0 +1,39 @@
+from test_base import TestBase
+
+
+class TestCallMethodTcp(TestBase):
+
+    def __init__(self, repository, ld_library_path=None, interface_ip="127.0.0.1"):
+        super().__init__()
+
+        self.ld_library_path = ld_library_path
+        self.vsomeip_app = [
+            f"{repository}/test_apps/install/call_method_tcp/call_method_tcp"
+        ]
+        self.someipy_app = [
+            "python3",
+            f"{repository}/example_apps/call_method_tcp.py",
+            f"--interface_ip {interface_ip}",
+        ]
+        self.vsomeip_config = (
+            f"{repository}/test_apps/install/call_method_tcp/vsomeip-client.json"
+        )
+
+    def evaluate(self) -> bool:
+        method_calls = 0
+        responses = 0
+        for l in self.output_someipy_app:
+            if "Received result for method" in l:
+                responses += 1
+            if "Try to call method" in l:
+                method_calls += 1
+
+        print(f"Method calls: {method_calls}. Responses: {responses}")
+        difference = method_calls - responses
+        tolerance = max(0.1 * method_calls, 1)
+        if abs(difference) <= tolerance and method_calls > 0 and responses > 0:
+            return True
+        else:
+            print(f"Method calls: {method_calls}. Responses: {responses}")
+            self.print_outputs()
+            return False

integration_tests/automated_tests/call_method_udp.py

@@ -0,0 +1,39 @@
+from test_base import TestBase
+
+
+class TestCallMethodUdp(TestBase):
+
+    def __init__(self, repository, ld_library_path=None, interface_ip="127.0.0.1"):
+        super().__init__()
+
+        self.ld_library_path = ld_library_path
+        self.vsomeip_app = [
+            f"{repository}/test_apps/install/call_method_udp/call_method_udp"
+        ]
+        self.someipy_app = [
+            "python3",
+            f"{repository}/example_apps/call_method_udp.py",
+            f"--interface_ip {interface_ip}",
+        ]
+        self.vsomeip_config = (
+            f"{repository}/test_apps/install/call_method_udp/vsomeip-client.json"
+        )
+
+    def evaluate(self) -> bool:
+        method_calls = 0
+        responses = 0
+        for l in self.output_someipy_app:
+            if "Received result for method" in l:
+                responses += 1
+            if "Try to call method" in l:
+                method_calls += 1
+
+        print(f"Method calls: {method_calls}. Responses: {responses}")
+        difference = method_calls - responses
+        tolerance = max(0.1 * method_calls, 1)
+        if abs(difference) <= tolerance and method_calls > 0 and responses > 0:
+            return True
+        else:
+            print(f"Method calls: {method_calls}. Responses: {responses}")
+            self.print_outputs()
+            return False