Enable MITM proxy for requestor probes

goth/address.py

@@ -95,12 +95,29 @@ def _with_default(self, mapping: Mapping[str, object]):
     default={"port": YAGNA_REST_PORT, "protocol": YAGNA_REST_PROTOCOL},
 )
 
-# Range of ports on the host which can be mapped to yagna daemons' REST API port
+# Range of ports on the host to which yagna daemon ports in containers can be mapped.
+# Port $YAGNA_REST_PORT on N-th started yagna container is mapped to port
+# $HOST_REST_PORT_START + (N-1) on the host.
+
 # NOTE: This variable is used in `nginx.conf` file in the proxy container:
 HOST_REST_PORT_START = 6001
 # NOTE: This variable is used in `nginx.conf` file in the proxy container:
 HOST_REST_PORT_END = 6100
 
+# Ports in the range $HOST_REST_PORT_START .. $HOST_REST_PORT_END are also used
+# by the nginx-proxy container. A request made to port $HOST_REST_PORT_START + (N-1)
+# on nginx-proxy container is forwarded to the MITM proxy (that runs on host)
+# which then forwards the request to $YAGNA_REST_PORT on the N-th yagna container.
+
+# To enable access to those nginx-proxy ports from outside of the Docker network
+# on non-Linux systems, we map ports $HOST_REST_PORT_START .. $HOST_REST_PORT_END on
+# the nginx-proxy to ports ($HOST_REST_PORT_START + O) .. ($HOST_REST_PORT_END + O)
+# on the host, where the offset O is the value of the variable HOST_NGINX_PORT_OFFSET
+# defined below.
+# NOTE: this value has to be consistent with a port mapping definition for the
+# `nginx-proxy` container in `docker-compose.yml`.
+HOST_NGINX_PORT_OFFSET = 10000
+
 
 # Port used by the mitmproxy instance
 # NOTE: This variable is used in `nginx.conf` file in the proxy container:

goth/api_monitor/router_addon.py

@@ -55,20 +55,19 @@ def request(self, flow: HTTPFlow) -> None:
             node_name = self._node_names[remote_addr]
 
             if server_port == YAGNA_REST_PORT:
-                # It's a provider agent calling a yagna daemon
-                # Since we assume that the provider agent runs on the same container
-                # as the provider daemon, we route this request to that container's
-                # host-mapped daemon port
+                # This should be a request from an agent running in a yagna container
+                # calling that container's daemon. We route this request to that
+                # container's host-mapped daemon port.
                 req.host = "127.0.0.1"
                 req.port = self._ports[remote_addr][server_port]
                 req.headers[CALLER_HEADER] = f"{node_name}:agent"
                 req.headers[CALLEE_HEADER] = f"{node_name}:daemon"
 
             elif HOST_REST_PORT_START <= server_port <= HOST_REST_PORT_END:
-                # It's a requestor agent calling a yagna daemon.
-                # We use localhost as the address together with the original port,
-                # since each daemon has its API port mapped to a port on the host
-                # chosen from the specified range.
+                # This should be a request from an agent running on the Docker host
+                # calling a yagna daemon in a container. We use localhost as the address
+                # together with the original port, since each daemon has its API port
+                # mapped to a port on the host chosen from the specified range.
                 req.host = "127.0.0.1"
                 req.port = server_port
                 req.headers[CALLER_HEADER] = f"{node_name}:agent"

goth/configuration.py

@@ -82,6 +82,7 @@ def add_node(
             subnet="goth",
             volumes=volumes,
             payment_id=self._id_pool.get_id(),
+            use_proxy=use_proxy,
             **kwargs,
         )
 

goth/default-assets/docker/docker-compose.yml

@@ -13,7 +13,16 @@ services:
         # harness on the host machine
         image: proxy-nginx
         ports:
-            - "6000:6000"
+            # Requests to ports 6001-6100 in proxy-nginx are forwarded
+            # to the MITM proxy started by the test runner, and further
+            # to yagna API port (usually 6000) in yagna containers:
+            # request to port 6001 is forwarded to (yagna API port in)
+            # the first yagna container, request to port 6002 -- to
+            # the second one, and so on.
+            # To make these ports available from Docker host (on some
+            # systems, Docker network may be unreachable from the host)
+            # we map them to ports 16001-16100 on the host.
+            - "16001-16100:6001-6100"
 
     ethereum:
         image: docker.pkg.github.com/golemfactory/gnt2/gnt2-docker-yagna:483c6f241edd

goth/default-assets/goth-config.yml

@@ -52,6 +52,7 @@ nodes:
 
   - name: "requestor"
     type: "Requestor"
+    use-proxy: True
 
   - name: "provider-1"
     type: "VM-Wasm-Provider"

goth/node.py

@@ -4,6 +4,7 @@
 
 from goth.address import (
     ACTIVITY_API_URL,
+    MARKET_API_URL,
     PAYMENT_API_URL,
     ROUTER_HOST,
     ROUTER_PORT,
@@ -45,7 +46,12 @@ def node_environment(
 
     if rest_api_url_base:
         agent_env = {
+            # Setting URLs for all three APIs has the same effect as setting
+            # YAGNA_API_URL to YAGNA_REST_URL.substitute(base=rest_api_url_base).
+            # We set all three separately so it's easier to selectively disable
+            # proxy for certain APIs (if a need to do so arises).
             "YAGNA_ACTIVITY_URL": ACTIVITY_API_URL.substitute(base=rest_api_url_base),
+            "YAGNA_MARKET_URL": MARKET_API_URL.substitute(base=rest_api_url_base),
             "YAGNA_PAYMENT_URL": PAYMENT_API_URL.substitute(base=rest_api_url_base),
         }
         node_env.update(agent_env)

goth/runner/__init__.py

@@ -42,6 +42,14 @@
 ProbeType = TypeVar("ProbeType", bound=Probe)
 
 
+PROXY_NGINX_SERVICE_NAME = "proxy-nginx"
+"""Name of the nginx proxy service in the Docker network.
+
+Must match the service name in the config file used by the runner's
+compose network manager.
+"""
+
+
 class Runner:
     """Manages the nodes and runs the scenario on them."""
 
@@ -75,10 +83,13 @@ class Runner:
     _exit_stack: AsyncExitStack
     """A stack of `AsyncContextManager` instances to be closed on runner shutdown."""
 
+    _nginx_service_address: Optional[str]
+    """The IP address of the nginx service in the Docker network."""
+
     _topology: List[YagnaContainerConfig]
     """A list of configuration objects for the containers to be instantiated."""
 
-    _web_server: WebServer
+    _web_server: Optional[WebServer]
     """A built-in web server."""
 
     def __init__(
@@ -107,6 +118,7 @@ def __init__(
             config=compose_config,
             docker_client=docker.from_env(),
         )
+        self._nginx_service_address = None
         self._web_server = (
             WebServer(web_root_path, web_server_port) if web_root_path else None
         )
@@ -234,6 +246,13 @@ def host_address(self) -> str:
         else:
             return "host.docker.internal"
 
+    @property
+    def nginx_container_address(self) -> str:
+        """Return the IP address of the proxy-nginx service in the Docker network."""
+        if not self._nginx_service_address:
+            raise RuntimeError("Docker network not started")
+        return self._nginx_service_address
+
     @property
     def web_server_port(self) -> Optional[int]:
         """Return the port of the build-in web server."""
@@ -275,9 +294,17 @@ async def _enter(self) -> None:
         self._exit_stack.enter_context(configure_logging_for_test(self.log_dir))
         logger.info(colors.yellow("Running test: %s"), self.test_name)
 
-        await self._exit_stack.enter_async_context(
+        container_info = await self._exit_stack.enter_async_context(
             run_compose_network(self._compose_manager, self.log_dir)
         )
+        for info in container_info.values():
+            if PROXY_NGINX_SERVICE_NAME in info.aliases:
+                self._nginx_service_address = info.address
+                break
+        else:
+            raise RuntimeError(
+                f"Service {PROXY_NGINX_SERVICE_NAME} not found in the Docker network"
+            )
 
         self._create_probes(self.log_dir)
 

goth/runner/container/compose.py

@@ -5,7 +5,7 @@
 import logging
 import os
 from pathlib import Path
-from typing import ClassVar, Dict, Optional
+from typing import AsyncIterator, ClassVar, Dict, List, Optional
 
 from docker import DockerClient
 import yaml
@@ -16,7 +16,7 @@
     build_yagna_image,
     YagnaBuildEnvironment,
 )
-from goth.runner.container.utils import get_container_address
+from goth.runner.container.utils import get_container_network_info
 from goth.runner.exceptions import ContainerNotFoundError
 from goth.runner.log import LogConfig
 from goth.runner.log_monitor import LogEventMonitor
@@ -47,6 +47,20 @@ class ComposeConfig:
     """
 
 
+@dataclass
+class ContainerInfo:
+    """Info on a Docker container started by Docker compose."""
+
+    address: str
+    """The container's IP address in the Docker network"""
+
+    aliases: List[str]
+    """Container aliases in the Docker network"""
+
+    image: str
+    """The container's image name"""
+
+
 class ComposeNetworkManager:
     """Class which manages a docker-compose network.
 
@@ -82,9 +96,12 @@ def __init__(
         self._log_monitors = {}
         self._network_gateway_address = ""
 
-    async def start_network(self, log_dir: Path, force_build: bool = False) -> None:
+    async def start_network(
+        self, log_dir: Path, force_build: bool = False
+    ) -> Dict[str, ContainerInfo]:
         """Start the compose network based on this manager's compose file.
 
+        Returns information on containers started in the compose network.
         This step may include (re)building the network's docker images.
         """
         # Stop the network in case it's already running (e.g. from a previous test)
@@ -106,7 +123,12 @@ async def start_network(self, log_dir: Path, force_build: bool = False) -> None:
 
         self._start_log_monitors(log_dir)
         await self._wait_for_containers()
-        self._log_running_containers()
+        container_infos = self._get_running_containers()
+        for name, info in container_infos.items():
+            logger.info(
+                "[%-25s] IP address: %-15s image: %s", name, info.address, info.image
+            )
+        return container_infos
 
     async def _wait_for_containers(self) -> None:
         logger.info("Waiting for compose containers to be ready")
@@ -162,14 +184,15 @@ def network_gateway_address(self) -> str:
 
         return self._network_gateway_address
 
-    def _log_running_containers(self):
+    def _get_running_containers(self) -> Dict[str, ContainerInfo]:
+        info = {}
         for container in self._docker_client.containers.list():
-            logger.info(
-                "[%-25s] IP address: %-15s image: %s",
-                container.name,
-                get_container_address(self._docker_client, container.name),
-                container.image.tags[0],
+            address, aliases = get_container_network_info(
+                self._docker_client, container.name
             )
+            image = container.image.tags[0]
+            info[container.name] = ContainerInfo(address, aliases, image)
+        return info
 
     def _start_log_monitors(self, log_dir: Path) -> None:
         for service_name in self._get_compose_services():
@@ -195,15 +218,18 @@ def _start_log_monitors(self, log_dir: Path) -> None:
 @contextlib.asynccontextmanager
 async def run_compose_network(
     compose_manager: ComposeNetworkManager, log_dir: Path, force_build: bool = False
-) -> None:
-    """Implement AsyncContextManager for starting/stopping docker compose network."""
+) -> AsyncIterator[Dict[str, ContainerInfo]]:
+    """Implement AsyncContextManager for starting/stopping docker compose network.
+
+    Yields information on containers started in the compose network.
+    """
 
     try:
         logger.debug(
             "Starting compose network. log_dir=%s, force_build=%s", log_dir, force_build
         )
-        await compose_manager.start_network(log_dir, force_build)
-        yield
+        containers = await compose_manager.start_network(log_dir, force_build)
+        yield containers
     finally:
         logger.debug("Stopping compose network")
         await compose_manager.stop_network()

goth/runner/container/utils.py

@@ -1,19 +1,19 @@
 """Utilities related to Docker containers."""
 from pathlib import Path
-from typing import Dict
+from typing import Dict, List, Tuple
 
 from docker import DockerClient
 
 from goth.runner.container import DockerContainer
 from goth.runner.exceptions import ContainerNotFoundError
 
 
-def get_container_address(
+def get_container_network_info(
     client: DockerClient,
     container_name: str,
     network_name: str = DockerContainer.DEFAULT_NETWORK,
-) -> str:
-    """Get the IP address of a container in a given network.
+) -> Tuple[str, List[str]]:
+    """Get the IP address and the aliases of a container in a given network.
 
     The name of the container does not have to be exact, it may be a substring.
     In case of more than one container name matching the given string, the first
@@ -29,7 +29,25 @@ def get_container_address(
 
     container = matching_containers[0]
     container_networks = container.attrs["NetworkSettings"]["Networks"]
-    return container_networks[network_name]["IPAddress"]
+    network = container_networks[network_name]
+    return network["IPAddress"], network["Aliases"]
+
+
+def get_container_address(
+    client: DockerClient,
+    container_name: str,
+    network_name: str = DockerContainer.DEFAULT_NETWORK,
+) -> str:
+    """Get the IP address and the aliases of a container in a given network.
+
+    The name of the container does not have to be exact, it may be a substring.
+    In case of more than one container name matching the given string, the first
+    container is returned, as listed by the Docker daemon.
+
+    Raises `ContainerNotFoundError` if no matching container is found.
+    Raises `KeyError` if the container is not connected to the specified network.
+    """
+    return get_container_network_info(client, container_name, network_name)[0]
 
 
 def get_volumes_spec(

goth/runner/container/yagna.py

@@ -38,6 +38,14 @@ class YagnaContainerConfig(DockerContainerConfig):
     payment_id: Optional[payment.PaymentId]
     """Custom key and payment accounts to be imported into yagna ID service."""
 
+    use_proxy: bool
+    """Whether the probe will route REST API calls through MITM proxy.
+
+    This setting is used when initializing the `api` component of the probe
+    instantiated from this config, as well as to define the environment in which
+    commands run by `Probe.run_command_on_host()` are run.
+    """
+
     def __init__(
         self,
         name: str,
@@ -47,13 +55,15 @@ def __init__(
         environment: Optional[Dict[str, str]] = None,
         privileged_mode: bool = False,
         payment_id: Optional[payment.PaymentId] = None,
+        use_proxy: bool = False,
         **probe_properties,
     ):
         super().__init__(name, volumes or {}, log_config, privileged_mode)
         self.probe_type = probe_type
         self.probe_properties = probe_properties or {}
         self.environment = environment or {}
         self.payment_id = payment_id
+        self.use_proxy = use_proxy
 
 
 class YagnaContainer(DockerContainer):