Feature: Proxy (getmoto#6848)

.github/workflows/build.yml

@@ -88,6 +88,11 @@ jobs:
     if: "!contains(github.event.pull_request.labels.*.name, 'java')"
     uses: ./.github/workflows/tests_servermode.yml
 
+  testproxy:
+    needs: [lint]
+    if: "!contains(github.event.pull_request.labels.*.name, 'java')"
+    uses: ./.github/workflows/tests_proxymode.yml
+
   release:
     name: Release
     runs-on: ubuntu-latest

.github/workflows/tests_proxymode.yml

@@ -0,0 +1,57 @@
+name: Unit tests in Proxy Mode
+on: [workflow_call]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.11"]
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Get pip cache dir
+      id: pip-cache
+      run: |
+        echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
+    - name: pip cache
+      uses: actions/cache@v3
+      with:
+        path: ${{ steps.pip-cache.outputs.dir }}
+        key: pip-${{ matrix.python-version }}-${{ hashFiles('**/setup.cfg') }}
+    - name: Update pip
+      run: |
+        python -m pip install --upgrade pip
+    - name: Install project dependencies
+      run: |
+        pip install -r requirements-tests.txt
+        pip install .[all,server]
+    - name: Start MotoProxy
+      run: |
+        moto_proxy -h > moto_proxy.log
+        moto_proxy -H 0.0.0.0 -v > moto_proxy.log &
+    - name: Test ProxyMode
+      env:
+        TEST_PROXY_MODE: ${{ true }}
+      run: |
+        pytest -sv tests/test_acmpca tests/test_awslambda tests/test_apigateway tests/test_s3
+    - name: "Stop MotoProxy"
+      if: always()
+      run: |
+        pwd
+        ls -la
+        kill $(lsof -t -i:5005)
+    - name: Archive Proxy logs
+      if: always()
+      uses: actions/upload-artifact@v3
+      with:
+        name: motoproxy-${{ matrix.python-version }}
+        path: |
+          moto_proxy.log

.gitignore

@@ -29,6 +29,9 @@ htmlcov/
 .coverage*
 docs/_build
 moto_recording
+moto/moto_proxy/certs/*.crt
+moto/moto_proxy/certs/*.csr
+moto/moto_proxy/certs/*.conf
 .hypothesis
 other_langs/tests_java/target
 other_langs/tests_dotnet/ExampleTestProject/bin

MANIFEST.in

@@ -10,6 +10,9 @@ include moto/ec2/resources/ecs/optimized_amis/*.json
 include moto/cognitoidp/resources/*.json
 include moto/dynamodb/parsing/reserved_keywords.txt
 include moto/moto_api/_internal/*
+include moto/moto_proxy/*
+include moto/moto_proxy/certs/__init__.py
+include moto/moto_proxy/certs/req.conf.tmpl
 include moto/rds/resources/cluster_options/*.json
 include moto/servicequotas/resources/*/*.json
 include moto/ssm/resources/*.json

docs/docs/proxy_mode.rst

@@ -0,0 +1,126 @@
+.. _proxy_mode:
+
+.. role:: bash(code)
+   :language: bash
+
+.. role:: raw-html(raw)
+    :format: html
+
+================================
+Proxy Mode
+================================
+
+Moto can be run as a proxy, intercepting all requests to AWS and mocking them instead.  :raw-html:`<br />`
+Some of the benefits:
+ - Easy to configure for all SDK's
+ - Can be reached by Lambda containers, allowing you to mock service-calls inside a Lambda-function
+
+
+Installation
+-------------
+
+Install the required dependencies using:
+
+.. code:: bash
+
+    pip install moto[proxy]
+
+
+You can then start the proxy like this:
+
+.. code:: bash
+
+    $ pip install moto[proxy]
+    $ moto_proxy
+
+Note that, if you want your Lambda functions to reach this proxy, you need to open up the moto_proxy:
+
+.. code:: bash
+
+    $ moto_proxy -H 0.0.0.0
+
+.. warning:: Be careful not to use this on a public network - this allows all network users access to your server.
+
+
+Quick usage
+--------------
+The help command shows a quick-guide on how to configure SDK's to  use the proxy.
+.. code-block:: bash
+
+    $ moto_proxy --help
+
+
+Extended Configuration
+------------------------
+
+To use the MotoProxy while running your tests, the AWS SDK needs to know two things:
+
+ - The proxy endpoint
+ - How to deal with SSL
+
+To set the proxy endpoint, use the `HTTPS_PROXY`-environment variable.
+
+Because the proxy does not have an approved SSL certificate, the SDK will not trust the proxy by default. This means that the SDK has to be configured to either
+
+1. Accept the proxy's custom certificate, by setting the `AWS_CA_BUNDLE`-environment variable
+2. Allow unverified SSL certificates
+
+The `AWS_CA_BUNDLE` needs to point to the location of the CA certificate that comes with Moto.  :raw-html:`<br />`
+You can run `moto_proxy --help` to get the exact location of this certificate, depending on where Moto is installed.
+
+Environment Variables Configuration:
+------------------------------
+
+.. code-block:: bash
+
+    export HTTPS_PROXY=http://localhost:5005
+    aws cloudformation list-stacks --no-verify-ssl
+
+Or by configuring the AWS_CA_BUNDLE:
+
+.. code-block:: bash
+
+    export HTTPS_PROXY=http://localhost:5005
+    export AWS_CA_BUNDLE=/location/of/moto/ca/cert.crt
+    aws cloudformation list-stacks
+
+
+Python Configuration
+--------------------------
+
+If you're already using Moto's `mock_service`-decorators, you can use a custom environment variable that configures everything automatically:
+
+.. code-block:: bash
+
+    TEST_PROXY_MODE=true pytest
+
+To configure this manually:
+
+.. code-block:: python
+
+    from botocore.config import Config
+
+    config = Config(proxies={"https": "http://localhost:5005"})
+    client = boto3.client("s3", config=config, verify=False)
+
+
+Terraform Configuration
+------------------------------
+
+.. code-block::
+
+    provider "aws" {
+        region                      = "us-east-1"
+        http_proxy                  = "http://localhost:5005"
+        custom_ca_bundle            = "/location/of/moto/ca/cert.crt"
+        # OR
+        insecure                    = true
+    }
+
+
+Drawbacks
+------------
+
+Configuring a proxy means that all requests are intercepted, but the MotoProxy can only handle requests to AWS.
+
+If your test includes a call to `https://www.thirdpartyservice.com`, that will also be intercepted by `MotoProxy` - and subsequently throw an error because it doesn't know how to handle non-AWS requests.

docs/index.rst

@@ -32,6 +32,7 @@ Additional Resources
 
    docs/getting_started
    docs/server_mode
+   docs/proxy_mode
    docs/faq
    docs/iam
    docs/aws_config

moto/awslambda/models.py

@@ -27,6 +27,7 @@
 from moto.core import BaseBackend, BackendDict, BaseModel, CloudFormationModel
 from moto.core.exceptions import RESTError
 from moto.core.utils import unix_time_millis, iso_8601_datetime_with_nanoseconds, utcnow
+from moto.utilities.utils import load_resource_as_bytes
 from moto.iam.models import iam_backends
 from moto.iam.exceptions import IAMNotFoundException
 from moto.ecr.exceptions import ImageNotFoundException
@@ -82,6 +83,17 @@ def zip2tar(zip_bytes: bytes) -> io.BytesIO:
     return tarstream
 
 
+def file2tar(file_content: bytes, file_name: str) -> io.BytesIO:
+    tarstream = io.BytesIO()
+    tarf = tarfile.TarFile(fileobj=tarstream, mode="w")
+    tarinfo = tarfile.TarInfo(name=file_name)
+    tarinfo.size = len(file_content)
+    tarf.addfile(tarinfo, io.BytesIO(file_content))
+
+    tarstream.seek(0)
+    return tarstream
+
+
 class _VolumeRefCount:
     __slots__ = "refcount", "volume"
 
@@ -132,6 +144,10 @@ def __enter__(self) -> "_DockerDataVolumeContext":
             try:
                 with zip2tar(self._lambda_func.code_bytes) as stream:
                     container.put_archive(settings.LAMBDA_DATA_DIR, stream)
+                if settings.test_proxy_mode():
+                    ca_cert = load_resource_as_bytes(__name__, "../moto_proxy/ca.crt")
+                    with file2tar(ca_cert, "ca.crt") as cert_stream:
+                        container.put_archive(settings.LAMBDA_DATA_DIR, cert_stream)
             finally:
                 container.remove(force=True)
 
@@ -862,10 +878,13 @@ def _invoke_lambda(self, event: Optional[str] = None) -> Tuple[str, bool, str]:
 
             env_vars.update(self.environment_vars)
             env_vars["MOTO_HOST"] = settings.moto_server_host()
-            env_vars["MOTO_PORT"] = settings.moto_server_port()
-            env_vars[
-                "MOTO_HTTP_ENDPOINT"
-            ] = f'{env_vars["MOTO_HOST"]}:{env_vars["MOTO_PORT"]}'
+            moto_port = settings.moto_server_port()
+            env_vars["MOTO_PORT"] = moto_port
+            env_vars["MOTO_HTTP_ENDPOINT"] = f'{env_vars["MOTO_HOST"]}:{moto_port}'
+
+            if settings.test_proxy_mode():
+                env_vars["HTTPS_PROXY"] = env_vars["MOTO_HTTP_ENDPOINT"]
+                env_vars["AWS_CA_BUNDLE"] = "/var/task/ca.crt"
 
             container = exit_code = None
             log_config = docker.types.LogConfig(type=docker.types.LogConfig.types.JSON)
@@ -1614,8 +1633,11 @@ class LambdaBackend(BaseBackend):
     Implementation of the AWS Lambda endpoint.
     Invoking functions is supported - they will run inside a Docker container, emulating the real AWS behaviour as closely as possible.
 
-    It is possible to connect from AWS Lambdas to other services, as long as you are running Moto in ServerMode.
-    The Lambda has access to environment variables `MOTO_HOST` and `MOTO_PORT`, which can be used to build the url that MotoServer runs on:
+    It is possible to connect from AWS Lambdas to other services, as long as you are running MotoProxy or the MotoServer.
+
+    When running the MotoProxy, calls to other AWS services are automatically proxied.
+
+    When running MotoServer, the Lambda has access to environment variables `MOTO_HOST` and `MOTO_PORT`, which can be used to build the url that MotoServer runs on:
 
     .. sourcecode:: python
 

moto/core/models.py

@@ -427,6 +427,69 @@ def disable_patching(self) -> None:
             self._resource_patcher.stop()
 
 
+class ProxyModeMockAWS(BaseMockAWS):
+
+    RESET_IN_PROGRESS = False
+
+    def __init__(self, *args: Any, **kwargs: Any):
+        self.test_proxy_mode_endpoint = settings.test_proxy_mode_endpoint()
+        super().__init__(*args, **kwargs)
+
+    def reset(self) -> None:
+        call_reset_api = os.environ.get("MOTO_CALL_RESET_API")
+        if not call_reset_api or call_reset_api.lower() != "false":
+            if not ProxyModeMockAWS.RESET_IN_PROGRESS:
+                ProxyModeMockAWS.RESET_IN_PROGRESS = True
+                import requests
+
+                requests.post(f"{self.test_proxy_mode_endpoint}/moto-api/reset")
+                ProxyModeMockAWS.RESET_IN_PROGRESS = False
+
+    def enable_patching(self, reset: bool = True) -> None:
+        if self.__class__.nested_count == 1 and reset:
+            # Just started
+            self.reset()
+
+        from boto3 import client as real_boto3_client, resource as real_boto3_resource
+
+        def fake_boto3_client(*args: Any, **kwargs: Any) -> botocore.client.BaseClient:
+            kwargs["verify"] = False
+            proxy_endpoint = (
+                f"http://localhost:{os.environ.get('MOTO_PROXY_PORT', 5005)}"
+            )
+            proxies = {"http": proxy_endpoint, "https": proxy_endpoint}
+            if "config" in kwargs:
+                kwargs["config"].__dict__["proxies"] = proxies
+            else:
+                config = Config(proxies=proxies)
+                kwargs["config"] = config
+
+            return real_boto3_client(*args, **kwargs)
+
+        def fake_boto3_resource(*args: Any, **kwargs: Any) -> Any:
+            kwargs["verify"] = False
+            proxy_endpoint = (
+                f"http://localhost:{os.environ.get('MOTO_PROXY_PORT', 5005)}"
+            )
+            proxies = {"http": proxy_endpoint, "https": proxy_endpoint}
+            if "config" in kwargs:
+                kwargs["config"].__dict__["proxies"] = proxies
+            else:
+                config = Config(proxies=proxies)
+                kwargs["config"] = config
+            return real_boto3_resource(*args, **kwargs)
+
+        self._client_patcher = patch("boto3.client", fake_boto3_client)
+        self._resource_patcher = patch("boto3.resource", fake_boto3_resource)
+        self._client_patcher.start()
+        self._resource_patcher.start()
+
+    def disable_patching(self) -> None:
+        if self._client_patcher:
+            self._client_patcher.stop()
+            self._resource_patcher.stop()
+
+
 class base_decorator:
     mock_backend = MockAWS
 
@@ -436,8 +499,10 @@ def __init__(self, backends: BackendDict):
     def __call__(
         self, func: Optional[Callable[..., Any]] = None
     ) -> Union[BaseMockAWS, Callable[..., BaseMockAWS]]:
-        if settings.TEST_SERVER_MODE:
-            mocked_backend: BaseMockAWS = ServerModeMockAWS(self.backends)
+        if settings.test_proxy_mode():
+            mocked_backend: BaseMockAWS = ProxyModeMockAWS(self.backends)
+        elif settings.TEST_SERVER_MODE:
+            mocked_backend: BaseMockAWS = ServerModeMockAWS(self.backends)  # type: ignore
         else:
             mocked_backend = self.mock_backend(self.backends)