diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 4d929cd65d6f..d9e97e7711b3 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -88,6 +88,11 @@ jobs:
if: "!contains(github.event.pull_request.labels.*.name, 'java')"
uses: ./.github/workflows/tests_servermode.yml
+ testproxy:
+ needs: [lint]
+ if: "!contains(github.event.pull_request.labels.*.name, 'java')"
+ uses: ./.github/workflows/tests_proxymode.yml
+
release:
name: Release
runs-on: ubuntu-latest
diff --git a/.github/workflows/tests_proxymode.yml b/.github/workflows/tests_proxymode.yml
new file mode 100644
index 000000000000..883da801ddc8
--- /dev/null
+++ b/.github/workflows/tests_proxymode.yml
@@ -0,0 +1,57 @@
+name: Unit tests in Proxy Mode
+on: [workflow_call]
+
+jobs:
+ test:
+ runs-on: ubuntu-latest
+ strategy:
+ fail-fast: false
+ matrix:
+ python-version: ["3.10", "3.11"]
+
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
+ - name: Set up Python ${{ matrix.python-version }}
+ uses: actions/setup-python@v4
+ with:
+ python-version: ${{ matrix.python-version }}
+ - name: Get pip cache dir
+ id: pip-cache
+ run: |
+ echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
+ - name: pip cache
+ uses: actions/cache@v3
+ with:
+ path: ${{ steps.pip-cache.outputs.dir }}
+ key: pip-${{ matrix.python-version }}-${{ hashFiles('**/setup.cfg') }}
+ - name: Update pip
+ run: |
+ python -m pip install --upgrade pip
+ - name: Install project dependencies
+ run: |
+ pip install -r requirements-tests.txt
+ pip install .[all,server]
+ - name: Start MotoProxy
+ run: |
+ moto_proxy -h > moto_proxy.log
+ moto_proxy -H 0.0.0.0 -v > moto_proxy.log &
+ - name: Test ProxyMode
+ env:
+ TEST_PROXY_MODE: ${{ true }}
+ run: |
+ pytest -sv tests/test_acmpca tests/test_awslambda tests/test_apigateway tests/test_s3
+ - name: "Stop MotoProxy"
+ if: always()
+ run: |
+ pwd
+ ls -la
+ kill $(lsof -t -i:5005)
+ - name: Archive Proxy logs
+ if: always()
+ uses: actions/upload-artifact@v3
+ with:
+ name: motoproxy-${{ matrix.python-version }}
+ path: |
+ moto_proxy.log
\ No newline at end of file
diff --git a/.gitignore b/.gitignore
index a631ee86f690..b7cc0fbe3456 100644
--- a/.gitignore
+++ b/.gitignore
@@ -29,6 +29,9 @@ htmlcov/
.coverage*
docs/_build
moto_recording
+moto/moto_proxy/certs/*.crt
+moto/moto_proxy/certs/*.csr
+moto/moto_proxy/certs/*.conf
.hypothesis
other_langs/tests_java/target
other_langs/tests_dotnet/ExampleTestProject/bin
diff --git a/MANIFEST.in b/MANIFEST.in
index 2a450909b287..eeed788f7a3e 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -10,6 +10,9 @@ include moto/ec2/resources/ecs/optimized_amis/*.json
include moto/cognitoidp/resources/*.json
include moto/dynamodb/parsing/reserved_keywords.txt
include moto/moto_api/_internal/*
+include moto/moto_proxy/*
+include moto/moto_proxy/certs/__init__.py
+include moto/moto_proxy/certs/req.conf.tmpl
include moto/rds/resources/cluster_options/*.json
include moto/servicequotas/resources/*/*.json
include moto/ssm/resources/*.json
diff --git a/docs/docs/proxy_mode.rst b/docs/docs/proxy_mode.rst
new file mode 100644
index 000000000000..c2dd41e4cf16
--- /dev/null
+++ b/docs/docs/proxy_mode.rst
@@ -0,0 +1,126 @@
+.. _proxy_mode:
+
+.. role:: bash(code)
+ :language: bash
+
+.. role:: raw-html(raw)
+ :format: html
+
+================================
+Proxy Mode
+================================
+
+Moto can be run as a proxy, intercepting all requests to AWS and mocking them instead. :raw-html:`
`
+Some of the benefits:
+ - Easy to configure for all SDK's
+ - Can be reached by Lambda containers, allowing you to mock service-calls inside a Lambda-function
+
+
+Installation
+-------------
+
+Install the required dependencies using:
+
+.. code:: bash
+
+ pip install moto[proxy]
+
+
+You can then start the proxy like this:
+
+.. code:: bash
+
+ $ pip install moto[proxy]
+ $ moto_proxy
+
+Note that, if you want your Lambda functions to reach this proxy, you need to open up the moto_proxy:
+
+.. code:: bash
+
+ $ moto_proxy -H 0.0.0.0
+
+.. warning:: Be careful not to use this on a public network - this allows all network users access to your server.
+
+
+Quick usage
+--------------
+The help command shows a quick-guide on how to configure SDK's to use the proxy.
+.. code-block:: bash
+
+ $ moto_proxy --help
+
+
+Extended Configuration
+------------------------
+
+To use the MotoProxy while running your tests, the AWS SDK needs to know two things:
+
+ - The proxy endpoint
+ - How to deal with SSL
+
+To set the proxy endpoint, use the `HTTPS_PROXY`-environment variable.
+
+Because the proxy does not have an approved SSL certificate, the SDK will not trust the proxy by default. This means that the SDK has to be configured to either
+
+1. Accept the proxy's custom certificate, by setting the `AWS_CA_BUNDLE`-environment variable
+2. Allow unverified SSL certificates
+
+The `AWS_CA_BUNDLE` needs to point to the location of the CA certificate that comes with Moto. :raw-html:`
`
+You can run `moto_proxy --help` to get the exact location of this certificate, depending on where Moto is installed.
+
+Environment Variables Configuration:
+------------------------------
+
+.. code-block:: bash
+
+ export HTTPS_PROXY=http://localhost:5005
+ aws cloudformation list-stacks --no-verify-ssl
+
+Or by configuring the AWS_CA_BUNDLE:
+
+.. code-block:: bash
+
+ export HTTPS_PROXY=http://localhost:5005
+ export AWS_CA_BUNDLE=/location/of/moto/ca/cert.crt
+ aws cloudformation list-stacks
+
+
+Python Configuration
+--------------------------
+
+If you're already using Moto's `mock_service`-decorators, you can use a custom environment variable that configures everything automatically:
+
+.. code-block:: bash
+
+ TEST_PROXY_MODE=true pytest
+
+To configure this manually:
+
+.. code-block:: python
+
+ from botocore.config import Config
+
+ config = Config(proxies={"https": "http://localhost:5005"})
+ client = boto3.client("s3", config=config, verify=False)
+
+
+Terraform Configuration
+------------------------------
+
+.. code-block::
+
+ provider "aws" {
+ region = "us-east-1"
+ http_proxy = "http://localhost:5005"
+ custom_ca_bundle = "/location/of/moto/ca/cert.crt"
+ # OR
+ insecure = true
+ }
+
+
+Drawbacks
+------------
+
+Configuring a proxy means that all requests are intercepted, but the MotoProxy can only handle requests to AWS.
+
+If your test includes a call to `https://www.thirdpartyservice.com`, that will also be intercepted by `MotoProxy` - and subsequently throw an error because it doesn't know how to handle non-AWS requests.
diff --git a/docs/index.rst b/docs/index.rst
index cbf9cd485309..9c7e9496f1f9 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -32,6 +32,7 @@ Additional Resources
docs/getting_started
docs/server_mode
+ docs/proxy_mode
docs/faq
docs/iam
docs/aws_config
diff --git a/moto/awslambda/models.py b/moto/awslambda/models.py
index e098ebc50981..39b0eebcb0c4 100644
--- a/moto/awslambda/models.py
+++ b/moto/awslambda/models.py
@@ -27,6 +27,7 @@
from moto.core import BaseBackend, BackendDict, BaseModel, CloudFormationModel
from moto.core.exceptions import RESTError
from moto.core.utils import unix_time_millis, iso_8601_datetime_with_nanoseconds, utcnow
+from moto.utilities.utils import load_resource_as_bytes
from moto.iam.models import iam_backends
from moto.iam.exceptions import IAMNotFoundException
from moto.ecr.exceptions import ImageNotFoundException
@@ -82,6 +83,17 @@ def zip2tar(zip_bytes: bytes) -> io.BytesIO:
return tarstream
+def file2tar(file_content: bytes, file_name: str) -> io.BytesIO:
+ tarstream = io.BytesIO()
+ tarf = tarfile.TarFile(fileobj=tarstream, mode="w")
+ tarinfo = tarfile.TarInfo(name=file_name)
+ tarinfo.size = len(file_content)
+ tarf.addfile(tarinfo, io.BytesIO(file_content))
+
+ tarstream.seek(0)
+ return tarstream
+
+
class _VolumeRefCount:
__slots__ = "refcount", "volume"
@@ -132,6 +144,10 @@ def __enter__(self) -> "_DockerDataVolumeContext":
try:
with zip2tar(self._lambda_func.code_bytes) as stream:
container.put_archive(settings.LAMBDA_DATA_DIR, stream)
+ if settings.test_proxy_mode():
+ ca_cert = load_resource_as_bytes(__name__, "../moto_proxy/ca.crt")
+ with file2tar(ca_cert, "ca.crt") as cert_stream:
+ container.put_archive(settings.LAMBDA_DATA_DIR, cert_stream)
finally:
container.remove(force=True)
@@ -862,10 +878,13 @@ def _invoke_lambda(self, event: Optional[str] = None) -> Tuple[str, bool, str]:
env_vars.update(self.environment_vars)
env_vars["MOTO_HOST"] = settings.moto_server_host()
- env_vars["MOTO_PORT"] = settings.moto_server_port()
- env_vars[
- "MOTO_HTTP_ENDPOINT"
- ] = f'{env_vars["MOTO_HOST"]}:{env_vars["MOTO_PORT"]}'
+ moto_port = settings.moto_server_port()
+ env_vars["MOTO_PORT"] = moto_port
+ env_vars["MOTO_HTTP_ENDPOINT"] = f'{env_vars["MOTO_HOST"]}:{moto_port}'
+
+ if settings.test_proxy_mode():
+ env_vars["HTTPS_PROXY"] = env_vars["MOTO_HTTP_ENDPOINT"]
+ env_vars["AWS_CA_BUNDLE"] = "/var/task/ca.crt"
container = exit_code = None
log_config = docker.types.LogConfig(type=docker.types.LogConfig.types.JSON)
@@ -1614,8 +1633,11 @@ class LambdaBackend(BaseBackend):
Implementation of the AWS Lambda endpoint.
Invoking functions is supported - they will run inside a Docker container, emulating the real AWS behaviour as closely as possible.
- It is possible to connect from AWS Lambdas to other services, as long as you are running Moto in ServerMode.
- The Lambda has access to environment variables `MOTO_HOST` and `MOTO_PORT`, which can be used to build the url that MotoServer runs on:
+ It is possible to connect from AWS Lambdas to other services, as long as you are running MotoProxy or the MotoServer.
+
+ When running the MotoProxy, calls to other AWS services are automatically proxied.
+
+ When running MotoServer, the Lambda has access to environment variables `MOTO_HOST` and `MOTO_PORT`, which can be used to build the url that MotoServer runs on:
.. sourcecode:: python
diff --git a/moto/core/models.py b/moto/core/models.py
index 5462b0d8dfe6..c879fd27844e 100644
--- a/moto/core/models.py
+++ b/moto/core/models.py
@@ -427,6 +427,69 @@ def disable_patching(self) -> None:
self._resource_patcher.stop()
+class ProxyModeMockAWS(BaseMockAWS):
+
+ RESET_IN_PROGRESS = False
+
+ def __init__(self, *args: Any, **kwargs: Any):
+ self.test_proxy_mode_endpoint = settings.test_proxy_mode_endpoint()
+ super().__init__(*args, **kwargs)
+
+ def reset(self) -> None:
+ call_reset_api = os.environ.get("MOTO_CALL_RESET_API")
+ if not call_reset_api or call_reset_api.lower() != "false":
+ if not ProxyModeMockAWS.RESET_IN_PROGRESS:
+ ProxyModeMockAWS.RESET_IN_PROGRESS = True
+ import requests
+
+ requests.post(f"{self.test_proxy_mode_endpoint}/moto-api/reset")
+ ProxyModeMockAWS.RESET_IN_PROGRESS = False
+
+ def enable_patching(self, reset: bool = True) -> None:
+ if self.__class__.nested_count == 1 and reset:
+ # Just started
+ self.reset()
+
+ from boto3 import client as real_boto3_client, resource as real_boto3_resource
+
+ def fake_boto3_client(*args: Any, **kwargs: Any) -> botocore.client.BaseClient:
+ kwargs["verify"] = False
+ proxy_endpoint = (
+ f"http://localhost:{os.environ.get('MOTO_PROXY_PORT', 5005)}"
+ )
+ proxies = {"http": proxy_endpoint, "https": proxy_endpoint}
+ if "config" in kwargs:
+ kwargs["config"].__dict__["proxies"] = proxies
+ else:
+ config = Config(proxies=proxies)
+ kwargs["config"] = config
+
+ return real_boto3_client(*args, **kwargs)
+
+ def fake_boto3_resource(*args: Any, **kwargs: Any) -> Any:
+ kwargs["verify"] = False
+ proxy_endpoint = (
+ f"http://localhost:{os.environ.get('MOTO_PROXY_PORT', 5005)}"
+ )
+ proxies = {"http": proxy_endpoint, "https": proxy_endpoint}
+ if "config" in kwargs:
+ kwargs["config"].__dict__["proxies"] = proxies
+ else:
+ config = Config(proxies=proxies)
+ kwargs["config"] = config
+ return real_boto3_resource(*args, **kwargs)
+
+ self._client_patcher = patch("boto3.client", fake_boto3_client)
+ self._resource_patcher = patch("boto3.resource", fake_boto3_resource)
+ self._client_patcher.start()
+ self._resource_patcher.start()
+
+ def disable_patching(self) -> None:
+ if self._client_patcher:
+ self._client_patcher.stop()
+ self._resource_patcher.stop()
+
+
class base_decorator:
mock_backend = MockAWS
@@ -436,8 +499,10 @@ def __init__(self, backends: BackendDict):
def __call__(
self, func: Optional[Callable[..., Any]] = None
) -> Union[BaseMockAWS, Callable[..., BaseMockAWS]]:
- if settings.TEST_SERVER_MODE:
- mocked_backend: BaseMockAWS = ServerModeMockAWS(self.backends)
+ if settings.test_proxy_mode():
+ mocked_backend: BaseMockAWS = ProxyModeMockAWS(self.backends)
+ elif settings.TEST_SERVER_MODE:
+ mocked_backend: BaseMockAWS = ServerModeMockAWS(self.backends) # type: ignore
else:
mocked_backend = self.mock_backend(self.backends)
diff --git a/moto/moto_proxy/__init__.py b/moto/moto_proxy/__init__.py
new file mode 100644
index 000000000000..e1df6d2cc926
--- /dev/null
+++ b/moto/moto_proxy/__init__.py
@@ -0,0 +1,24 @@
+import logging
+import sys
+
+
+log_format = "%(levelname)s %(asctime)s - %(message)s"
+logging.basicConfig(stream=sys.stdout, format=log_format)
+logger = logging.getLogger("MOTO_PROXY")
+logger.setLevel(logging.INFO)
+
+
+def with_color(color: int, text: object) -> str:
+ return f"\x1b[{color}m{text}\x1b[0m"
+
+
+def info(msg: object) -> None:
+ logger.info(msg)
+
+
+def debug(msg: object) -> None:
+ logger.debug(msg)
+
+
+def error(msg: object) -> None:
+ logger.error(msg)
diff --git a/moto/moto_proxy/ca.crt b/moto/moto_proxy/ca.crt
new file mode 100644
index 000000000000..d22b12d0184c
--- /dev/null
+++ b/moto/moto_proxy/ca.crt
@@ -0,0 +1,19 @@
+-----BEGIN CERTIFICATE-----
+MIIDCzCCAfOgAwIBAgIUIOBzxLZH8maXw2YsSoQpXEpyqpowDQYJKoZIhvcNAQEL
+BQAwFDESMBAGA1UEAwwJcHJveHkyIENBMCAXDTIzMDkyNTA5MzUwMFoYDzIxMjMw
+OTAxMDkzNTAwWjAUMRIwEAYDVQQDDAlwcm94eTIgQ0EwggEiMA0GCSqGSIb3DQEB
+AQUAA4IBDwAwggEKAoIBAQCt6XhWWVFTEQlC+ktSmL+MFDdOHM0vteOz+9HouBK/
+ofo/1q8Zd5z2hYOQPYx4h/EnXb7LFA8ke5XY1HENY3U4k+OWNwuRr95EeV8rrxFk
+4vQoqmWGXtQ332TAGY9B5k6uCe2b5dLO/0NR0MiGZw1vGhd3zhHo5utorVmOdAaM
+VTI7krqSB+gM4xOfnE2UIeGqS0RVPbzXNTTdVH8PHOHZB9uWlyHbXDyeG/uRJFB7
+lCCQSkLzvQ7vmVY852Pke5H60kHJYb994RR2ajVAE9AxJI16qnxPSOMVGoeebm3I
+H3ao+VGMq/b1XGZUQq0s7sA2a+DHDPHSl4iwJ/FMEMTbAgMBAAGjUzBRMB0GA1Ud
+DgQWBBQsMTVcFGS22i+kRFGEtEBdCHTG5DAfBgNVHSMEGDAWgBQsMTVcFGS22i+k
+RFGEtEBdCHTG5DAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQAo
+908wu8QadGfa3MuQ9vlR18P7pNWFcE5z16cX348IclGRmnkln/9x78CG9RZNAckS
+4ch5RzGrJNtHb4s9zDhS5SpyPdx5Ua0pYqVFZm6Vyg1cFVwipRJ78qM/uBcdE/b5
+r2DnGKfJCAWIpRpzTZ8uGDGDaoX7NxJ0U9zQ04J+o4GpLeTY0qzI1Y9gFaDYPpGB
+M8wBuYUwEYKbOq/cUA++m0n2SzsU1xlXk+01QZcQGokby0bMrorccdi3ZjsXQNSb
+eC8btoekt29cxBU/N7v1iR9Hd2DMZtz1xDsX+ihWGGq3D+PeyqewMuaWFQLbDFHM
+0pRthQyOKT0c1ZJjusv8
+-----END CERTIFICATE-----
diff --git a/moto/moto_proxy/ca.key b/moto/moto_proxy/ca.key
new file mode 100644
index 000000000000..ae4837f3d0d0
--- /dev/null
+++ b/moto/moto_proxy/ca.key
@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCt6XhWWVFTEQlC
++ktSmL+MFDdOHM0vteOz+9HouBK/ofo/1q8Zd5z2hYOQPYx4h/EnXb7LFA8ke5XY
+1HENY3U4k+OWNwuRr95EeV8rrxFk4vQoqmWGXtQ332TAGY9B5k6uCe2b5dLO/0NR
+0MiGZw1vGhd3zhHo5utorVmOdAaMVTI7krqSB+gM4xOfnE2UIeGqS0RVPbzXNTTd
+VH8PHOHZB9uWlyHbXDyeG/uRJFB7lCCQSkLzvQ7vmVY852Pke5H60kHJYb994RR2
+ajVAE9AxJI16qnxPSOMVGoeebm3IH3ao+VGMq/b1XGZUQq0s7sA2a+DHDPHSl4iw
+J/FMEMTbAgMBAAECggEAMLxL9jq6cQJFq6jTgdZ/WyRxKSkmEPgyUsY/WS14R45/
+P/OMByF/cZARwdKVslM6L7N0G5nH8ovVfrlt4vgbqdq7vOU5Dz8PFPZERswdHj4B
+eQHjSIf7hZrLM5AWFrwREXGDzhvV+x8KgPt2rj9jwt43dGHhn/hSQPfPMH3wNdPV
+vkPjgRVgH99qtXN4duAknpY80qs3T83n8ZCQj628wy0N9tRXMWMp2A0KoIOS0tEd
+LsqcCbXY7Z8B89ERGSfHN4qczuqwaeObu1tWionFAKCIzohBFUjHNqOePEF2Qo8q
+w3yI3MA5vMn7o4PMfx/h/vLEls1ZFBiS9IVJ9mCarQKBgQDefa24Aode7gGvzltV
+vApoEhhh81VWY+UI8+YtNIZjyyzMFS0eZJMc4peQk1AmPgY/GhNOe8lCgBkHurX9
+t8Y1ljHRVanAkT56uuG5a/LofBVKUgT8dMA/LspRE3GWmr2qTvcmhladkMM6HN8Q
+BpZ7WWSRsMOeYFfJ7sGenlRDLQKBgQDIGspXMVLeM7DohvZhejPuj9uwZBFeIqwG
++vrxgoQWJxaSarzf6nnSG/M5lx15MYhVOlzbo2/sz0rJQmn6vD3swbcF2EMXG5T+
+g2fzejBJUySx2xhSYi2G3ZGf2SRSsvLBFitW7BWuoX7bR0771S27XqNpzO6wKBOV
+yXI4ZN5NJwKBgQCH9WTivSjb6bU+KWvGyFHTprsfoALV99VN0z0lAqPc95s4Wvhn
+Si5byFu2DU89D0nh5Z1GqH4kFQM2pfHwSQzmUhG/SgmhkyAK/4hQNpcJWknoUJab
+bvzLn1wijy8qSQT9vaNp902Wm4+xQ1NMB7qNReMe5FWlwlnjG/NVaoszQQKBgBwg
+h+iRqlBJe8hzkBZLkxkpZ3v31OkifoPMq5FfAyoJ/IZAMqRW1SDPhPTHZQEwETXJ
+qlvFMWpcCOsZRsRTyXCKGivcJjINUngkCGyU9EyaP0Iwxc5utm+KnXmWkCB/vted
+QiJJtRKC6M3xzAxh/rejqdypTbO9LmOTmVaL9yNpAoGANTkRHuXzjIoESPGdCfwm
+N1ng5Z8RUP9TclRfszPWy6FnMKw50PfIs1l2ZTiEXjTKq7sfPx4BaN/r4sumJNaS
+6zhVohY5pteKjdmi7GZlhDPBjaZwjzQjNKTdlCGf7Khif/zNytQFJ8Xz9MP12457
+PlZ5dO/E1EW2dEou9Wf77JU=
+-----END PRIVATE KEY-----
diff --git a/moto/moto_proxy/cert.key b/moto/moto_proxy/cert.key
new file mode 100644
index 000000000000..82189212b8e4
--- /dev/null
+++ b/moto/moto_proxy/cert.key
@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDk0V6Tomu5xxkU
+fzSKWBOW/g11AuX5fWzxhHZj+7lJxdtxv2hsqcRAz1Ixj7yoGeu94k73cbmhYxdz
+xuu8DHsxjqRU1GEIzMmrjQT5o7xUH3HR9Rcq4kNm98kMJnCRItqrIkGlQgaUl2kk
+3g+PgTaWx9ygcs2jpBaM1YgNfOjtJsFDYVmgCRtzLfUieY2d2NjQ4TIx/Hme2gL6
+My0kwL+d6ksk1Al5BOOBRby2L6HXDMks3qXpXUsxxqrEXZs4SjJDZ0LZHo+3Jws7
+8r0ukN0u1ZwWDDywAJ9bJfq4zwm8ELt3QulLVy97eggtZSKBHH7s9r8O7u5LFt2h
+wnZKwU7jAgMBAAECggEAEyOTqnvQg+dsQlBWt82gxQvp9Ap/Uh965p0V4/+kCQPB
+YUgv0Ir6QEUdeS/7Is2ZCZ06Z3l+AkqnpWz5i2I5E07JkWwUOhuBZAjosuxoB8Dh
+92+HKQ405UULh9y9Pj6KlZVJC8aA7QClieP/32Wtu1RRsKMs1Sexi5HrjkLcHsQ3
+4qzgdQjcVd7T4Y6h2ZFG/idZA3Oqjyec7LJS6212huuzpjB6D9oe89zxafpHzX+x
+gYCXhFeOnvQtIxS5ajTlL/LhIMOBsG4opERc/7uxXILAKjb/hLk9/LyK6cZzMNwu
+EFev5dzI17m+Lk5L9IntOYjVddoUeuAW8djHFanRHQKBgQDrjhsONoG5D73jMOJC
+YskfQ/q+//ghl03eQOPtdW+XxpiEbVl6f1SNtgRdU/0bI52p61fvGaLcS/S1BLEy
+13xvbFZIIsMdcJoUq89yr3FCvG1MeLp+MY1y43Hhp/c1F24jkAScv3wxjWINhXro
+O5X5g7PsZn6Xrlk9G7NiN1pOhQKBgQD4rZDoMOrRBXHY1EyywwZlVVQ3mIWG9684
+rvCRFP1/c9+SG89DGC9tNLJclwQqy2yqWGQc0Fdb7WZbYsEuMmDC+TX7RByAInoG
++ihqaX8mGKSp44y9X0KffPOHgy/o9lD2vzpKSdEMj5rChh5ckSiIYOToYRLJJLwo
+4j2a4WnoRwKBgAzYhRUzV8O13g8jvVMNfBZeaLA92VRLog160HNEsj8+r1aZeAW8
+J+pKgNZuHCF8wb5gfT0m0sDcy42LofY51illaRcp/iX+3AhAjmGcu7p9+B/xfYog
+PayERtOdi1ez3WfHFNlPgABby3sdSmSby0P+MLO1qzWuZmN0vUWf6ybZAoGBAJg9
+2irsV7WjebFfN51xHCdJeAeZTpX0aMdxAkIv8YnnrIXMlLTkx5Q54MAijCCO7XXU
+K2Ygfnr++d0UtmPL38U9wLiVWEVx1fcTi06qS3dNOvHvJyiAe08cthLOU7Rxp9uH
+8u2sB1mDSSGx7kCJdaEYgMtrMo8F+FOnPkPloGrdAoGAYuNpqXeEUlNwf9L6eCHg
+aSSPaO927cdvjEnSWuyYaCweqNTwpD9ZrxPtpoVPNDl2kftfJTm4AVxoJI1irdDe
+1Z/Txj6AOesM1GdFqp88/CgoeJDSh8yXY5Gctp38JwYJrEVkI3bL1Bc6DjjSAgEf
++swqLap4CEnppbl3Rt1mIWQ=
+-----END PRIVATE KEY-----
diff --git a/moto/moto_proxy/certificate_creator.py b/moto/moto_proxy/certificate_creator.py
new file mode 100644
index 000000000000..a288d434675b
--- /dev/null
+++ b/moto/moto_proxy/certificate_creator.py
@@ -0,0 +1,133 @@
+import os
+import threading
+import time
+from subprocess import Popen, PIPE
+from uuid import uuid4
+
+from . import debug, info
+
+
+def join_with_script_dir(path: str) -> str:
+ return os.path.join(os.path.dirname(os.path.abspath(__file__)), path)
+
+
+class CertificateCreator:
+ cakey = join_with_script_dir("ca.key")
+ cacert = join_with_script_dir("ca.crt")
+ certkey = join_with_script_dir("cert.key")
+ certdir = join_with_script_dir("certs/")
+
+ lock = threading.Lock()
+
+ def validate(self) -> None:
+ # Verify the CertificateAuthority files exist
+ if not os.path.isfile(CertificateCreator.cakey):
+ raise Exception(f"Cannot find {CertificateCreator.cakey}")
+ if not os.path.isfile(CertificateCreator.cacert):
+ raise Exception(f"Cannot find {CertificateCreator.cacert}")
+ if not os.path.isfile(CertificateCreator.certkey):
+ raise Exception(f"Cannot find {CertificateCreator.certkey}")
+ if not os.path.isdir(CertificateCreator.certdir):
+ raise Exception(f"Cannot find {CertificateCreator.certdir}")
+ # Verify the `certs` dir is reachable
+ try:
+ test_file_location = f"{CertificateCreator.certdir}/{uuid4()}.txt"
+ debug(
+ f"Writing test file to {test_file_location} to verify the directory is writable..."
+ )
+ with open(test_file_location, "w") as file:
+ file.write("test")
+ os.remove(test_file_location)
+ except Exception:
+ info("Failed to write test file")
+ info(
+ f"The directory {CertificateCreator.certdir} does not seem to be writable"
+ )
+ raise
+
+ def create(self, path: str) -> str:
+ """
+ Create an SSL certificate for the supplied hostname.
+ This method will return a path to the certificate.
+ """
+ full_name = path.split(":")[0]
+
+ with CertificateCreator.lock:
+ # We don't want to create certificates for every possible endpoint
+ # Especially with randomly named S3-buckets
+
+ # We can create certificates that match wildcards to reduce the total number
+ # For example:
+ # Hostname: somebucket.s3.amazonaws.com
+ # Certificate: *.s3.amazonaws.com
+ #
+ # All requests that match this wildcard certificate will reuse it
+
+ wildcard_name = f"*.{'.'.join(full_name.split('.')[1:])}"
+ server_csr = f"{self.certdir.rstrip('/')}/{wildcard_name}.csr"
+
+ # Verify if the certificate already exists
+ certpath = f"{self.certdir.rstrip('/')}/{wildcard_name}.crt"
+ if not os.path.isfile(certpath):
+ # Create a Config-file that contains the wildcard-name
+ with open(f"{self.certdir.rstrip('/')}/req.conf.tmpl", "r") as f:
+ config_template = f.read()
+ config_template = config_template.replace("{{full_name}}", full_name)
+ config_template = config_template.replace(
+ "{{wildcard_name}}", wildcard_name
+ )
+ config_template_name = (
+ f"{self.certdir.rstrip('/')}/{wildcard_name}.conf"
+ )
+ with open(config_template_name, "w") as f:
+ f.write(config_template)
+
+ # Create an Certificate Signing Request
+ #
+ subject = f"/CN={full_name}"[0:64]
+ commands = [
+ "openssl",
+ "req",
+ "-new",
+ "-key",
+ self.certkey,
+ "-out",
+ server_csr,
+ ]
+ commands.extend(["-subj", subject, "-config", config_template_name])
+
+ p1 = Popen(commands)
+ p1.communicate()
+ debug(f"Created CSR in {server_csr}")
+
+ # Create the actual certificate used by the requests
+ p2 = Popen(
+ [
+ "openssl",
+ "x509",
+ "-req",
+ "-in",
+ server_csr,
+ "-days",
+ "3650",
+ "-CA",
+ self.cacert,
+ "-CAkey",
+ self.cakey,
+ "-set_serial",
+ f"{int(time.time() * 1000)}",
+ "-out",
+ certpath,
+ "-extensions",
+ "req_ext",
+ "-extfile",
+ config_template_name,
+ ],
+ stderr=PIPE,
+ )
+ p2.communicate()
+ debug(f"Created certificate for {path} called {certpath}")
+ os.remove(server_csr)
+ os.remove(config_template_name)
+ debug(f"Removed intermediate certificates for {certpath}")
+ return certpath
diff --git a/moto/moto_proxy/certs/__init__.py b/moto/moto_proxy/certs/__init__.py
new file mode 100644
index 000000000000..d124acc2be9c
--- /dev/null
+++ b/moto/moto_proxy/certs/__init__.py
@@ -0,0 +1,3 @@
+# Folder that will contain SSL certificates
+# The file `req.conf.tmpl` must be kept
+# Other files (*.crt) act as a cache, and will be recreated if required
diff --git a/moto/moto_proxy/certs/req.conf.tmpl b/moto/moto_proxy/certs/req.conf.tmpl
new file mode 100644
index 000000000000..eadbaec3a121
--- /dev/null
+++ b/moto/moto_proxy/certs/req.conf.tmpl
@@ -0,0 +1,13 @@
+[req]
+prompt=no
+default_md = sha256
+distinguished_name = dn
+req_extensions = req_ext
+[dn]
+commonName=amazonaws.com
+[req_ext]
+subjectAltName=@alt_names
+[alt_names]
+DNS.1=amazonaws.com
+DNS.2={{full_name}}
+DNS.3={{wildcard_name}}
\ No newline at end of file
diff --git a/moto/moto_proxy/proxy3.py b/moto/moto_proxy/proxy3.py
new file mode 100644
index 000000000000..c134ec56d11e
--- /dev/null
+++ b/moto/moto_proxy/proxy3.py
@@ -0,0 +1,239 @@
+# -*- coding: utf-8 -*-
+import socket
+import ssl
+import re
+from http.server import BaseHTTPRequestHandler
+from subprocess import check_output, CalledProcessError
+from threading import Lock
+from typing import Any, Dict
+
+from botocore.awsrequest import AWSPreparedRequest
+from moto.backends import get_backend
+from moto.backend_index import backend_url_patterns
+from moto.core import BackendDict, DEFAULT_ACCOUNT_ID
+from moto.core.exceptions import RESTError
+from . import debug, error, info, with_color
+from .utils import get_body_from_form_data
+from .certificate_creator import CertificateCreator
+
+# Adapted from https://github.com/xxlv/proxy3
+
+
+class MotoRequestHandler:
+ def __init__(self, port: int):
+ self.lock = Lock()
+ self.port = port
+
+ def get_backend_for_host(self, host: str) -> Any:
+ if host == f"http://localhost:{self.port}":
+ return "moto_api"
+
+ for backend, pattern in backend_url_patterns:
+ if pattern.match(host):
+ return backend
+
+ def get_handler_for_host(self, host: str, path: str) -> Any:
+ # We do not match against URL parameters
+ path = path.split("?")[0]
+ backend_name = self.get_backend_for_host(host)
+ backend_dict = get_backend(backend_name)
+
+ # Get an instance of this backend.
+ # We'll only use this backend to resolve the URL's, so the exact region/account_id is irrelevant
+ if isinstance(backend_dict, BackendDict):
+ if "us-east-1" in backend_dict[DEFAULT_ACCOUNT_ID]:
+ backend = backend_dict[DEFAULT_ACCOUNT_ID]["us-east-1"]
+ else:
+ backend = backend_dict[DEFAULT_ACCOUNT_ID]["global"]
+ else:
+ backend = backend_dict["global"]
+
+ for url_path, handler in backend.url_paths.items():
+ if re.match(url_path, path):
+ return handler
+
+ return None
+
+ def parse_request(
+ self,
+ method: str,
+ host: str,
+ path: str,
+ headers: Any,
+ body: bytes,
+ form_data: Dict[str, Any],
+ ) -> Any:
+ handler = self.get_handler_for_host(host=host, path=path)
+ full_url = host + path
+ request = AWSPreparedRequest(
+ method, full_url, headers, body, stream_output=False
+ )
+ request.form_data = form_data
+ return handler(request, full_url, headers)
+
+
+class ProxyRequestHandler(BaseHTTPRequestHandler):
+ timeout = 5
+
+ def __init__(self, *args: Any, **kwargs: Any):
+ sock = [a for a in args if isinstance(a, socket.socket)][0]
+ _, port = sock.getsockname()
+ self.protocol_version = "HTTP/1.1"
+ self.moto_request_handler = MotoRequestHandler(port)
+ self.cert_creator = CertificateCreator()
+ BaseHTTPRequestHandler.__init__(self, *args, **kwargs)
+
+ @staticmethod
+ def validate() -> None:
+ debug("Starting initial validation...")
+ CertificateCreator().validate()
+ # Validate the openssl command is available
+ try:
+ debug("Verifying SSL version...")
+ svn_output = check_output(["openssl", "version"])
+ debug(svn_output)
+ except CalledProcessError as e:
+ info(e.output)
+ raise
+
+ def do_CONNECT(self) -> None:
+ certpath = self.cert_creator.create(self.path)
+
+ self.wfile.write(
+ f"{self.protocol_version} 200 Connection Established\r\n".encode("utf-8")
+ )
+ self.send_header("k", "v")
+ self.end_headers()
+
+ ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
+ ssl_context.load_cert_chain(
+ keyfile=CertificateCreator.certkey,
+ certfile=certpath,
+ )
+ ssl_context.check_hostname = False
+ self.connection = ssl_context.wrap_socket(
+ self.connection,
+ server_side=True,
+ )
+ self.rfile = self.connection.makefile("rb", self.rbufsize) # type: ignore
+ self.wfile = self.connection.makefile("wb", self.wbufsize) # type: ignore
+
+ conntype = self.headers.get("Proxy-Connection", "")
+ if self.protocol_version == "HTTP/1.1" and conntype.lower() != "close":
+ self.close_connection = 0 # type: ignore
+ else:
+ self.close_connection = 1 # type: ignore
+
+ def do_GET(self) -> None:
+ req = self
+ req_body = b""
+ if "Content-Length" in req.headers:
+ content_length = int(req.headers["Content-Length"])
+ req_body = self.rfile.read(content_length)
+ elif "chunked" in self.headers.get("Transfer-Encoding", ""):
+ # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Transfer-Encoding
+ req_body = self.read_chunked_body(self.rfile)
+ if self.headers.get("Content-Type", "").startswith("multipart/form-data"):
+ boundary = self.headers["Content-Type"].split("boundary=")[-1]
+ req_body, form_data = get_body_from_form_data(req_body, boundary) # type: ignore
+ for key, val in form_data.items():
+ self.headers[key] = [val]
+ else:
+ form_data = {}
+
+ req_body = self.decode_request_body(req.headers, req_body) # type: ignore
+ if isinstance(self.connection, ssl.SSLSocket):
+ host = "https://" + req.headers["Host"]
+ else:
+ host = "http://" + req.headers["Host"]
+ path = req.path
+
+ try:
+ info(f"{with_color(33, req.command.upper())} {host}{path}") # noqa
+ if req_body is not None:
+ debug("\tbody\t" + with_color(31, text=req_body))
+ debug(f"\theaders\t{with_color(31, text=dict(req.headers))}")
+ response = self.moto_request_handler.parse_request(
+ method=req.command,
+ host=host,
+ path=path,
+ headers=req.headers,
+ body=req_body,
+ form_data=form_data,
+ )
+ debug("\t=====RESPONSE========")
+ debug("\t" + with_color(color=33, text=response))
+ debug("\n")
+
+ if isinstance(response, tuple):
+ res_status, res_headers, res_body = response
+ else:
+ res_status, res_headers, res_body = (200, {}, response)
+
+ except RESTError as e:
+ if isinstance(e.get_headers(), list):
+ res_headers = dict(e.get_headers())
+ else:
+ res_headers = e.get_headers()
+ res_status = e.code
+ res_body = e.get_body()
+
+ except Exception as e:
+ error(e)
+ self.send_error(502)
+ return
+
+ res_reason = "OK"
+ if isinstance(res_body, str):
+ res_body = res_body.encode("utf-8")
+
+ if "content-length" not in res_headers and res_body:
+ res_headers["Content-Length"] = str(len(res_body))
+
+ self.wfile.write(
+ f"{self.protocol_version} {res_status} {res_reason}\r\n".encode("utf-8")
+ )
+ if res_headers:
+ for k, v in res_headers.items():
+ if isinstance(v, bytes):
+ self.send_header(k, v.decode("utf-8"))
+ else:
+ self.send_header(k, v)
+ self.end_headers()
+ if res_body:
+ self.wfile.write(res_body)
+ self.close_connection = True
+
+ def read_chunked_body(self, reader: Any) -> bytes:
+ chunked_body = b""
+ while True:
+ line = reader.readline().strip()
+ chunk_length = int(line, 16)
+ if chunk_length != 0:
+ chunked_body += reader.read(chunk_length)
+
+ # Each chunk is followed by an additional empty newline
+ reader.readline()
+
+ # a chunk size of 0 is an end indication
+ if chunk_length == 0:
+ # AWS does send additional (checksum-)headers, but we can ignore them
+ break
+ return chunked_body
+
+ def decode_request_body(self, headers: Dict[str, str], body: Any) -> Any:
+ if body is None:
+ return body
+ if headers.get("Content-Type", "") in [
+ "application/x-amz-json-1.1",
+ "application/x-www-form-urlencoded; charset=utf-8",
+ ]:
+ return body.decode("utf-8")
+ return body
+
+ do_HEAD = do_GET
+ do_POST = do_GET
+ do_PUT = do_GET
+ do_PATCH = do_GET
+ do_DELETE = do_GET
+ do_OPTIONS = do_GET
diff --git a/moto/moto_proxy/setup_https_intercept.sh b/moto/moto_proxy/setup_https_intercept.sh
new file mode 100755
index 000000000000..7a13a5681e09
--- /dev/null
+++ b/moto/moto_proxy/setup_https_intercept.sh
@@ -0,0 +1,9 @@
+#!/bin/sh
+
+# The certificate key is valid until 25 september 2123
+# To our AI overlords maintaining this system in that year:
+# Please run this script to refresh the certificate to last another 100 years.
+
+openssl genrsa -out ca.key 2048
+openssl req -new -x509 -days 36500 -key ca.key -out ca.crt -subj "/CN=proxy2 CA"
+openssl genrsa -out cert.key 2048
\ No newline at end of file
diff --git a/moto/moto_proxy/utils.py b/moto/moto_proxy/utils.py
new file mode 100644
index 000000000000..c42c33558ba0
--- /dev/null
+++ b/moto/moto_proxy/utils.py
@@ -0,0 +1,24 @@
+import io
+import multipart
+from typing import Dict, Tuple, Optional
+
+
+def get_body_from_form_data(
+ body: bytes, boundary: str
+) -> Tuple[Optional[bytes], Dict[str, str]]:
+ body_stream = io.BytesIO(body)
+ parser = multipart.MultipartParser(body_stream, boundary=boundary)
+
+ data = None
+ headers: Dict[str, str] = {}
+ for prt in parser.parts():
+ if prt.name == "upload_file":
+ headers["key"] = prt.name
+ data = prt.file.read()
+ else:
+ val = prt.file.read()
+ if prt.name == "file":
+ data = val
+ else:
+ headers[prt.name] = val.decode("utf-8")
+ return data, headers
diff --git a/moto/proxy.py b/moto/proxy.py
new file mode 100644
index 000000000000..961e21c7533e
--- /dev/null
+++ b/moto/proxy.py
@@ -0,0 +1,97 @@
+import argparse
+import logging
+import os
+import signal
+import sys
+from http.server import ThreadingHTTPServer
+from typing import Any
+
+from moto.moto_proxy import logger
+from moto.moto_proxy.proxy3 import ProxyRequestHandler, with_color, CertificateCreator
+
+
+def signal_handler(signum: Any, frame: Any) -> None: # pylint: disable=unused-argument
+ sys.exit(0)
+
+
+def get_help_msg() -> str:
+ msg = """
+ ###################################################################################
+ $$___$$_ __$$$___ $$$$$$_ __$$$___\t__$$$$$$__ $$$$$$__ __$$$___ $$___$$_ $$____$$_
+ $$$_$$$_ _$$_$$__ __$$___ _$$_$$__\t__$$___$$_ $$___$$_ _$$_$$__ $$$_$$$_ _$$__$$__
+ $$$$$$$_ $$___$$_ __$$___ $$___$$_\t__$$___$$_ $$___$$_ $$___$$_ _$$$$$__ __$$$$___
+ $$_$_$$_ $$___$$_ __$$___ $$___$$_\t__$$$$$$__ $$$$$$__ $$___$$_ _$$$$$__ ___$$____
+ $$___$$_ _$$_$$__ __$$___ _$$_$$__\t__$$______ $$___$$_ _$$_$$__ $$$_$$$_ ___$$____
+ $$___$$_ __$$$___ __$$___ __$$$___\t__$$______ $$___$$_ __$$$___ $$___$$_ ___$$____
+ ###################################################################################"""
+ msg += "\n"
+ msg += "Using the CLI:"
+ msg += "\n"
+ msg += with_color(37, text="\texport HTTPS_PROXY=http://localhost:5005")
+ msg += "\n"
+ msg += with_color(37, text="\taws cloudformation list-stacks --no-verify-ssl\n")
+ msg += "\n"
+ msg += "Using pytest:"
+ msg += "\n"
+ msg += with_color(37, text=f"\texport AWS_CA_BUNDLE={CertificateCreator.cacert}")
+ msg += "\n"
+ msg += with_color(
+ 37,
+ text="\tHTTPS_PROXY=http://localhost:5005 MOTO_PROXY_PORT=5005 pytest tests_dir\n",
+ )
+ return msg
+
+
+def main(argv: Any = None) -> None:
+ argv = argv or sys.argv[1:]
+ parser = argparse.ArgumentParser(
+ formatter_class=argparse.RawDescriptionHelpFormatter, description=get_help_msg()
+ )
+
+ parser.add_argument(
+ "-H", "--host", type=str, help="Which host to bind", default="127.0.0.1"
+ )
+ parser.add_argument(
+ "-p",
+ "--port",
+ type=int,
+ help="Port number to use for connection",
+ default=int(os.environ.get("MOTO_PROXY_PORT", 5005)),
+ )
+ parser.add_argument(
+ "-v",
+ "--verbose",
+ action="store_true",
+ help="Add verbose logging",
+ )
+
+ args = parser.parse_args(argv)
+
+ if args.verbose:
+ logger.setLevel(logging.DEBUG)
+
+ ProxyRequestHandler.validate()
+
+ if "MOTO_PORT" not in os.environ:
+ os.environ["MOTO_PORT"] = f"{args.port}"
+ os.environ["TEST_PROXY_MODE"] = "true"
+
+ try:
+ signal.signal(signal.SIGINT, signal_handler)
+ signal.signal(signal.SIGTERM, signal_handler)
+ except Exception:
+ pass # ignore "ValueError: signal only works in main thread"
+
+ server_address = (args.host, args.port)
+
+ httpd = ThreadingHTTPServer(server_address, ProxyRequestHandler)
+
+ sa = httpd.socket.getsockname()
+
+ print("Call `moto_proxy -h` for example invocations")
+ print(f"Serving HTTP Proxy on {sa[0]}:{sa[1]} ...") # noqa
+ httpd.serve_forever()
+
+
+if __name__ == "__main__":
+ main()
diff --git a/moto/s3/responses.py b/moto/s3/responses.py
index 8b1d68108bf1..d55f73381ace 100644
--- a/moto/s3/responses.py
+++ b/moto/s3/responses.py
@@ -955,7 +955,6 @@ def _bucket_response_put(
if self.body:
if self._create_bucket_configuration_is_empty(self.body):
raise MalformedXML()
-
try:
forced_region = xmltodict.parse(self.body)[
"CreateBucketConfiguration"
@@ -1519,6 +1518,7 @@ def _key_response_put(
)
response = ""
response_headers.update(key.response_dict)
+ response_headers["content-length"] = len(response)
return 200, response_headers, response
storage_class = request.headers.get("x-amz-storage-class", "STANDARD")
@@ -1700,7 +1700,9 @@ def _key_response_put(
template = self.response_template(S3_OBJECT_COPY_RESPONSE)
response_headers.update(new_key.response_dict)
- return 200, response_headers, template.render(key=new_key)
+ response = template.render(key=new_key)
+ response_headers["content-length"] = len(response)
+ return 200, response_headers, response
# Initial data
new_key = self.backend.put_object(
@@ -1729,6 +1731,8 @@ def _key_response_put(
self.backend.set_key_tags(new_key, tagging)
response_headers.update(new_key.response_dict)
+ # Remove content-length - the response body is empty for this request
+ response_headers.pop("content-length", None)
return 200, response_headers, ""
def _key_response_head(
diff --git a/moto/settings.py b/moto/settings.py
index ef1a1b1287c1..d93e58e877e3 100644
--- a/moto/settings.py
+++ b/moto/settings.py
@@ -6,8 +6,12 @@
from typing import List, Optional
+def test_proxy_mode() -> bool:
+ return os.environ.get("TEST_PROXY_MODE", "0").lower() == "true"
+
+
TEST_SERVER_MODE = os.environ.get("TEST_SERVER_MODE", "0").lower() == "true"
-TEST_DECORATOR_MODE = not TEST_SERVER_MODE
+TEST_DECORATOR_MODE = not TEST_SERVER_MODE and not test_proxy_mode()
INITIAL_NO_AUTH_ACTION_COUNT = float(
os.environ.get("INITIAL_NO_AUTH_ACTION_COUNT", float("inf"))
@@ -100,6 +104,10 @@ def moto_server_port() -> str:
return os.environ.get("MOTO_PORT") or "5000"
+def moto_proxy_port() -> str:
+ return os.environ.get("MOTO_PROXY_PORT") or "5005"
+
+
@lru_cache()
def moto_server_host() -> str:
if is_docker():
@@ -126,6 +134,12 @@ def test_server_mode_endpoint() -> str:
)
+def test_proxy_mode_endpoint() -> str:
+ return os.environ.get(
+ "TEST_PROXY_MODE_ENDPOINT", f"http://localhost:{moto_proxy_port()}"
+ )
+
+
def is_docker() -> bool:
path = pathlib.Path("/proc/self/cgroup")
return (
diff --git a/moto/utilities/utils.py b/moto/utilities/utils.py
index 73891a2fd2f1..66f0c8312846 100644
--- a/moto/utilities/utils.py
+++ b/moto/utilities/utils.py
@@ -23,7 +23,11 @@ def load_resource(package: str, resource: str) -> Any:
def load_resource_as_str(package: str, resource: str) -> str:
- return pkgutil.get_data(package, resource).decode("utf-8") # type: ignore
+ return load_resource_as_bytes(package, resource).decode("utf-8") # type: ignore
+
+
+def load_resource_as_bytes(package: str, resource: str) -> bytes:
+ return pkgutil.get_data(package, resource) # type: ignore
def merge_multiple_dicts(*args: Any) -> Dict[str, Any]:
diff --git a/setup.cfg b/setup.cfg
index 8d4c9bcf098e..ed8590084c7e 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -55,6 +55,22 @@ all =
py-partiql-parser==0.3.7
aws-xray-sdk!=0.96,>=0.93
setuptools
+ multipart
+proxy =
+ python-jose[cryptography]>=3.1.0,<4.0.0
+ ecdsa!=0.15
+ docker>=2.5.1
+ graphql-core
+ PyYAML>=5.1
+ cfn-lint>=0.40.0
+ sshpubkeys>=3.1.0
+ openapi-spec-validator>=0.2.8
+ pyparsing>=3.0.7
+ jsondiff>=1.1.2
+ py-partiql-parser==0.3.7
+ aws-xray-sdk!=0.96,>=0.93
+ setuptools
+ multipart
server =
python-jose[cryptography]>=3.1.0,<4.0.0
ecdsa!=0.15
@@ -229,6 +245,7 @@ xray =
[options.entry_points]
console_scripts =
moto_server = moto.server:main
+ moto_proxy = moto.proxy:main
[bdist_wheel]
universal=1
diff --git a/tests/test_acm/test_acm.py b/tests/test_acm/test_acm.py
index c350e70ac925..a1f1d403616a 100644
--- a/tests/test_acm/test_acm.py
+++ b/tests/test_acm/test_acm.py
@@ -581,7 +581,7 @@ def test_request_certificate_issued_status():
assert resp["Certificate"]["CertificateArn"] == arn
assert resp["Certificate"]["Status"] == "PENDING_VALIDATION"
- if not settings.TEST_SERVER_MODE:
+ if settings.TEST_DECORATOR_MODE:
# Move time to get it issued.
with freeze_time("2012-01-01 12:02:00"):
resp = client.describe_certificate(CertificateArn=arn)
@@ -593,7 +593,7 @@ def test_request_certificate_issued_status():
@mock_acm
def test_request_certificate_issued_status_with_wait_in_envvar():
# After requesting a certificate, it should then auto-validate after 3 seconds
- if settings.TEST_SERVER_MODE:
+ if not settings.TEST_DECORATOR_MODE:
raise SkipTest("Cant manipulate time in server mode")
client = boto3.client("acm", region_name="eu-central-1")
@@ -621,7 +621,7 @@ def test_request_certificate_issued_status_with_wait_in_envvar():
@mock_acm
def test_request_certificate_with_mutiple_times():
- if settings.TEST_SERVER_MODE:
+ if not settings.TEST_DECORATOR_MODE:
raise SkipTest("Cant manipulate time in server mode")
# After requesting a certificate, it should then auto-validate after 1 minute
diff --git a/tests/test_athena/test_athena.py b/tests/test_athena/test_athena.py
index af2973f061e2..143d7f5ccd5b 100644
--- a/tests/test_athena/test_athena.py
+++ b/tests/test_athena/test_athena.py
@@ -323,7 +323,7 @@ def test_get_query_results_queue():
assert result["ResultSet"]["Rows"] == []
assert result["ResultSet"]["ResultSetMetadata"]["ColumnInfo"] == []
- if not settings.TEST_SERVER_MODE:
+ if settings.TEST_DECORATOR_MODE:
backend = athena_backends[DEFAULT_ACCOUNT_ID]["us-east-1"]
rows = [{"Data": [{"VarCharValue": ".."}]}]
column_info = [
diff --git a/tests/test_awslambda/test_lambda.py b/tests/test_awslambda/test_lambda.py
index 2899a3eb7bb6..fd57c91106db 100644
--- a/tests/test_awslambda/test_lambda.py
+++ b/tests/test_awslambda/test_lambda.py
@@ -30,6 +30,8 @@
@pytest.mark.parametrize("region", ["us-west-2", "cn-northwest-1", "us-isob-east-1"])
@mock_lambda
def test_lambda_regions(region):
+ if not settings.TEST_DECORATOR_MODE:
+ raise SkipTest("Can only set EnvironVars in DecoratorMode")
client = boto3.client("lambda", region_name=region)
resp = client.list_functions()
assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200
diff --git a/tests/test_awslambda/test_lambda_invoke.py b/tests/test_awslambda/test_lambda_invoke.py
index f83ddd71a7f2..3507d234e8fb 100644
--- a/tests/test_awslambda/test_lambda_invoke.py
+++ b/tests/test_awslambda/test_lambda_invoke.py
@@ -16,6 +16,7 @@
get_lambda_using_environment_port,
get_lambda_using_network_mode,
get_test_zip_largeresponse,
+ get_proxy_zip_file,
)
from ..markers import requires_docker
@@ -339,3 +340,39 @@ def test_invoke_function_large_response():
# Absolutely fine when invoking async
resp = conn.invoke(FunctionName=fxn["FunctionArn"], InvocationType="Event")
assert "FunctionError" not in resp
+
+
+@mock_lambda
+def test_invoke_lambda_with_proxy():
+ if not settings.test_proxy_mode():
+ raise SkipTest("We only want to test this in ProxyMode")
+
+ conn = boto3.resource("ec2", _lambda_region)
+ vol = conn.create_volume(Size=99, AvailabilityZone=_lambda_region)
+ vol = conn.Volume(vol.id)
+
+ conn = boto3.client("lambda", _lambda_region)
+ function_name = str(uuid4())[0:6]
+ conn.create_function(
+ FunctionName=function_name,
+ Runtime=PYTHON_VERSION,
+ Role=get_role_name(),
+ Handler="lambda_function.lambda_handler",
+ Code={"ZipFile": get_proxy_zip_file()},
+ Description="test lambda function",
+ Timeout=3,
+ MemorySize=128,
+ Publish=True,
+ )
+
+ in_data = {"volume_id": vol.id}
+ result = conn.invoke(
+ FunctionName=function_name,
+ InvocationType="RequestResponse",
+ Payload=json.dumps(in_data),
+ )
+ assert result["StatusCode"] == 200
+ payload = result["Payload"].read().decode("utf-8")
+
+ expected_payload = {"id": vol.id, "state": vol.state, "size": vol.size}
+ assert json.loads(payload) == expected_payload
diff --git a/tests/test_awslambda/utilities.py b/tests/test_awslambda/utilities.py
index ad2c9edd32fa..1495e886136e 100644
--- a/tests/test_awslambda/utilities.py
+++ b/tests/test_awslambda/utilities.py
@@ -85,6 +85,21 @@ def lambda_handler(event, context):
return _process_lambda(func_str)
+def get_proxy_zip_file():
+ func_str = """
+import boto3
+
+def lambda_handler(event, context):
+ ec2 = boto3.resource('ec2', region_name='us-west-2')
+
+ volume_id = event.get('volume_id')
+ vol = ec2.Volume(volume_id)
+
+ return {'id': vol.id, 'state': vol.state, 'size': vol.size}
+"""
+ return _process_lambda(func_str)
+
+
def get_test_zip_file3():
pfunc = """
def lambda_handler(event, context):
diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py
index 7b270d0da061..1c06a3a0d793 100644
--- a/tests/test_s3/test_s3.py
+++ b/tests/test_s3/test_s3.py
@@ -21,6 +21,7 @@
from moto import settings, mock_s3, mock_config
from moto.moto_api import state_manager
from moto.core.utils import utcnow
+from moto import moto_proxy
from moto.s3.responses import DEFAULT_REGION_NAME
import moto.s3.models as s3model
@@ -113,7 +114,7 @@ def test_key_save_to_missing_bucket():
@mock_s3
def test_missing_key_request():
if not settings.TEST_DECORATOR_MODE:
- raise SkipTest("Only test status code in non-ServerMode")
+ raise SkipTest("Only test status code in DecoratorMode")
s3_client = boto3.client("s3", region_name=DEFAULT_REGION_NAME)
s3_client.create_bucket(Bucket="foobar")
@@ -460,6 +461,8 @@ def test_bucket_name_with_special_chars(name):
)
@mock_s3
def test_key_with_special_characters(key):
+ if settings.test_proxy_mode():
+ raise SkipTest("Keys starting with a / don't work well in ProxyMode")
s3_resource = boto3.resource("s3", region_name=DEFAULT_REGION_NAME)
client = boto3.client("s3", region_name=DEFAULT_REGION_NAME)
bucket = s3_resource.Bucket("testname")
@@ -774,7 +777,10 @@ def test_streaming_upload_from_file_to_presigned_url():
"put_object", params, ExpiresIn=900
)
with open(__file__, "rb") as fhandle:
- response = requests.get(presigned_url, data=fhandle)
+ get_kwargs = {"data": fhandle}
+ if settings.test_proxy_mode():
+ add_proxy_details(get_kwargs)
+ response = requests.get(presigned_url, **get_kwargs)
assert response.status_code == 200
@@ -793,7 +799,10 @@ def test_upload_from_file_to_presigned_url():
file.close()
files = {"upload_file": open("text.txt", "rb")}
- requests.put(presigned_url, files=files)
+ put_kwargs = {"files": files}
+ if settings.test_proxy_mode():
+ add_proxy_details(put_kwargs)
+ requests.put(presigned_url, **put_kwargs)
resp = s3_client.get_object(Bucket="mybucket", Key="file_upload")
data = resp["Body"].read()
assert data == b"test"
@@ -2837,6 +2846,8 @@ def test_root_dir_with_empty_name_works():
@mock_s3
def test_leading_slashes_not_removed(bucket_name):
"""Make sure that leading slashes are not removed internally."""
+ if settings.test_proxy_mode():
+ raise SkipTest("Doesn't quite work right with the Proxy")
s3_client = boto3.client("s3", region_name=DEFAULT_REGION_NAME)
s3_client.create_bucket(Bucket=bucket_name)
@@ -3020,9 +3031,14 @@ def test_creating_presigned_post():
Conditions=conditions,
ExpiresIn=1000,
)
- resp = requests.post(
- data["url"], data=data["fields"], files={"file": fdata}, allow_redirects=False
- )
+ kwargs = {
+ "data": data["fields"],
+ "files": {"file": fdata},
+ "allow_redirects": False,
+ }
+ if settings.test_proxy_mode():
+ add_proxy_details(kwargs)
+ resp = requests.post(data["url"], **kwargs)
assert resp.status_code == 303
redirect = resp.headers["Location"]
assert redirect.startswith(success_url)
@@ -3051,7 +3067,10 @@ def test_presigned_put_url_with_approved_headers():
)
# Verify S3 throws an error when the header is not provided
- response = requests.put(url, data=content)
+ kwargs = {"data": content}
+ if settings.test_proxy_mode():
+ add_proxy_details(kwargs)
+ response = requests.put(url, **kwargs)
assert response.status_code == 403
assert "SignatureDoesNotMatch
" in str(response.content)
assert (
@@ -3060,9 +3079,10 @@ def test_presigned_put_url_with_approved_headers():
) in str(response.content)
# Verify S3 throws an error when the header has the wrong value
- response = requests.put(
- url, data=content, headers={"Content-Type": "application/unknown"}
- )
+ kwargs = {"data": content, "headers": {"Content-Type": "application/unknown"}}
+ if settings.test_proxy_mode():
+ add_proxy_details(kwargs)
+ response = requests.put(url, **kwargs)
assert response.status_code == 403
assert "SignatureDoesNotMatch
" in str(response.content)
assert (
@@ -3071,9 +3091,10 @@ def test_presigned_put_url_with_approved_headers():
) in str(response.content)
# Verify S3 uploads correctly when providing the meta data
- response = requests.put(
- url, data=content, headers={"Content-Type": expected_contenttype}
- )
+ kwargs = {"data": content, "headers": {"Content-Type": expected_contenttype}}
+ if settings.test_proxy_mode():
+ add_proxy_details(kwargs)
+ response = requests.put(url, **kwargs)
assert response.status_code == 200
# Assert the object exists
@@ -3103,7 +3124,10 @@ def test_presigned_put_url_with_custom_headers():
)
# Verify S3 uploads correctly when providing the meta data
- response = requests.put(url, data=content)
+ kwargs = {"data": content}
+ if settings.test_proxy_mode():
+ add_proxy_details(kwargs)
+ response = requests.put(url, **kwargs)
assert response.status_code == 200
# Assert the object exists
@@ -3430,3 +3454,8 @@ def test_checksum_response(algorithm):
ChecksumAlgorithm=algorithm,
)
assert f"Checksum{algorithm}" in response
+
+
+def add_proxy_details(kwargs):
+ kwargs["proxies"] = {"https": "http://localhost:5005"}
+ kwargs["verify"] = moto_proxy.__file__.replace("__init__.py", "ca.crt")
diff --git a/tests/test_s3/test_s3_acl.py b/tests/test_s3/test_s3_acl.py
index 798b6c5e7a7c..b355973b41bf 100644
--- a/tests/test_s3/test_s3_acl.py
+++ b/tests/test_s3/test_s3_acl.py
@@ -7,7 +7,8 @@
from botocore.exceptions import ClientError
from botocore.handlers import disable_signing
-from moto import mock_s3
+from moto import mock_s3, settings
+from .test_s3 import add_proxy_details
DEFAULT_REGION_NAME = "us-east-1"
@@ -116,8 +117,11 @@ def test_s3_object_in_public_bucket_using_multiple_presigned_urls():
presigned_url = boto3.client("s3").generate_presigned_url(
"get_object", params, ExpiresIn=900
)
+ kwargs = {}
+ if settings.test_proxy_mode():
+ add_proxy_details(kwargs)
for i in range(1, 10):
- response = requests.get(presigned_url)
+ response = requests.get(presigned_url, **kwargs)
assert response.status_code == 200, f"Failed on req number {i}"
@@ -263,8 +267,10 @@ def test_object_acl_with_presigned_post():
)
with open(object_name, "rb") as fhandle:
- files = {"file": (object_name, fhandle)}
- requests.post(response["url"], data=response["fields"], files=files)
+ kwargs = {"files": {"file": (object_name, fhandle)}}
+ if settings.test_proxy_mode():
+ add_proxy_details(kwargs)
+ requests.post(response["url"], data=response["fields"], **kwargs)
response = s3_client.get_object_acl(Bucket=bucket_name, Key=object_name)
diff --git a/tests/test_s3/test_s3_bucket_policy.py b/tests/test_s3/test_s3_bucket_policy.py
index 7a100fcf40e2..3fdd5e595ff7 100644
--- a/tests/test_s3/test_s3_bucket_policy.py
+++ b/tests/test_s3/test_s3_bucket_policy.py
@@ -5,12 +5,16 @@
import pytest
from botocore.exceptions import ClientError
+from moto import settings
from moto.moto_server.threaded_moto_server import ThreadedMotoServer
+from unittest import SkipTest
class TestBucketPolicy:
@classmethod
def setup_class(cls):
+ if not settings.TEST_DECORATOR_MODE:
+ raise SkipTest("No point testing the ThreadedServer in Server/Proxy-mode")
cls.server = ThreadedMotoServer(port="6000", verbose=False)
cls.server.start()
diff --git a/tests/test_s3/test_s3_file_handles.py b/tests/test_s3/test_s3_file_handles.py
index 63bea4885663..c13e72492406 100644
--- a/tests/test_s3/test_s3_file_handles.py
+++ b/tests/test_s3/test_s3_file_handles.py
@@ -230,6 +230,8 @@ def test_reset_other_backend(self):
class TestS3FileHandleClosuresUsingMocks(TestCase):
def setUp(self) -> None:
+ if not settings.TEST_DECORATOR_MODE:
+ raise SkipTest("No point in testing ServerMode, we're not using boto3")
self.s3_client = boto3.client("s3", "us-east-1")
@verify_zero_warnings
diff --git a/tests/test_s3/test_s3_multipart.py b/tests/test_s3/test_s3_multipart.py
index 7f2259b6369d..0040c0c3abbd 100644
--- a/tests/test_s3/test_s3_multipart.py
+++ b/tests/test_s3/test_s3_multipart.py
@@ -11,7 +11,12 @@
from moto import settings, mock_s3
import moto.s3.models as s3model
from moto.s3.responses import DEFAULT_REGION_NAME
-from moto.settings import get_s3_default_key_buffer_size, S3_UPLOAD_PART_MIN_SIZE
+from moto.settings import (
+ get_s3_default_key_buffer_size,
+ S3_UPLOAD_PART_MIN_SIZE,
+ test_proxy_mode,
+)
+from .test_s3 import add_proxy_details
if settings.TEST_DECORATOR_MODE:
REDUCED_PART_SIZE = 256
@@ -977,7 +982,10 @@ def test_generate_presigned_url_on_multipart_upload_without_acl():
url = client.generate_presigned_url(
"head_object", Params={"Bucket": bucket_name, "Key": object_key}
)
- res = requests.get(url)
+ kwargs = {}
+ if test_proxy_mode():
+ add_proxy_details(kwargs)
+ res = requests.get(url, **kwargs)
assert res.status_code == 200
diff --git a/tests/test_s3/test_s3_tagging.py b/tests/test_s3/test_s3_tagging.py
index 8abd6a9da829..ae1755c77d8b 100644
--- a/tests/test_s3/test_s3_tagging.py
+++ b/tests/test_s3/test_s3_tagging.py
@@ -3,8 +3,9 @@
import pytest
import requests
-from moto import mock_s3
+from moto import mock_s3, settings
from moto.s3.responses import DEFAULT_REGION_NAME
+from .test_s3 import add_proxy_details
@mock_s3
@@ -470,6 +471,9 @@ def test_generate_url_for_tagged_object():
url = s3_client.generate_presigned_url(
"get_object", Params={"Bucket": "my-bucket", "Key": "test.txt"}
)
- response = requests.get(url)
+ kwargs = {}
+ if settings.test_proxy_mode():
+ add_proxy_details(kwargs)
+ response = requests.get(url, **kwargs)
assert response.content == b"abc"
assert response.headers["x-amz-tagging-count"] == "1"