diff --git a/config.yaml b/config.yaml index 745a8f2..197ebca 100644 --- a/config.yaml +++ b/config.yaml @@ -154,3 +154,118 @@ options: type: string default: "/etc/docker/registry/registry.key" description: Path to the TLS certificate private key. + storage-s3-accesskey: + type: string + default: + description: | + S3 storage access key. More info about S3 storage configuration could be found here: + https://distribution.github.io/distribution/storage-drivers/s3/ . + storage-s3-secretkey: + type: string + default: + description: S3 storage secret key. + storage-s3-region: + type: string + default: + description: The region containing the S3 service. + storage-s3-regionendpoint: + type: string + default: + description: Endpoint for S3 compatible storage services (Minio, etc). + storage-s3-forcepathstyle: + type: boolean + default: false + description: To enable path-style addressing when the value is set to "true". + storage-s3-bucket: + type: string + default: + description: The bucket name in which you want to store the registry’s data. + storage-s3-encrypt: + type: boolean + default: false + description: Specifies whether the registry stores the image in encrypted format or not. + storage-s3-keyid: + type: string + default: + description: | + Optional KMS key ID to use for encryption + (encrypt must be true, or this parameter is ignored). + storage-s3-secure: + type: boolean + default: true + description: Indicates whether to use HTTPS instead of HTTP. + storage-s3-skip-verify: + type: boolean + default: false + description: Skips TLS verification when the value is set to "true". + storage-s3-skip-v4-auth: + type: boolean + default: true + description: Indicates whether the registry uses Version 4 of AWS’s authentication. + storage-s3-chunksize: + type: int + default: 10485760 + description: | + The S3 API requires multipart upload chunks to be at least 5MB. + This value should be a number that is larger than 5 * 1024 * 1024. + storage-s3-multipartcopychunksize: + type: int + default: 33554432 + description: | + The default chunk size for all but the last Upload Part in the S3 Multipart Upload + operation when copying stored objects. + storage-s3-multipartcopymaxconcurrency: + type: int + default: 100 + description: | + The default maximum number of concurrent Upload Part operations in the + S3 Multipart Upload when copying stored objects. + storage-s3-multipartcopythresholdsize: + type: int + default: 33554432 + description: | + The default S3 object size above which multipart copy will be used when copying the + object. Otherwise the object is copied with a single S3 API operation. + storage-s3-rootdirectory: + type: string + default: + description: The root directory tree in which all registry files are stored. + storage-s3-storageclass: + type: string + default: "STANDARD" + description: | + The storage class applied to each registry file. + Valid options are STANDARD and REDUCED_REDUNDANCY. + storage-s3-useragent: + type: string + default: + description: The User-Agent header value for S3 API operations. + storage-s3-usedualstack: + type: boolean + default: false + description: | + Use AWS dual-stack API endpoints which support requests to + S3 buckets over IPv6 and IPv4. + storage-s3-accelerate: + type: boolean + default: false + description: | + Enable S3 transfer acceleration for faster transfers of files over + long distances. + storage-s3-objectacl: + type: string + default: "private" + description: | + The canned object ACL to be applied to each registry object. + If you are using a bucket owned by another AWS account, it is recommended that you set this + to "bucket-owner-full-control" so that the bucket owner can access your objects. + Other valid options are available in the AWS S3 documentation: + https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl + storage-s3-loglevel: + type: string + default: "off" + description: | + Valid values are: off (default), debug, debugwithsigning, debugwithhttpbody, + debugwithrequestretries, debugwithrequesterrors and debugwitheventstreambody. + See the AWS SDK for Go API reference for details: + https://docs.aws.amazon.com/sdk-for-go/api/aws/#LogLevelType \ No newline at end of file diff --git a/lib/charms/layer/docker_registry.py b/lib/charms/layer/docker_registry.py index 0a996a6..8abee11 100644 --- a/lib/charms/layer/docker_registry.py +++ b/lib/charms/layer/docker_registry.py @@ -144,6 +144,45 @@ def configure_registry(): storage['swift'].update({'domain': val}) storage['redirect'] = {'disable': True} + elif ( + charm_config.get('storage-s3-region') and + charm_config.get('storage-s3-bucket') + ): + storage['s3'] = { + 'region': charm_config.get('storage-s3-region'), + 'bucket': charm_config.get('storage-s3-bucket'), + 'forcepathstyle': charm_config.get('storage-s3-forcepathstyle', False), + 'encrypt': charm_config.get('storage-s3-encrypt', False), + 'secure': charm_config.get('storage-s3-secure', True), + 'skipverify': charm_config.get('storage-s3-skipverify', False), + 'v4auth': charm_config.get('storage-s3-v4auth', True), + 'chunksize': charm_config.get('storage-s3-chunksize', 10 * 1024 * 1024), + 'multipartcopychunksize': charm_config.get( + 'storage-s3-multipartcopychunksize', 30 * 1024 * 1024 + ), + 'multipartcopymaxconcurrency': charm_config.get( + 'storage-s3-multipartcopymaxconcurrency', 100 + ), + 'multipartcopythresholdsize': charm_config.get( + 'storage-s3-multipartcopythresholdsize', 30 * 1024 * 1024 + ), + 'storageclass': charm_config.get('storage-s3-storageclass', 'STANDARD'), + 'usedualstack': charm_config.get('storage-s3-usedualstack', False), + 'accelerate': charm_config.get('storage-s3-accelerate', False), + 'loglevel': charm_config.get('storage-s3-loglevel', 'off'), + } + optional_params = ( + 'accesskey', + 'secretkey', + 'regionendpoint', + 'keyid', + 'useragent', + 'rootdirectory', + ) + for short_name in optional_params: + full_name = f'storage-s3-{short_name}' + if charm_config.get(full_name): + storage['s3'][short_name] = charm_config.get(full_name) else: # If we're not swift, we're local. container_registry_path = '/var/lib/registry' diff --git a/tests/unit/test_docker_registry.py b/tests/unit/test_docker_registry.py index 243a836..7034ff2 100644 --- a/tests/unit/test_docker_registry.py +++ b/tests/unit/test_docker_registry.py @@ -76,3 +76,153 @@ def test_has_invalid_config(config): "storage-cache": "bananas", } assert "storage-cache" in layer.docker_registry.has_invalid_config() + + +@mock.patch("os.makedirs", mock.Mock(return_value=0)) +@mock.patch("charms.layer.docker_registry._write_tls_blobs_to_files") +@mock.patch("charms.layer.docker_registry._configure_local_client") +@mock.patch("charms.layer.docker_registry._write_tls_blobs_to_files") +@mock.patch("charms.layer.docker_registry.unitdata") +@mock.patch("charmhelpers.core.hookenv.config") +def test_configure_registry_s3_storage_smoke(config, *args): + config.return_value = { + "log-level": "info", + "storage-s3-region": "ns1", + "storage-s3-bucket": "test_bucket", + } + expected_storage = { + "s3": { + "bucket": "test_bucket", + "region": "ns1", + # "regionendpoint": "https://ns1-region.internal", + "forcepathstyle": False, + "encrypt": False, + "secure": True, + "skipverify": False, + "v4auth": True, + "chunksize": 10485760, + "multipartcopychunksize": 31457280, + "multipartcopymaxconcurrency": 100, + "multipartcopythresholdsize": 31457280, + "storageclass": "STANDARD", + "usedualstack": False, + "accelerate": False, + "loglevel": "off", + }, + } + with mock.patch("charms.layer.docker_registry.yaml") as mock_yaml: + layer.docker_registry.configure_registry() + args, _ = mock_yaml.safe_dump.call_args_list[0] + assert 'storage' in args[0] + assert 's3' in args[0]['storage'] + actual_storage_config = args[0]['storage']['s3'] + assert expected_storage['s3'].items() == actual_storage_config.items() + + + +@mock.patch("os.makedirs", mock.Mock(return_value=0)) +@mock.patch("charms.layer.docker_registry._write_tls_blobs_to_files") +@mock.patch("charms.layer.docker_registry._configure_local_client") +@mock.patch("charms.layer.docker_registry._write_tls_blobs_to_files") +@mock.patch("charms.layer.docker_registry.unitdata") +@mock.patch("charmhelpers.core.hookenv.config") +def test_configure_registry_s3_storage_region_endpoint(config, *args): + config.return_value = { + "log-level": "info", + "storage-s3-region": "ns1", + "storage-s3-regionendpoint": "https://ns1-region.internal", + "storage-s3-bucket": "test_bucket", + } + expected_storage = { + "s3": { + "bucket": "test_bucket", + "region": "ns1", + "regionendpoint": "https://ns1-region.internal", + "forcepathstyle": False, + "encrypt": False, + "secure": True, + "skipverify": False, + "v4auth": True, + "chunksize": 10485760, + "multipartcopychunksize": 31457280, + "multipartcopymaxconcurrency": 100, + "multipartcopythresholdsize": 31457280, + "storageclass": "STANDARD", + "usedualstack": False, + "accelerate": False, + "loglevel": "off", + }, + } + with mock.patch("charms.layer.docker_registry.yaml") as mock_yaml: + layer.docker_registry.configure_registry() + args, _ = mock_yaml.safe_dump.call_args_list[0] + assert 'storage' in args[0] + assert 's3' in args[0]['storage'] + actual_storage_config = args[0]['storage']['s3'] + assert expected_storage['s3'].items() == actual_storage_config.items() + + +@mock.patch("os.makedirs", mock.Mock(return_value=0)) +@mock.patch("charms.layer.docker_registry._write_tls_blobs_to_files") +@mock.patch("charms.layer.docker_registry._configure_local_client") +@mock.patch("charms.layer.docker_registry._write_tls_blobs_to_files") +@mock.patch("charms.layer.docker_registry.unitdata") +@mock.patch("charmhelpers.core.hookenv.config") +def test_configure_registry_s3_storage_override_default(config, *args): + config.return_value = { + "log-level": "info", + "storage-s3-region": "ns1", + "storage-s3-bucket": "test_bucket", + "storage-s3-forcepathstyle": True, + "storage-s3-multipartcopythresholdsize": 100500, + } + expected_storage = { + "s3": { + "bucket": "test_bucket", + "region": "ns1", + "forcepathstyle": True, + "encrypt": False, + "secure": True, + "skipverify": False, + "v4auth": True, + "chunksize": 10485760, + "multipartcopychunksize": 31457280, + "multipartcopymaxconcurrency": 100, + "multipartcopythresholdsize": 100500, + "storageclass": "STANDARD", + "usedualstack": False, + "accelerate": False, + "loglevel": "off", + }, + } + with mock.patch("charms.layer.docker_registry.yaml") as mock_yaml: + layer.docker_registry.configure_registry() + args, _ = mock_yaml.safe_dump.call_args_list[0] + assert 'storage' in args[0] + assert 's3' in args[0]['storage'] + actual_storage_config = args[0]['storage']['s3'] + assert expected_storage['s3'].items() == actual_storage_config.items() + + +@mock.patch("os.makedirs", mock.Mock(return_value=0)) +@mock.patch("charms.layer.docker_registry._write_tls_blobs_to_files") +@mock.patch("charms.layer.docker_registry._configure_local_client") +@mock.patch("charms.layer.docker_registry._write_tls_blobs_to_files") +@mock.patch("charms.layer.docker_registry.unitdata") +@mock.patch("charmhelpers.core.hookenv.config") +def test_configure_registry_default_file_storage(config, *args): + config.return_value = { + "log-level": "info" + } + expected_storage = { + "filesystem": { + "rootdirectory": "/var/lib/registry" + } + } + with mock.patch("charms.layer.docker_registry.yaml") as mock_yaml: + layer.docker_registry.configure_registry() + args, _ = mock_yaml.safe_dump.call_args_list[0] + assert 'storage' in args[0] + assert 'filesystem' in args[0]['storage'] + actual_storage_config = args[0]['storage']['filesystem'] + assert expected_storage['filesystem'].items() == actual_storage_config.items() \ No newline at end of file