From a3a23a4bc660291fa2b5fd3d77092383b4ea0c09 Mon Sep 17 00:00:00 2001 From: David Rubin Date: Tue, 26 Dec 2017 20:51:01 +0100 Subject: [PATCH 1/4] Correct template file for AWS_BACKUP_MULTIPART_CHUNK_SIZE env Fixes #1293 Correctly setup and configure the template in order to use chunked upload. --- assets/runtime/config/gitlabhq/gitlab.yml | 6 +++--- assets/runtime/env-defaults | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/assets/runtime/config/gitlabhq/gitlab.yml b/assets/runtime/config/gitlabhq/gitlab.yml index 2bc50aa23..0eb5c121c 100644 --- a/assets/runtime/config/gitlabhq/gitlab.yml +++ b/assets/runtime/config/gitlabhq/gitlab.yml @@ -512,9 +512,9 @@ production: &base aws_secret_access_key: '{{AWS_BACKUP_SECRET_ACCESS_KEY}}' # The remote 'directory' to store your backups. For S3, this would be the bucket name. remote_directory: '{{AWS_BACKUP_BUCKET}}' - # # Use multipart uploads when file size reaches 100MB, see - # # http://docs.aws.amazon.com/AmazonS3/latest/dev/uploadobjusingmpu.html - # multipart_chunk_size: 104857600 + # Use multipart uploads when file size reaches 100MB, see + # http://docs.aws.amazon.com/AmazonS3/latest/dev/uploadobjusingmpu.html + multipart_chunk_size: {{AWS_BACKUP_MULTIPART_CHUNK_SIZE}} # # Turns on AWS Server-Side Encryption with Amazon S3-Managed Keys for backups, this is optional # # encryption: 'AES256' # Fog storage connection settings, see http://fog.io/storage/ . diff --git a/assets/runtime/env-defaults b/assets/runtime/env-defaults index a039b1b11..66a65bcfc 100644 --- a/assets/runtime/env-defaults +++ b/assets/runtime/env-defaults @@ -157,7 +157,7 @@ AWS_BACKUP_PATH_STYLE=${AWS_BACKUP_PATH_STYLE:-false} AWS_BACKUP_ACCESS_KEY_ID=${AWS_BACKUP_ACCESS_KEY_ID} AWS_BACKUP_SECRET_ACCESS_KEY=${AWS_BACKUP_SECRET_ACCESS_KEY} AWS_BACKUP_BUCKET=${AWS_BACKUP_BUCKET} -AWS_BACKUP_MULTIPART_CHUNK_SIZE=${AWS_BACKUP_MULTIPART_CHUNK_SIZE} +AWS_BACKUP_MULTIPART_CHUNK_SIZE=${AWS_BACKUP_MULTIPART_CHUNK_SIZE:-104857600} ### GCS BACKUPS GCS_BACKUPS=${GCS_BACKUPS:-false} From 94d5f687df130d9436429ee434550ab289da7f57 Mon Sep 17 00:00:00 2001 From: David Rubin Date: Tue, 26 Dec 2017 21:58:22 +0100 Subject: [PATCH 2/4] Remove multipart configs if AWS_BACKUP_MULTIPART_CHUNK_SIZE is not set --- assets/runtime/config/gitlabhq/gitlab.yml | 8 +++++--- assets/runtime/env-defaults | 2 +- assets/runtime/functions | 8 ++++++++ 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/assets/runtime/config/gitlabhq/gitlab.yml b/assets/runtime/config/gitlabhq/gitlab.yml index 0eb5c121c..4b8f7d0ed 100644 --- a/assets/runtime/config/gitlabhq/gitlab.yml +++ b/assets/runtime/config/gitlabhq/gitlab.yml @@ -512,11 +512,13 @@ production: &base aws_secret_access_key: '{{AWS_BACKUP_SECRET_ACCESS_KEY}}' # The remote 'directory' to store your backups. For S3, this would be the bucket name. remote_directory: '{{AWS_BACKUP_BUCKET}}' + #start-multipart # Use multipart uploads when file size reaches 100MB, see - # http://docs.aws.amazon.com/AmazonS3/latest/dev/uploadobjusingmpu.html + # http://docs.aws.amazon.com/AmazonS3/latest/dev/uploadobjusingmpu.html multipart_chunk_size: {{AWS_BACKUP_MULTIPART_CHUNK_SIZE}} - # # Turns on AWS Server-Side Encryption with Amazon S3-Managed Keys for backups, this is optional - # # encryption: 'AES256' + #end-multipart + # # Turns on AWS Server-Side Encryption with Amazon S3-Managed Keys for backups, this is optional + # # encryption: 'AES256' # Fog storage connection settings, see http://fog.io/storage/ . #end-aws #start-gcs diff --git a/assets/runtime/env-defaults b/assets/runtime/env-defaults index 66a65bcfc..a039b1b11 100644 --- a/assets/runtime/env-defaults +++ b/assets/runtime/env-defaults @@ -157,7 +157,7 @@ AWS_BACKUP_PATH_STYLE=${AWS_BACKUP_PATH_STYLE:-false} AWS_BACKUP_ACCESS_KEY_ID=${AWS_BACKUP_ACCESS_KEY_ID} AWS_BACKUP_SECRET_ACCESS_KEY=${AWS_BACKUP_SECRET_ACCESS_KEY} AWS_BACKUP_BUCKET=${AWS_BACKUP_BUCKET} -AWS_BACKUP_MULTIPART_CHUNK_SIZE=${AWS_BACKUP_MULTIPART_CHUNK_SIZE:-104857600} +AWS_BACKUP_MULTIPART_CHUNK_SIZE=${AWS_BACKUP_MULTIPART_CHUNK_SIZE} ### GCS BACKUPS GCS_BACKUPS=${GCS_BACKUPS:-false} diff --git a/assets/runtime/functions b/assets/runtime/functions index e92f26190..33bbaaf20 100644 --- a/assets/runtime/functions +++ b/assets/runtime/functions @@ -786,6 +786,14 @@ gitlab_configure_backups_aws() { exec_as_git sed -i "/#start-aws/d" ${GITLAB_CONFIG} exec_as_git sed -i "/#end-aws/d" ${GITLAB_CONFIG} + if [[ -z ${AWS_BACKUP_MULTIPART_CHUNK_SIZE} ]]; then + exec_as_git sed -i "/#start-multipart/,/#end-multipart/d" ${GITLAB_CONFIG} + fi + + if [[ -z ${AWS_BACKUP_MULTIPART_CHUNK_SIZE} ]]; then + exec_as_git sed -i "/#start-multipart/,/#end-multipart/d" ${GITLAB_CONFIG} + fi + if [[ -z ${AWS_BACKUP_REGION} && -z ${AWS_BACKUP_ENDPOINT} ]]; then echo "\nMissing AWS region or endpoint. Aborting...\n" return 1 From 5ea41eb3d6e42e69331ece954f7469e9cc853f11 Mon Sep 17 00:00:00 2001 From: David Rubin Date: Fri, 29 Dec 2017 16:26:41 +0100 Subject: [PATCH 3/4] Add AWS_BACKUP_ENCRYPTION ENV to enable aws backup aes encryption backups Also refactor the multi part config to match --- README.md | 1 + assets/runtime/config/gitlabhq/gitlab.yml | 10 ++++++---- assets/runtime/env-defaults | 1 + assets/runtime/functions | 6 +++++- 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index b6ddb80c1..935f01de6 100644 --- a/README.md +++ b/README.md @@ -1027,6 +1027,7 @@ Below is the complete list of available options that can be used to customize yo | `AWS_BACKUP_SECRET_ACCESS_KEY` | AWS secret access key. No defaults. | | `AWS_BACKUP_BUCKET` | AWS bucket for backup uploads. No defaults. | | `AWS_BACKUP_MULTIPART_CHUNK_SIZE` | Enables mulitpart uploads when file size reaches a defined size. See at [AWS S3 Docs](http://docs.aws.amazon.com/AmazonS3/latest/dev/uploadobjusingmpu.html) | +| `AWS_BACKUP_ENCRYPTION` | Turns on AWS Server-Side Encryption. Defaults to `false`. See at [AWS s3 Docs](http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingServerSideEncryption.html) | | `GCS_BACKUPS` | Enables automatic uploads to an Google Cloud Storage (GCS) instance. Defaults to `false`. | | `GCS_BACKUP_ACCESS_KEY_ID` | GCS access key id. No defaults | | `GCS_BACKUP_SECRET_ACCESS_KEY` | GCS secret access key. No defaults | diff --git a/assets/runtime/config/gitlabhq/gitlab.yml b/assets/runtime/config/gitlabhq/gitlab.yml index 4b8f7d0ed..883a91ec3 100644 --- a/assets/runtime/config/gitlabhq/gitlab.yml +++ b/assets/runtime/config/gitlabhq/gitlab.yml @@ -512,13 +512,15 @@ production: &base aws_secret_access_key: '{{AWS_BACKUP_SECRET_ACCESS_KEY}}' # The remote 'directory' to store your backups. For S3, this would be the bucket name. remote_directory: '{{AWS_BACKUP_BUCKET}}' - #start-multipart + #start-multipart-aws # Use multipart uploads when file size reaches 100MB, see # http://docs.aws.amazon.com/AmazonS3/latest/dev/uploadobjusingmpu.html multipart_chunk_size: {{AWS_BACKUP_MULTIPART_CHUNK_SIZE}} - #end-multipart - # # Turns on AWS Server-Side Encryption with Amazon S3-Managed Keys for backups, this is optional - # # encryption: 'AES256' + #end-multipart-aws + #start-encryption-aws + # Turns on AWS Server-Side Encryption with Amazon S3-Managed Keys for backups, this is optional + encryption: 'AES256' + #end-encryption-aws # Fog storage connection settings, see http://fog.io/storage/ . #end-aws #start-gcs diff --git a/assets/runtime/env-defaults b/assets/runtime/env-defaults index a039b1b11..b39e164b1 100644 --- a/assets/runtime/env-defaults +++ b/assets/runtime/env-defaults @@ -158,6 +158,7 @@ AWS_BACKUP_ACCESS_KEY_ID=${AWS_BACKUP_ACCESS_KEY_ID} AWS_BACKUP_SECRET_ACCESS_KEY=${AWS_BACKUP_SECRET_ACCESS_KEY} AWS_BACKUP_BUCKET=${AWS_BACKUP_BUCKET} AWS_BACKUP_MULTIPART_CHUNK_SIZE=${AWS_BACKUP_MULTIPART_CHUNK_SIZE} +AWS_BACKUP_ENCRYPTION=${AWS_BACKUP_ENCRYPTION} ### GCS BACKUPS GCS_BACKUPS=${GCS_BACKUPS:-false} diff --git a/assets/runtime/functions b/assets/runtime/functions index 33bbaaf20..5d188a534 100644 --- a/assets/runtime/functions +++ b/assets/runtime/functions @@ -791,7 +791,11 @@ gitlab_configure_backups_aws() { fi if [[ -z ${AWS_BACKUP_MULTIPART_CHUNK_SIZE} ]]; then - exec_as_git sed -i "/#start-multipart/,/#end-multipart/d" ${GITLAB_CONFIG} + exec_as_git sed -i "/#start-multipart-aws/,/#end-multipart-aws/d" ${GITLAB_CONFIG} + fi + + if [[ ${AWS_BACKUP_ENCRYPTION} != true ]]; then + exec_as_git sed -i "/#start-encryption-aws/,/#end-encryption-aws/d" ${GITLAB_CONFIG} fi if [[ -z ${AWS_BACKUP_REGION} && -z ${AWS_BACKUP_ENDPOINT} ]]; then From 14c3225d03f245fb7f833998f8323a5567476d02 Mon Sep 17 00:00:00 2001 From: David Rubin Date: Fri, 29 Dec 2017 16:46:58 +0100 Subject: [PATCH 4/4] Add AWS_BACKUP_STORAGE_CLASS configuration option --- README.md | 3 ++- assets/runtime/config/gitlabhq/gitlab.yml | 2 ++ assets/runtime/env-defaults | 1 + assets/runtime/functions | 3 ++- 4 files changed, 7 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 935f01de6..3157b7ffe 100644 --- a/README.md +++ b/README.md @@ -1027,7 +1027,8 @@ Below is the complete list of available options that can be used to customize yo | `AWS_BACKUP_SECRET_ACCESS_KEY` | AWS secret access key. No defaults. | | `AWS_BACKUP_BUCKET` | AWS bucket for backup uploads. No defaults. | | `AWS_BACKUP_MULTIPART_CHUNK_SIZE` | Enables mulitpart uploads when file size reaches a defined size. See at [AWS S3 Docs](http://docs.aws.amazon.com/AmazonS3/latest/dev/uploadobjusingmpu.html) | -| `AWS_BACKUP_ENCRYPTION` | Turns on AWS Server-Side Encryption. Defaults to `false`. See at [AWS s3 Docs](http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingServerSideEncryption.html) | +| `AWS_BACKUP_ENCRYPTION` | Turns on AWS Server-Side Encryption. Defaults to `false`. See at [AWS S3 Docs](http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingServerSideEncryption.html) | +| `AWS_BACKUP_STORAGE_CLASS` | Configure the storage class for the item. Defaults to `STANDARD` See at [AWS S3 Docs](http://docs.aws.amazon.com/AmazonS3/latest/dev/storage-class-intro.html) | | `GCS_BACKUPS` | Enables automatic uploads to an Google Cloud Storage (GCS) instance. Defaults to `false`. | | `GCS_BACKUP_ACCESS_KEY_ID` | GCS access key id. No defaults | | `GCS_BACKUP_SECRET_ACCESS_KEY` | GCS secret access key. No defaults | diff --git a/assets/runtime/config/gitlabhq/gitlab.yml b/assets/runtime/config/gitlabhq/gitlab.yml index 883a91ec3..a2973187c 100644 --- a/assets/runtime/config/gitlabhq/gitlab.yml +++ b/assets/runtime/config/gitlabhq/gitlab.yml @@ -521,6 +521,8 @@ production: &base # Turns on AWS Server-Side Encryption with Amazon S3-Managed Keys for backups, this is optional encryption: 'AES256' #end-encryption-aws + # Specifies Amazon S3 storage class to use for backups, this is optional + storage_class: '{{AWS_BACKUP_STORAGE_CLASS}}' # Fog storage connection settings, see http://fog.io/storage/ . #end-aws #start-gcs diff --git a/assets/runtime/env-defaults b/assets/runtime/env-defaults index b39e164b1..5b0f2dbe6 100644 --- a/assets/runtime/env-defaults +++ b/assets/runtime/env-defaults @@ -159,6 +159,7 @@ AWS_BACKUP_SECRET_ACCESS_KEY=${AWS_BACKUP_SECRET_ACCESS_KEY} AWS_BACKUP_BUCKET=${AWS_BACKUP_BUCKET} AWS_BACKUP_MULTIPART_CHUNK_SIZE=${AWS_BACKUP_MULTIPART_CHUNK_SIZE} AWS_BACKUP_ENCRYPTION=${AWS_BACKUP_ENCRYPTION} +AWS_BACKUP_STORAGE_CLASS=${AWS_BACKUP_STORAGE_CLASS:-STANDARD} ### GCS BACKUPS GCS_BACKUPS=${GCS_BACKUPS:-false} diff --git a/assets/runtime/functions b/assets/runtime/functions index 5d188a534..620eed154 100644 --- a/assets/runtime/functions +++ b/assets/runtime/functions @@ -819,7 +819,8 @@ gitlab_configure_backups_aws() { AWS_BACKUP_ACCESS_KEY_ID \ AWS_BACKUP_SECRET_ACCESS_KEY \ AWS_BACKUP_BUCKET \ - AWS_BACKUP_MULTIPART_CHUNK_SIZE + AWS_BACKUP_MULTIPART_CHUNK_SIZE \ + AWS_BACKUP_STORAGE_CLASS } gitlab_configure_backup_gcs() {