From 89fdaf4dea56872013560151fa2f879648a5cf27 Mon Sep 17 00:00:00 2001 From: Abraham Chavez Date: Thu, 4 May 2023 11:05:58 -0700 Subject: [PATCH] [u] Send GitLab host logs to CloudWatch (#3894) --- UPGRADING.rst | 10 ++ terraform/gitlab/gitlab.tf.json.template.py | 105 +++++++++++++++++++- 2 files changed, 112 insertions(+), 3 deletions(-) diff --git a/UPGRADING.rst b/UPGRADING.rst index e0ec5587e1..620d53b9c1 100644 --- a/UPGRADING.rst +++ b/UPGRADING.rst @@ -19,6 +19,16 @@ branch that does not have the listed changes, the steps would need to be reverted. This is all fairly informal and loosely defined. Hopefully we won't have too many entries in this file. +#3894 Send GitLab host logs to CloudWatch +========================================= + +Operator +~~~~~~~~ + +Manually deploy the ``gitlab`` component of any main deployment just before +pushing the merge commit to the GitLab instance in that deployment. + + #5207 Fix: Partition sizing ignores supplementary bundles ========================================================= diff --git a/terraform/gitlab/gitlab.tf.json.template.py b/terraform/gitlab/gitlab.tf.json.template.py index 8a880d751e..117b92bebf 100644 --- a/terraform/gitlab/gitlab.tf.json.template.py +++ b/terraform/gitlab/gitlab.tf.json.template.py @@ -688,7 +688,15 @@ def qq(*words): 'config:BatchGetResourceConfig' ], 'resources': ['*'] - } + }, + { + 'actions': [ + 'logs:CreateLogGroup', + 'logs:CreateLogStream', + 'logs:PutLogEvents' + ], + 'resources': ['arn:aws:logs:*:*:*'] + }, ] } }, @@ -900,6 +908,10 @@ def qq(*words): 'gitlab_vpc': { 'name': '/aws/vpc/azul-gitlab', 'retention_in_days': config.audit_log_retention_days, + }, + 'gitlab_cwagent': { + 'name': '/aws/cwagent/azul-gitlab', + 'retention_in_days': config.audit_log_retention_days, } }, 'aws_flow_log': { @@ -1305,7 +1317,7 @@ def qq(*words): 'mounts': [ ['/dev/nvme1n1', '/mnt/gitlab', 'ext4', ''] ], - 'packages': ['docker'], + 'packages': ['docker', 'amazon-cloudwatch-agent'], 'ssh_authorized_keys': other_public_keys.get(config.deployment_stage, []), 'write_files': [ { @@ -1368,6 +1380,7 @@ def qq(*words): 'ExecStart=/usr/bin/docker', 'run', '--name gitlab', + '--env GITLAB_SKIP_TAIL_LOGS=true', '--hostname ${aws_route53_record.gitlab.name}', '--publish 80:80', '--publish 2222:22', @@ -1527,7 +1540,86 @@ def qq(*words): '[Install]', 'WantedBy=timers.target' ) - } + }, + { + # AWS recommends placing the amazon-cloudwatch-agent config file at this path. + # Note that the parent of etc/ is where the agent is installed. + 'path': '/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json', + 'permissions': '0664', + 'owner': 'root', + 'content': json.dumps({ + 'agent': { + 'region': aws.region_name, + 'logfile': '/opt/aws/amazon-cloudwatch-agent/logs/amazon-cloudwatch-agent.log', + 'debug': bool(config.debug) + }, + 'logs': { + 'logs_collected': { + 'files': { + 'collect_list': [ + { + 'file_path': path, + 'log_group_name': '${aws_cloudwatch_log_group.gitlab_cwagent.name}', + # https://docs.aws.amazon.com/AmazonCloudWatchLogs/latest/APIReference/API_CreateLogStream.html + # Characters disallowed for use in a log stream name are `:` and + # `*`, so we replace any occurrence of `*` in `path` with `?`. + 'log_stream_name': path.replace('*', '?') + } + for path in + [ + f'/var/log/{file}' + for file in + [ + 'amazon/ssm/amazon-ssm-agent.log', + 'audit/audit.log', + 'cloud-init.log', + 'cron', + 'maillog', + 'messages', + 'secure' + ] + + ] + [ + f'/mnt/gitlab/logs/{file}.log' + for file in + [ + 'gitaly/gitaly_ruby_json', + 'gitlab-shell/gitlab-shell', + 'nginx/gitlab_access', + 'nginx/gitlab_error', + 'nginx/gitlab_registry_access', + 'puma/puma_stderr', + 'puma/puma_stdout', + # The '*' is used in order to get the most recent GitLab + # reconfigure logs (name based on UNIX timestamp of when + # reconfigure initiated). Only the most recent file, by + # modification time, matching the wildcard is collected. + 'reconfigure/*' + ] + ] + [ + f'/mnt/gitlab/logs/gitlab-rails/{file}.log' + for file in + [ + 'api_json', + 'application_json', + 'application', + 'audit_json', + 'auth', + 'database_load_balancing', + 'exceptions_json', + 'graphql_json', + 'migrations', + 'production_json', + 'production', + 'sidekiq_client' + ] + ] + ] + } + } + } + }, indent=4) + }, ], 'runcmd': [ ['systemctl', 'daemon-reload'], @@ -1542,6 +1634,13 @@ def qq(*words): 'gitlab-runner', 'clamscan.timer', 'prune-images.timer' + ], + [ + 'amazon-cloudwatch-agent-ctl', + '-m', 'ec2', # mode + '-a', 'fetch-config', # action (fetch file from location specified in -c) + '-c', 'file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json', + '-s' # restart agent afterwards ] ], }, indent=2),