From 53526c95d2e040c894d48c23190d2b5e52df0086 Mon Sep 17 00:00:00 2001 From: Abraham Chavez Date: Thu, 4 May 2023 11:05:58 -0700 Subject: [PATCH] [u] Send GitLab host logs to CloudWatch (#3894) --- UPGRADING.rst | 10 ++ terraform/gitlab/gitlab.tf.json.template.py | 105 +++++++++++++++++++- 2 files changed, 112 insertions(+), 3 deletions(-) diff --git a/UPGRADING.rst b/UPGRADING.rst index 0acccf70cd..e2d7160546 100644 --- a/UPGRADING.rst +++ b/UPGRADING.rst @@ -19,6 +19,16 @@ branch that does not have the listed changes, the steps would need to be reverted. This is all fairly informal and loosely defined. Hopefully we won't have too many entries in this file. +#3894 Send GitLab host logs to CloudWatch +========================================= + +Operator +~~~~~~~~ + +Manually deploy the ``gitlab`` component of any main deployment just before +pushing the merge commit to the GitLab instance in that deployment. + + #4022 Encrypt GitLab data and root volume and snapshots ======================================================= diff --git a/terraform/gitlab/gitlab.tf.json.template.py b/terraform/gitlab/gitlab.tf.json.template.py index 8a880d751e..428828db3c 100644 --- a/terraform/gitlab/gitlab.tf.json.template.py +++ b/terraform/gitlab/gitlab.tf.json.template.py @@ -688,7 +688,15 @@ def qq(*words): 'config:BatchGetResourceConfig' ], 'resources': ['*'] - } + }, + { + 'actions': [ + 'logs:CreateLogGroup', + 'logs:CreateLogStream', + 'logs:PutLogEvents' + ], + 'resources': ['arn:aws:logs:*:*:*'] + }, ] } }, @@ -900,6 +908,10 @@ def qq(*words): 'gitlab_vpc': { 'name': '/aws/vpc/azul-gitlab', 'retention_in_days': config.audit_log_retention_days, + }, + 'gitlab_cwagent': { + 'name': '/aws/cwagent/azul-gitlab', + 'retention_in_days': config.audit_log_retention_days, } }, 'aws_flow_log': { @@ -1305,7 +1317,7 @@ def qq(*words): 'mounts': [ ['/dev/nvme1n1', '/mnt/gitlab', 'ext4', ''] ], - 'packages': ['docker'], + 'packages': ['docker', 'amazon-cloudwatch-agent'], 'ssh_authorized_keys': other_public_keys.get(config.deployment_stage, []), 'write_files': [ { @@ -1368,6 +1380,7 @@ def qq(*words): 'ExecStart=/usr/bin/docker', 'run', '--name gitlab', + '--env GITLAB_SKIP_TAIL_LOGS=true', '--hostname ${aws_route53_record.gitlab.name}', '--publish 80:80', '--publish 2222:22', @@ -1527,7 +1540,86 @@ def qq(*words): '[Install]', 'WantedBy=timers.target' ) - } + }, + { + # AWS recommends placing the amazon-cloudwatch-agent config file at this path. + # Note that the parent of etc/ is where the agent is installed. + 'path': '/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json', + 'permissions': '0664', + 'owner': 'root', + 'content': json.dumps({ + 'agent': { + 'region': aws.region_name, + 'logfile': '/opt/aws/amazon-cloudwatch-agent/logs/amazon-cloudwatch-agent.log', + 'debug': bool(config.debug) + }, + 'logs': { + 'logs_collected': { + 'files': { + 'collect_list': [ + { + 'file_path': path, + 'log_group_name': '/aws/cwagent/azul-gitlab', + 'log_stream_name': + path + if not path.endswith('*.log') + else + path.replace('*', 'reconfigure') + } + for path in + [ + f'/mnt/gitlab/logs/{file}.log' + for file in + [ + 'gitaly/gitaly_ruby_json', + 'gitlab-shell/gitlab-shell', + 'nginx/gitlab_access', + 'nginx/gitlab_error', + 'nginx/gitlab_registry_access', + 'puma/puma_stderr', + 'puma/puma_stdout', + # The '*' is used in order to get the most recent GitLab + # reconfigure logs (name based on UNIX timestamp of when + # reconfigure initiated). Only the most recent file, by + # modification time, matching the wildcard is collected. + 'reconfigure/*.log' + ] + ] + [ + f'/mnt/gitlab/logs/gitlab-rails/{file}.log' + for file in + [ + 'api_json', + 'application_json', + 'application', + 'audit_json', + 'auth', + 'database_load_balancing', + 'exceptions_json', + 'graphql_json', + 'migrations', + 'production_json', + 'production', + 'sidekiq_client' + ] + ] + [ + f'/var/log/{file}' + for file in + [ + 'amazon/ssm/amazon-ssm-agent.log', + 'audit/audit.log', + 'cloud-init.log', + 'cron', + 'maillog', + 'messages', + 'secure' + ] + ] + ] + } + } + } + }, indent=4) + }, ], 'runcmd': [ ['systemctl', 'daemon-reload'], @@ -1542,6 +1634,13 @@ def qq(*words): 'gitlab-runner', 'clamscan.timer', 'prune-images.timer' + ], + [ + 'amazon-cloudwatch-agent-ctl', + '-a', 'fetch-config', + '-m', 'ec2', + '-s', + '-c', 'file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json' ] ], }, indent=2),