diff --git a/UPGRADING.rst b/UPGRADING.rst index f04a1d4387..246290703c 100644 --- a/UPGRADING.rst +++ b/UPGRADING.rst @@ -19,6 +19,16 @@ branch that does not have the listed changes, the steps would need to be reverted. This is all fairly informal and loosely defined. Hopefully we won't have too many entries in this file. +#3894 Send GitLab host logs to CloudWatch +========================================= + +Operator +~~~~~~~~ + +Manually deploy the ``gitlab`` component of any main deployment just before +pushing the merge commit to the GitLab instance in that deployment. + + #5110 Update GitLab IAM policy for FedRAMP inventory ==================================================== diff --git a/terraform/gitlab/gitlab.tf.json.template.py b/terraform/gitlab/gitlab.tf.json.template.py index 16e07e240d..fd56b5f628 100644 --- a/terraform/gitlab/gitlab.tf.json.template.py +++ b/terraform/gitlab/gitlab.tf.json.template.py @@ -708,7 +708,15 @@ def qq(*words): 'config:BatchGetResourceConfig' ], 'resources': ['*'] - } + }, + { + 'actions': [ + 'logs:CreateLogGroup', + 'logs:CreateLogStream', + 'logs:PutLogEvents' + ], + 'resources': ['arn:aws:logs:*:*:*'] + }, ] } }, @@ -920,6 +928,10 @@ def qq(*words): 'gitlab_vpc': { 'name': '/aws/vpc/azul-gitlab', 'retention_in_days': config.audit_log_retention_days, + }, + 'gitlab_cwagent': { + 'name': '/aws/cwagent/azul-gitlab', + 'retention_in_days': config.audit_log_retention_days, } }, 'aws_flow_log': { @@ -1324,7 +1336,7 @@ def qq(*words): 'mounts': [ ['/dev/nvme1n1', '/mnt/gitlab', 'ext4', ''] ], - 'packages': ['docker'], + 'packages': ['docker', 'amazon-cloudwatch-agent'], 'ssh_authorized_keys': other_public_keys.get(config.deployment_stage, []), 'write_files': [ { @@ -1387,6 +1399,7 @@ def qq(*words): 'ExecStart=/usr/bin/docker', 'run', '--name gitlab', + '--env GITLAB_SKIP_TAIL_LOGS=true', '--hostname ${aws_route53_record.gitlab.name}', '--publish 80:80', '--publish 2222:22', @@ -1546,7 +1559,95 @@ def qq(*words): '[Install]', 'WantedBy=timers.target' ) - } + }, + { + # This is the AWS recommended placement of the agent's config file, and also + # the installation parent dir of the amazon-cloudwatch-agent package. + 'path': '/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json', + 'permissions': '0664', + 'owner': 'root', + 'content': json.dumps({ + "agent": { + "metrics_collection_interval": config.audit_log_retention_days, + "region": aws.region_name, + "logfile": "/opt/aws/amazon-cloudwatch-agent/logs/amazon-cloudwatch-agent.log", + "debug": bool(config.debug) + }, + "logs": { + "logs_collected": { + "files": { + "collect_list": [ + { + # The '*' allows the timestamped file to go in a specific logstream, + # no other type of file exists in ./reconfigure. Only the latest + # file, by modification time, is processed by the CloudWatch agent. + "file_path": '/mnt/gitlab/logs/reconfigure/*.log', + "log_group_name": "/aws/cwagent/azul-gitlab", + "log_stream_name": '/mnt/gitlab/logs/reconfigure/dated.log' + }, + *( + { + "file_path": f'/mnt/gitlab/logs/{path}.log', + "log_group_name": "/aws/cwagent/azul-gitlab", + "log_stream_name": f'/mnt/gitlab/logs/{path}.log' + } + for path in + [ + 'gitaly/gitaly_ruby_json', + 'gitlab-shell/gitlab-shell', + 'nginx/gitlab_access', + 'nginx/gitlab_error', + 'nginx/gitlab_registry_access', + 'puma/puma_stderr', + 'puma/puma_stdout' + ] + ), + *( + { + "file_path": f'/mnt/gitlab/logs/gitlab-rails/{path}.log', + "log_group_name": "/aws/cwagent/azul-gitlab", + "log_stream_name": f'/mnt/gitlab/logs/gitlab-rails/{path}.log' + } + for path in + [ + 'api_json', + 'application_json', + 'application', + 'audit_json', + 'auth', + 'database_load_balancing', + 'exceptions_json', + 'gitlab-shell', + 'graphql_json', + 'migrations', + 'production_json', + 'production', + 'sidekiq_client' + ] + ), + *( + { + "file_path": f'/var/log/{path}', + "log_group_name": "/aws/cwagent/azul-gitlab", + "log_stream_name": f'/var/log/{path}' + } + for path in + [ + 'amazon/ssm/amazon-ssm-agent.log', + 'audit/audit.log', + 'cloud-init.log', + 'cron', + 'maillog', + 'messages', + 'secure' + ] + ), + ] + } + } + } + }, indent=4) + }, ], 'runcmd': [ ['systemctl', 'daemon-reload'], @@ -1561,6 +1662,13 @@ def qq(*words): 'gitlab-runner', 'clamscan.timer', 'prune-images.timer' + ], + [ + 'amazon-cloudwatch-agent-ctl', + '-a', 'fetch-config', + '-m', 'ec2', + '-s', + '-c', 'file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json' ] ], }, indent=2),