Skip to content

Commit

Permalink
Send GitLab host logs to CloudWatch (#3894)
Browse files Browse the repository at this point in the history
  • Loading branch information
@achave11-ucsc
achave11-ucsc committed May 11, 2023
1 parent 75d51cc commit 0d10bdf
Show file tree
Hide file tree
Showing 2 changed files with 121 additions and 3 deletions.
10 changes: 10 additions & 0 deletions UPGRADING.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,16 @@ branch that does not have the listed changes, the steps would need to be
reverted. This is all fairly informal and loosely defined. Hopefully we won't
have too many entries in this file.

#3894 Send GitLab host logs to CloudWatch
=========================================

Operator
~~~~~~~~

Manually deploy the ``gitlab`` component of any main deployment just before
pushing the merge commit to the GitLab instance in that deployment.


#5110 Update GitLab IAM policy for FedRAMP inventory
====================================================

Expand Down
114 changes: 111 additions & 3 deletions terraform/gitlab/gitlab.tf.json.template.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -708,7 +708,15 @@ def qq(*words):
'config:BatchGetResourceConfig'
],
'resources': ['*']
}
},
{
'actions': [
'logs:CreateLogGroup',
'logs:CreateLogStream',
'logs:PutLogEvents'
],
'resources': ['arn:aws:logs:*:*:*']
},
]
}
},
Expand Down Expand Up @@ -920,6 +928,10 @@ def qq(*words):
'gitlab_vpc': {
'name': '/aws/vpc/azul-gitlab',
'retention_in_days': config.audit_log_retention_days,
},
'gitlab_cwagent': {
'name': '/aws/cwagent/azul-gitlab',
'retention_in_days': config.audit_log_retention_days,
}
},
'aws_flow_log': {
Expand Down Expand Up @@ -1324,7 +1336,7 @@ def qq(*words):
'mounts': [
['/dev/nvme1n1', '/mnt/gitlab', 'ext4', '']
],
'packages': ['docker'],
'packages': ['docker', 'amazon-cloudwatch-agent'],
'ssh_authorized_keys': other_public_keys.get(config.deployment_stage, []),
'write_files': [
{
Expand Down Expand Up @@ -1387,6 +1399,7 @@ def qq(*words):
'ExecStart=/usr/bin/docker',
'run',
'--name gitlab',
'--env GITLAB_SKIP_TAIL_LOGS=true',
'--hostname ${aws_route53_record.gitlab.name}',
'--publish 80:80',
'--publish 2222:22',
Expand Down Expand Up @@ -1546,7 +1559,95 @@ def qq(*words):
'[Install]',
'WantedBy=timers.target'
)
}
},
{
# This is the AWS recommended placement of the agent's config file, and also
# the installation parent dir of the amazon-cloudwatch-agent package.
'path': '/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json',
'permissions': '0664',
'owner': 'root',
'content': json.dumps({
"agent": {
"metrics_collection_interval": config.audit_log_retention_days,
"region": aws.region_name,
"logfile": "/opt/aws/amazon-cloudwatch-agent/logs/amazon-cloudwatch-agent.log",
"debug": bool(config.debug)
},
"logs": {
"logs_collected": {
"files": {
"collect_list": [
{
# The '*' allows the timestamped file to go in a specific logstream,
# no other type of file exists in ./reconfigure. Only the latest
# file, by modification time, is processed by the CloudWatch agent.
"file_path": '/mnt/gitlab/logs/reconfigure/*.log',
"log_group_name": "/aws/cwagent/azul-gitlab",
"log_stream_name": '/mnt/gitlab/logs/reconfigure/dated.log'
},
*(
{
"file_path": f'/mnt/gitlab/logs/{path}.log',
"log_group_name": "/aws/cwagent/azul-gitlab",
"log_stream_name": f'/mnt/gitlab/logs/{path}.log'
}
for path in
[
'gitaly/gitaly_ruby_json',
'gitlab-shell/gitlab-shell',
'nginx/gitlab_access',
'nginx/gitlab_error',
'nginx/gitlab_registry_access',
'puma/puma_stderr',
'puma/puma_stdout'
]
),
*(
{
"file_path": f'/mnt/gitlab/logs/gitlab-rails/{path}.log',
"log_group_name": "/aws/cwagent/azul-gitlab",
"log_stream_name": f'/mnt/gitlab/logs/gitlab-rails/{path}.log'
}
for path in
[
'api_json',
'application_json',
'application',
'audit_json',
'auth',
'database_load_balancing',
'exceptions_json',
'gitlab-shell',
'graphql_json',
'migrations',
'production_json',
'production',
'sidekiq_client'
]
),
*(
{
"file_path": f'/var/log/{path}',
"log_group_name": "/aws/cwagent/azul-gitlab",
"log_stream_name": f'/var/log/{path}'
}
for path in
[
'amazon/ssm/amazon-ssm-agent.log',
'audit/audit.log',
'cloud-init.log',
'cron',
'maillog',
'messages',
'secure'
]
),
]
}
}
}
}, indent=4)
},
],
'runcmd': [
['systemctl', 'daemon-reload'],
Expand All @@ -1561,6 +1662,13 @@ def qq(*words):
'gitlab-runner',
'clamscan.timer',
'prune-images.timer'
],
[
'amazon-cloudwatch-agent-ctl',
'-a', 'fetch-config',
'-m', 'ec2',
'-s',
'-c', 'file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json'
]
],
}, indent=2),
Expand Down

0 comments on commit 0d10bdf

Please sign in to comment.