Skip to content

Commit

Permalink
[u] Send GitLab host logs to CloudWatch (#3894)
Browse files Browse the repository at this point in the history
  • Loading branch information
@achave11-ucsc
achave11-ucsc committed May 24, 2023
1 parent 8575b08 commit d5075dd
Show file tree
Hide file tree
Showing 2 changed files with 112 additions and 3 deletions.
10 changes: 10 additions & 0 deletions UPGRADING.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,16 @@ branch that does not have the listed changes, the steps would need to be
reverted. This is all fairly informal and loosely defined. Hopefully we won't
have too many entries in this file.

#3894 Send GitLab host logs to CloudWatch
=========================================

Operator
~~~~~~~~

Manually deploy the ``gitlab`` component of any main deployment just before
pushing the merge commit to the GitLab instance in that deployment.


#5207 Fix: Partition sizing ignores supplementary bundles
=========================================================

Expand Down
105 changes: 102 additions & 3 deletions terraform/gitlab/gitlab.tf.json.template.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -688,7 +688,15 @@ def qq(*words):
'config:BatchGetResourceConfig'
],
'resources': ['*']
}
},
{
'actions': [
'logs:CreateLogGroup',
'logs:CreateLogStream',
'logs:PutLogEvents'
],
'resources': ['arn:aws:logs:*:*:*']
},
]
}
},
Expand Down Expand Up @@ -900,6 +908,10 @@ def qq(*words):
'gitlab_vpc': {
'name': '/aws/vpc/azul-gitlab',
'retention_in_days': config.audit_log_retention_days,
},
'gitlab_cwagent': {
'name': '/aws/cwagent/azul-gitlab',
'retention_in_days': config.audit_log_retention_days,
}
},
'aws_flow_log': {
Expand Down Expand Up @@ -1305,7 +1317,7 @@ def qq(*words):
'mounts': [
['/dev/nvme1n1', '/mnt/gitlab', 'ext4', '']
],
'packages': ['docker'],
'packages': ['docker', 'amazon-cloudwatch-agent'],
'ssh_authorized_keys': other_public_keys.get(config.deployment_stage, []),
'write_files': [
{
Expand Down Expand Up @@ -1368,6 +1380,7 @@ def qq(*words):
'ExecStart=/usr/bin/docker',
'run',
'--name gitlab',
'--env GITLAB_SKIP_TAIL_LOGS=true',
'--hostname ${aws_route53_record.gitlab.name}',
'--publish 80:80',
'--publish 2222:22',
Expand Down Expand Up @@ -1527,7 +1540,86 @@ def qq(*words):
'[Install]',
'WantedBy=timers.target'
)
}
},
{
# AWS recommends placing the amazon-cloudwatch-agent config file at this path.
# Note that the parent of etc/ is where the agent is installed.
'path': '/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json',
'permissions': '0664',
'owner': 'root',
'content': json.dumps({
'agent': {
'region': aws.region_name,
'logfile': '/opt/aws/amazon-cloudwatch-agent/logs/amazon-cloudwatch-agent.log',
'debug': bool(config.debug)
},
'logs': {
'logs_collected': {
'files': {
'collect_list': [
{
'file_path': path,
'log_group_name': '/aws/cwagent/azul-gitlab',
'log_stream_name':
path
if not path.endswith('*.log')
else
path.replace('*', 'reconfigure')
}
for path in
[
f'/mnt/gitlab/logs/{file}.log'
for file in
[
'gitaly/gitaly_ruby_json',
'gitlab-shell/gitlab-shell',
'nginx/gitlab_access',
'nginx/gitlab_error',
'nginx/gitlab_registry_access',
'puma/puma_stderr',
'puma/puma_stdout',
# The '*' is used in order to get the most recent GitLab
# reconfigure logs (name based on UNIX timestamp of when
# reconfigure initiated). Only the most recent file, by
# modification time, matching the wildcard is collected.
'reconfigure/*.log'
]
] + [
f'/mnt/gitlab/logs/gitlab-rails/{file}.log'
for file in
[
'api_json',
'application_json',
'application',
'audit_json',
'auth',
'database_load_balancing',
'exceptions_json',
'graphql_json',
'migrations',
'production_json',
'production',
'sidekiq_client'
]
] + [
f'/var/log/{file}'
for file in
[
'amazon/ssm/amazon-ssm-agent.log',
'audit/audit.log',
'cloud-init.log',
'cron',
'maillog',
'messages',
'secure'
]
]
]
}
}
}
}, indent=4)
},
],
'runcmd': [
['systemctl', 'daemon-reload'],
Expand All @@ -1542,6 +1634,13 @@ def qq(*words):
'gitlab-runner',
'clamscan.timer',
'prune-images.timer'
],
[
'amazon-cloudwatch-agent-ctl',
'-a', 'fetch-config',
'-m', 'ec2',
'-s',
'-c', 'file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json'
]
],
}, indent=2),
Expand Down

0 comments on commit d5075dd

Please sign in to comment.