Skip to content

Commit

Permalink
[u] Send GitLab host logs to CloudWatch (#3894)
Browse files Browse the repository at this point in the history
  • Loading branch information
@achave11-ucsc @hannes-ucsc
achave11-ucsc authored and hannes-ucsc committed May 27, 2023
1 parent 61f6e0d commit 6bedce5
Show file tree
Hide file tree
Showing 2 changed files with 112 additions and 3 deletions.
10 changes: 10 additions & 0 deletions UPGRADING.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,16 @@ branch that does not have the listed changes, the steps would need to be
reverted. This is all fairly informal and loosely defined. Hopefully we won't
have too many entries in this file.

#3894 Send GitLab host logs to CloudWatch
=========================================

Operator
~~~~~~~~

Manually deploy the ``gitlab`` component of any main deployment just before
pushing the merge commit to the GitLab instance in that deployment.


#5207 Fix: Partition sizing ignores supplementary bundles
=========================================================

Expand Down
105 changes: 102 additions & 3 deletions terraform/gitlab/gitlab.tf.json.template.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -688,7 +688,15 @@ def qq(*words):
'config:BatchGetResourceConfig'
],
'resources': ['*']
}
},
{
'actions': [
'logs:CreateLogGroup',
'logs:CreateLogStream',
'logs:PutLogEvents'
],
'resources': ['arn:aws:logs:*:*:*']
},
]
}
},
Expand Down Expand Up @@ -900,6 +908,10 @@ def qq(*words):
'gitlab_vpc': {
'name': '/aws/vpc/azul-gitlab',
'retention_in_days': config.audit_log_retention_days,
},
'gitlab_cwagent': {
'name': '/aws/cwagent/azul-gitlab',
'retention_in_days': config.audit_log_retention_days,
}
},
'aws_flow_log': {
Expand Down Expand Up @@ -1305,7 +1317,7 @@ def qq(*words):
'mounts': [
['/dev/nvme1n1', '/mnt/gitlab', 'ext4', '']
],
'packages': ['docker'],
'packages': ['docker', 'amazon-cloudwatch-agent'],
'ssh_authorized_keys': other_public_keys.get(config.deployment_stage, []),
'write_files': [
{
Expand Down Expand Up @@ -1368,6 +1380,7 @@ def qq(*words):
'ExecStart=/usr/bin/docker',
'run',
'--name gitlab',
'--env GITLAB_SKIP_TAIL_LOGS=true',
'--hostname ${aws_route53_record.gitlab.name}',
'--publish 80:80',
'--publish 2222:22',
Expand Down Expand Up @@ -1527,7 +1540,86 @@ def qq(*words):
'[Install]',
'WantedBy=timers.target'
)
}
},
{
# AWS recommends placing the amazon-cloudwatch-agent config file at this path.
# Note that the parent of etc/ is where the agent is installed.
'path': '/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json',
'permissions': '0664',
'owner': 'root',
'content': json.dumps({
'agent': {
'region': aws.region_name,
'logfile': '/opt/aws/amazon-cloudwatch-agent/logs/amazon-cloudwatch-agent.log',
'debug': bool(config.debug)
},
'logs': {
'logs_collected': {
'files': {
'collect_list': [
{
'file_path': path,
'log_group_name': '${aws_cloudwatch_log_group.gitlab_cwagent.name}',
# https://docs.aws.amazon.com/AmazonCloudWatchLogs/latest/APIReference/API_CreateLogStream.html
# Characters disallowed for use in a log stream name are `:` and
# `*`, so we replace any occurrence of `*` in `path` with `?`.
'log_stream_name': path.replace('*', '?')
}
for path in
[
f'/var/log/{file}'
for file in
[
'amazon/ssm/amazon-ssm-agent.log',
'audit/audit.log',
'cloud-init.log',
'cron',
'maillog',
'messages',
'secure'
]

] + [
f'/mnt/gitlab/logs/{file}.log'
for file in
[
'gitaly/gitaly_ruby_json',
'gitlab-shell/gitlab-shell',
'nginx/gitlab_access',
'nginx/gitlab_error',
'nginx/gitlab_registry_access',
'puma/puma_stderr',
'puma/puma_stdout',
# The '*' is used in order to get the most recent GitLab
# reconfigure logs (name based on UNIX timestamp of when
# reconfigure initiated). Only the most recent file, by
# modification time, matching the wildcard is collected.
'reconfigure/*'
]
] + [
f'/mnt/gitlab/logs/gitlab-rails/{file}.log'
for file in
[
'api_json',
'application_json',
'application',
'audit_json',
'auth',
'database_load_balancing',
'exceptions_json',
'graphql_json',
'migrations',
'production_json',
'production',
'sidekiq_client'
]
]
]
}
}
}
}, indent=4)
},
],
'runcmd': [
['systemctl', 'daemon-reload'],
Expand All @@ -1542,6 +1634,13 @@ def qq(*words):
'gitlab-runner',
'clamscan.timer',
'prune-images.timer'
],
[
'amazon-cloudwatch-agent-ctl',
'-m', 'ec2', # mode
'-a', 'fetch-config', # action (fetch file from location specified in -c)
'-c', 'file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json',
'-s' # restart agent afterwards
]
],
}, indent=2),
Expand Down

0 comments on commit 6bedce5

Please sign in to comment.