Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Send GitLab host logs to CloudWatch (#3894) #5187

Merged
merged 1 commit into from
May 30, 2023
Merged

Send GitLab host logs to CloudWatch (#3894) #5187

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions UPGRADING.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,16 @@ branch that does not have the listed changes, the steps would need to be
reverted. This is all fairly informal and loosely defined. Hopefully we won't
have too many entries in this file.

#3894 Send GitLab host logs to CloudWatch
=========================================

Operator
~~~~~~~~

Manually deploy the ``gitlab`` component of any main deployment just before
pushing the merge commit to the GitLab instance in that deployment.


#5207 Fix: Partition sizing ignores supplementary bundles
=========================================================

Expand Down
105 changes: 102 additions & 3 deletions terraform/gitlab/gitlab.tf.json.template.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -688,7 +688,15 @@ def qq(*words):
'config:BatchGetResourceConfig'
],
'resources': ['*']
}
},
{
'actions': [
'logs:CreateLogGroup',
'logs:CreateLogStream',
'logs:PutLogEvents'
],
'resources': ['arn:aws:logs:*:*:*']
},
]
}
},
Expand Down Expand Up @@ -900,6 +908,10 @@ def qq(*words):
'gitlab_vpc': {
'name': '/aws/vpc/azul-gitlab',
'retention_in_days': config.audit_log_retention_days,
},
'gitlab_cwagent': {
'name': '/aws/cwagent/azul-gitlab',
'retention_in_days': config.audit_log_retention_days,
}
},
'aws_flow_log': {
Expand Down Expand Up @@ -1305,7 +1317,7 @@ def qq(*words):
'mounts': [
['/dev/nvme1n1', '/mnt/gitlab', 'ext4', '']
],
'packages': ['docker'],
'packages': ['docker', 'amazon-cloudwatch-agent'],
'ssh_authorized_keys': other_public_keys.get(config.deployment_stage, []),
'write_files': [
{
Expand Down Expand Up @@ -1368,6 +1380,7 @@ def qq(*words):
'ExecStart=/usr/bin/docker',
'run',
'--name gitlab',
'--env GITLAB_SKIP_TAIL_LOGS=true',
'--hostname ${aws_route53_record.gitlab.name}',
'--publish 80:80',
'--publish 2222:22',
Expand Down Expand Up @@ -1527,7 +1540,86 @@ def qq(*words):
'[Install]',
'WantedBy=timers.target'
)
}
},
{
# AWS recommends placing the amazon-cloudwatch-agent config file at this path.
# Note that the parent of etc/ is where the agent is installed.
'path': '/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json',
'permissions': '0664',
'owner': 'root',
'content': json.dumps({
'agent': {
'region': aws.region_name,
'logfile': '/opt/aws/amazon-cloudwatch-agent/logs/amazon-cloudwatch-agent.log',
'debug': bool(config.debug)
},
'logs': {
'logs_collected': {
'files': {
'collect_list': [
{
'file_path': path,
'log_group_name': '${aws_cloudwatch_log_group.gitlab_cwagent.name}',
# https://docs.aws.amazon.com/AmazonCloudWatchLogs/latest/APIReference/API_CreateLogStream.html
# Characters disallowed for use in a log stream name are `:` and
# `*`, so we replace any occurrence of `*` in `path` with `?`.
'log_stream_name': path.replace('*', '?')
hannes-ucsc marked this conversation as resolved.
Show resolved Hide resolved
}
for path in
[
f'/var/log/{file}'
for file in
[
'amazon/ssm/amazon-ssm-agent.log',
'audit/audit.log',
'cloud-init.log',
'cron',
'maillog',
'messages',
'secure'
]

] + [
f'/mnt/gitlab/logs/{file}.log'
for file in
[
'gitaly/gitaly_ruby_json',
'gitlab-shell/gitlab-shell',
'nginx/gitlab_access',
'nginx/gitlab_error',
'nginx/gitlab_registry_access',
'puma/puma_stderr',
'puma/puma_stdout',
# The '*' is used in order to get the most recent GitLab
# reconfigure logs (name based on UNIX timestamp of when
# reconfigure initiated). Only the most recent file, by
# modification time, matching the wildcard is collected.
'reconfigure/*'
hannes-ucsc marked this conversation as resolved.
Show resolved Hide resolved
]
] + [
f'/mnt/gitlab/logs/gitlab-rails/{file}.log'
for file in
[
'api_json',
'application_json',
'application',
'audit_json',
'auth',
'database_load_balancing',
'exceptions_json',
'graphql_json',
'migrations',
'production_json',
'production',
'sidekiq_client'
]
]
]
}
}
}
}, indent=4)
},
],
'runcmd': [
['systemctl', 'daemon-reload'],
Expand All @@ -1542,6 +1634,13 @@ def qq(*words):
'gitlab-runner',
'clamscan.timer',
'prune-images.timer'
],
[
'amazon-cloudwatch-agent-ctl',
'-m', 'ec2', # mode
'-a', 'fetch-config', # action (fetch file from location specified in -c)
'-c', 'file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json',
'-s' # restart agent afterwards
]
],
}, indent=2),
Expand Down