diff --git a/docs/add-ons/promtail.md b/docs/add-ons/promtail.md new file mode 100644 index 0000000000..ff54972bca --- /dev/null +++ b/docs/add-ons/promtail.md @@ -0,0 +1,39 @@ +# Promtail + +Promtail is an agent which ships the contents of local logs to a Loki instance. + +[Promtail](https://github.com/grafana/helm-charts/tree/main/charts/promtail) chart bootstraps Promtail infrastructure on a Kubernetes cluster using the Helm package manager. + +For complete project documentation, please visit the [Promtail documentation site](https://grafana.com/docs/loki/latest/clients/promtail/). + +## Usage + +Promtail can be deployed by enabling the add-on via the following. + +```hcl +enable_promtail = true +``` + +Deploy Promtail with custom `values.yaml` + +```hcl + # Optional Map value; pass promtail-values.yaml from consumer module + promtail_helm_config = { + name = "promtail" # (Required) Release name. + repository = "https://grafana.github.io/helm-charts" # (Optional) Repository URL where to locate the requested chart. + chart = "promtail" # (Required) Chart name to be installed. + version = "6.3.0" # (Optional) Specify the exact chart version to install. If this is not specified, it defaults to the version set within default_helm_config: https://github.com/aws-ia/terraform-aws-eks-blueprints/blob/main/modules/kubernetes-addons/promtail/locals.tf + namespace = "promtail" # (Optional) The namespace to install the release into. + values = [templatefile("${path.module}/promtail-values.yaml", {})] + } +``` + +### GitOps Configuration + +The following properties are made available for use when managing the add-on via GitOps. + +```hcl +promtail = { + enable = true +} +``` diff --git a/examples/grafana-loki/main.tf b/examples/grafana-loki/main.tf new file mode 100644 index 0000000000..6ee47b035a --- /dev/null +++ b/examples/grafana-loki/main.tf @@ -0,0 +1,113 @@ +provider "aws" { + region = local.region +} + +provider "kubernetes" { + host = module.eks_blueprints.eks_cluster_endpoint + cluster_ca_certificate = base64decode(module.eks_blueprints.eks_cluster_certificate_authority_data) + token = data.aws_eks_cluster_auth.this.token +} + +provider "helm" { + kubernetes { + host = module.eks_blueprints.eks_cluster_endpoint + cluster_ca_certificate = base64decode(module.eks_blueprints.eks_cluster_certificate_authority_data) + token = data.aws_eks_cluster_auth.this.token + } +} + +data "aws_eks_cluster_auth" "this" { + name = module.eks_blueprints.eks_cluster_id +} + +data "aws_availability_zones" "available" {} + +locals { + name = basename(path.cwd) + region = "us-west-2" + + vpc_cidr = "10.0.0.0/16" + azs = slice(data.aws_availability_zones.available.names, 0, 3) + + tags = { + Blueprint = local.name + GithubRepo = "github.com/aws-ia/terraform-aws-eks-blueprints" + } +} + +#--------------------------------------------------------------- +# EKS Blueprints +#--------------------------------------------------------------- + +module "eks_blueprints" { + source = "../.." + + cluster_name = local.name + cluster_version = "1.23" + + vpc_id = module.vpc.vpc_id + private_subnet_ids = module.vpc.private_subnets + + managed_node_groups = { + velero = { + node_group_name = "velero" + launch_template_os = "amazonlinux2eks" + subnet_ids = module.vpc.private_subnets + } + } + + tags = local.tags +} + +module "eks_blueprints_kubernetes_addons" { + source = "../../modules/kubernetes-addons" + + eks_cluster_id = module.eks_blueprints.eks_cluster_id + eks_cluster_endpoint = module.eks_blueprints.eks_cluster_endpoint + eks_oidc_provider = module.eks_blueprints.oidc_provider + eks_cluster_version = module.eks_blueprints.eks_cluster_version + + enable_promtail = true + + tags = local.tags +} + +#--------------------------------------------------------------- +# Supporting Resources +#--------------------------------------------------------------- + +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "~> 3.0" + + name = local.name + cidr = local.vpc_cidr + + azs = local.azs + public_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k)] + private_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 10)] + + enable_nat_gateway = true + single_nat_gateway = true + enable_dns_hostnames = true + + # Manage so we can name + manage_default_network_acl = true + default_network_acl_tags = { Name = "${local.name}-default" } + manage_default_route_table = true + default_route_table_tags = { Name = "${local.name}-default" } + manage_default_security_group = true + default_security_group_tags = { Name = "${local.name}-default" } + + public_subnet_tags = { + "kubernetes.io/cluster/${local.name}" = "shared" + "kubernetes.io/role/elb" = 1 + } + + private_subnet_tags = { + "kubernetes.io/cluster/${local.name}" = "shared" + "kubernetes.io/role/internal-elb" = 1 + } + + tags = local.tags +} diff --git a/examples/grafana-loki/outputs.tf b/examples/grafana-loki/outputs.tf new file mode 100644 index 0000000000..55552d3138 --- /dev/null +++ b/examples/grafana-loki/outputs.tf @@ -0,0 +1,4 @@ +output "configure_kubectl" { + description = "Configure kubectl: make sure you're logged in with the correct AWS profile and run the following command to update your kubeconfig" + value = module.eks_blueprints.configure_kubectl +} diff --git a/examples/grafana-loki/variables.tf b/examples/grafana-loki/variables.tf new file mode 100644 index 0000000000..e69de29bb2 diff --git a/examples/grafana-loki/versions.tf b/examples/grafana-loki/versions.tf new file mode 100644 index 0000000000..0523ba5521 --- /dev/null +++ b/examples/grafana-loki/versions.tf @@ -0,0 +1,29 @@ +terraform { + required_version = ">= 1.0.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 3.72" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = ">= 2.10" + } + helm = { + source = "hashicorp/helm" + version = ">= 2.4.1" + } + random = { + source = "hashicorp/random" + version = ">= 3.0" + } + } + + # ## Used for end-to-end testing on project; update to suit your needs + # backend "s3" { + # bucket = "terraform-ssp-github-actions-state" + # region = "us-west-2" + # key = "e2e/grafana-loki/terraform.tfstate" + # } +} diff --git a/modules/kubernetes-addons/README.md b/modules/kubernetes-addons/README.md index bb12d6d718..b4a5fbd361 100644 --- a/modules/kubernetes-addons/README.md +++ b/modules/kubernetes-addons/README.md @@ -56,6 +56,7 @@ | [ondat](#module\_ondat) | ondat/ondat-addon/eksblueprints | 0.1.1 | | [opentelemetry\_operator](#module\_opentelemetry\_operator) | ./opentelemetry-operator | n/a | | [prometheus](#module\_prometheus) | ./prometheus | n/a | +| [promtail](#module\_promtail) | ./promtail | n/a | | [secrets\_store\_csi\_driver](#module\_secrets\_store\_csi\_driver) | ./secrets-store-csi-driver | n/a | | [spark\_history\_server](#module\_spark\_history\_server) | ./spark-history-server | n/a | | [spark\_k8s\_operator](#module\_spark\_k8s\_operator) | ./spark-k8s-operator | n/a | @@ -172,6 +173,7 @@ | [enable\_ondat](#input\_enable\_ondat) | Enable Ondat add-on | `bool` | `false` | no | | [enable\_opentelemetry\_operator](#input\_enable\_opentelemetry\_operator) | Enable opentelemetry operator add-on | `bool` | `false` | no | | [enable\_prometheus](#input\_enable\_prometheus) | Enable Community Prometheus add-on | `bool` | `false` | no | +| [enable\_promtail](#input\_enable\_promtail) | Enable Promtail add-on | `bool` | `false` | no | | [enable\_secrets\_store\_csi\_driver](#input\_enable\_secrets\_store\_csi\_driver) | Enable CSI Secrets Store Provider | `bool` | `false` | no | | [enable\_secrets\_store\_csi\_driver\_provider\_aws](#input\_enable\_secrets\_store\_csi\_driver\_provider\_aws) | Enable AWS CSI Secrets Store Provider | `bool` | `false` | no | | [enable\_self\_managed\_coredns](#input\_enable\_self\_managed\_coredns) | Enable self-managed CoreDNS add-on | `bool` | `false` | no | @@ -217,6 +219,7 @@ | [ondat\_irsa\_policies](#input\_ondat\_irsa\_policies) | IAM policy ARNs for Ondat IRSA | `list(string)` | `[]` | no | | [opentelemetry\_operator\_helm\_config](#input\_opentelemetry\_operator\_helm\_config) | Opentelemetry Operator Helm Chart config | `any` | `{}` | no | | [prometheus\_helm\_config](#input\_prometheus\_helm\_config) | Community Prometheus Helm Chart config | `any` | `{}` | no | +| [promtail\_helm\_config](#input\_promtail\_helm\_config) | Promtail Helm Chart config | `any` | `{}` | no | | [secrets\_store\_csi\_driver\_helm\_config](#input\_secrets\_store\_csi\_driver\_helm\_config) | CSI Secrets Store Provider Helm Configurations | `any` | `null` | no | | [self\_managed\_coredns\_helm\_config](#input\_self\_managed\_coredns\_helm\_config) | Self-managed CoreDNS Helm chart config | `any` | `{}` | no | | [spark\_history\_server\_helm\_config](#input\_spark\_history\_server\_helm\_config) | Spark History Server Helm Chart config | `any` | `{}` | no | diff --git a/modules/kubernetes-addons/locals.tf b/modules/kubernetes-addons/locals.tf index 086369bd4c..36f12355b7 100644 --- a/modules/kubernetes-addons/locals.tf +++ b/modules/kubernetes-addons/locals.tf @@ -33,6 +33,7 @@ locals { awsCloudWatchMetrics = var.enable_aws_cloudwatch_metrics ? module.aws_cloudwatch_metrics[0].argocd_gitops_config : null externalDns = var.enable_external_dns ? module.external_dns[0].argocd_gitops_config : null velero = var.enable_velero ? module.velero[0].argocd_gitops_config : null + promtail = var.enable_promtail ? module.promtail[0].argocd_gitops_config : null } addon_context = { diff --git a/modules/kubernetes-addons/main.tf b/modules/kubernetes-addons/main.tf index 26229de737..97ce37b017 100644 --- a/modules/kubernetes-addons/main.tf +++ b/modules/kubernetes-addons/main.tf @@ -498,3 +498,11 @@ module "external_secrets" { external_secrets_ssm_parameter_arns = var.external_secrets_ssm_parameter_arns external_secrets_secrets_manager_arns = var.external_secrets_secrets_manager_arns } + +module "promtail" { + count = var.enable_promtail ? 1 : 0 + source = "./promtail" + helm_config = var.promtail_helm_config + manage_via_gitops = var.argocd_manage_add_ons + addon_context = local.addon_context +} diff --git a/modules/kubernetes-addons/promtail/README.md b/modules/kubernetes-addons/promtail/README.md new file mode 100644 index 0000000000..47b94fd158 --- /dev/null +++ b/modules/kubernetes-addons/promtail/README.md @@ -0,0 +1,42 @@ +# Promtail Helm Chart +Promtail is an agent which ships the contents of local logs to a Loki instance + +For more details checkout [promtail](https://grafana.com/docs/loki/latest/clients/promtail/installation/) docs + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0.0 | +| [aws](#requirement\_aws) | >= 3.72 | +| [kubernetes](#requirement\_kubernetes) | >= 2.10 | + +## Providers + +No providers. + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [helm\_addon](#module\_helm\_addon) | ../helm-addon | n/a | + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [addon\_context](#input\_addon\_context) | Input configuration for the addon |
object({
aws_caller_identity_account_id = string
aws_caller_identity_arn = string
aws_eks_cluster_endpoint = string
aws_partition_id = string
aws_region_name = string
eks_cluster_id = string
eks_oidc_issuer_url = string
eks_oidc_provider_arn = string
tags = map(string)
irsa_iam_role_path = string
irsa_iam_permissions_boundary = string
})
| n/a | yes | +| [helm\_config](#input\_helm\_config) | Helm Config for promtail | `any` | `{}` | no | +| [manage\_via\_gitops](#input\_manage\_via\_gitops) | Determines if the add-on should be managed via GitOps. | `bool` | `false` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [argocd\_gitops\_config](#output\_argocd\_gitops\_config) | Configuration used for managing the add-on with ArgoCD | + diff --git a/modules/kubernetes-addons/promtail/locals.tf b/modules/kubernetes-addons/promtail/locals.tf new file mode 100644 index 0000000000..d70d2df278 --- /dev/null +++ b/modules/kubernetes-addons/promtail/locals.tf @@ -0,0 +1,22 @@ +locals { + name = "promtail" + default_helm_config = { + name = local.name + chart = local.name + repository = "https://grafana.github.io/helm-charts" + version = "6.3.0" + namespace = local.name + values = [] + create_namespace = true + description = "Promtail helm Chart deployment configuration" + } + + helm_config = merge( + local.default_helm_config, + var.helm_config + ) + + argocd_gitops_config = { + enable = true + } +} diff --git a/modules/kubernetes-addons/promtail/main.tf b/modules/kubernetes-addons/promtail/main.tf new file mode 100644 index 0000000000..d03500f57c --- /dev/null +++ b/modules/kubernetes-addons/promtail/main.tf @@ -0,0 +1,6 @@ +module "helm_addon" { + source = "../helm-addon" + helm_config = local.helm_config + manage_via_gitops = var.manage_via_gitops + addon_context = var.addon_context +} diff --git a/modules/kubernetes-addons/promtail/outputs.tf b/modules/kubernetes-addons/promtail/outputs.tf new file mode 100644 index 0000000000..b30c86b380 --- /dev/null +++ b/modules/kubernetes-addons/promtail/outputs.tf @@ -0,0 +1,4 @@ +output "argocd_gitops_config" { + description = "Configuration used for managing the add-on with ArgoCD" + value = var.manage_via_gitops ? local.argocd_gitops_config : null +} diff --git a/modules/kubernetes-addons/promtail/variables.tf b/modules/kubernetes-addons/promtail/variables.tf new file mode 100644 index 0000000000..1556b2392c --- /dev/null +++ b/modules/kubernetes-addons/promtail/variables.tf @@ -0,0 +1,28 @@ +variable "helm_config" { + description = "Helm Config for promtail" + type = any + default = {} +} + +variable "manage_via_gitops" { + description = "Determines if the add-on should be managed via GitOps." + type = bool + default = false +} + +variable "addon_context" { + description = "Input configuration for the addon" + type = object({ + aws_caller_identity_account_id = string + aws_caller_identity_arn = string + aws_eks_cluster_endpoint = string + aws_partition_id = string + aws_region_name = string + eks_cluster_id = string + eks_oidc_issuer_url = string + eks_oidc_provider_arn = string + tags = map(string) + irsa_iam_role_path = string + irsa_iam_permissions_boundary = string + }) +} diff --git a/modules/kubernetes-addons/promtail/versions.tf b/modules/kubernetes-addons/promtail/versions.tf new file mode 100644 index 0000000000..d2ddf87cc2 --- /dev/null +++ b/modules/kubernetes-addons/promtail/versions.tf @@ -0,0 +1,14 @@ +terraform { + required_version = ">= 1.0.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 3.72" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = ">= 2.10" + } + } +} diff --git a/modules/kubernetes-addons/variables.tf b/modules/kubernetes-addons/variables.tf index 87ddb4b739..477479453c 100644 --- a/modules/kubernetes-addons/variables.tf +++ b/modules/kubernetes-addons/variables.tf @@ -1026,3 +1026,16 @@ variable "airflow_helm_config" { type = any default = {} } + +#-----------Promtail ADDON------------- +variable "enable_promtail" { + description = "Enable Promtail add-on" + type = bool + default = false +} + +variable "promtail_helm_config" { + description = "Promtail Helm Chart config" + type = any + default = {} +}