Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

🔥 Retire OpenMetadata and OpenSearch #6985

Merged
merged 1 commit into from
Jul 8, 2024
Merged

🔥 Retire OpenMetadata and OpenSearch #6985

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
120 changes: 60 additions & 60 deletions terraform/environments/data-platform-apps-and-tools/helm-charts-applications.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,66 +11,66 @@ resource "helm_release" "static_assets" {
depends_on = [helm_release.cert_manager_additional]
}

resource "helm_release" "openmetadata_dependencies" {
name = "openmetadata-dependencies"
repository = "https://helm.open-metadata.org"
chart = "openmetadata-dependencies"
version = "1.2.1"
namespace = kubernetes_namespace.openmetadata.metadata[0].name
values = [
templatefile(
"${path.module}/src/helm/openmetadata-dependencies/values.yml.tftpl",
{
openmetadata_airflow_password = random_password.openmetadata_airflow.result
openmetadata_airflow_eks_role_arn = module.openmetadata_airflow_iam_role.iam_role_arn
openmetadata_airflow_rds_host = module.openmetadata_airflow_rds.db_instance_address
openmetadata_airflow_rds_user = module.openmetadata_airflow_rds.db_instance_username
openmetadata_airflow_rds_db = module.openmetadata_airflow_rds.db_instance_name
openmetadata_airflow_rds_password_secret = kubernetes_secret.openmetadata_airflow_rds_credentials.metadata[0].name
openmetadata_airflow_rds_password_secret_key = "password"
openmetadata_airflow_admin_email = "${local.environment_configuration.airflow_mail_from_address}@${local.environment_configuration.ses_domain_identity}"
}
)
]
wait = true
timeout = 600
# resource "helm_release" "openmetadata_dependencies" {
# name = "openmetadata-dependencies"
# repository = "https://helm.open-metadata.org"
# chart = "openmetadata-dependencies"
# version = "1.2.1"
# namespace = kubernetes_namespace.openmetadata.metadata[0].name
# values = [
# templatefile(
# "${path.module}/src/helm/openmetadata-dependencies/values.yml.tftpl",
# {
# openmetadata_airflow_password = random_password.openmetadata_airflow.result
# openmetadata_airflow_eks_role_arn = module.openmetadata_airflow_iam_role.iam_role_arn
# openmetadata_airflow_rds_host = module.openmetadata_airflow_rds.db_instance_address
# openmetadata_airflow_rds_user = module.openmetadata_airflow_rds.db_instance_username
# openmetadata_airflow_rds_db = module.openmetadata_airflow_rds.db_instance_name
# openmetadata_airflow_rds_password_secret = kubernetes_secret.openmetadata_airflow_rds_credentials.metadata[0].name
# openmetadata_airflow_rds_password_secret_key = "password"
# openmetadata_airflow_admin_email = "${local.environment_configuration.airflow_mail_from_address}@${local.environment_configuration.ses_domain_identity}"
# }
# )
# ]
# wait = true
# timeout = 600

depends_on = [kubernetes_secret.openmetadata_airflow]
}
# depends_on = [kubernetes_secret.openmetadata_airflow]
# }

resource "helm_release" "openmetadata" {
name = "openmetadata"
repository = "https://helm.open-metadata.org"
chart = "openmetadata"
version = "1.2.1"
namespace = kubernetes_namespace.openmetadata.metadata[0].name
values = [
templatefile(
"${path.module}/src/helm/openmetadata/values.yml.tftpl",
{
host = "catalogue.${local.environment_configuration.route53_zone}"
eks_role_arn = module.openmetadata_iam_role.iam_role_arn
client_id = data.aws_secretsmanager_secret_version.openmetadata_entra_id_client_id.secret_string
tenant_id = data.aws_secretsmanager_secret_version.openmetadata_entra_id_tenant_id.secret_string
jwt_key_id = random_uuid.openmetadata_jwt.result
openmetadata_airflow_username = "${local.environment_configuration.airflow_mail_from_address}@${local.environment_configuration.ses_domain_identity}"
openmetadata_airflow_password_secret = kubernetes_secret.openmetadata_airflow.metadata[0].name
#checkov:skip=CKV_SECRET_6:Reference to Kubernetes secret not a sensitive value
openmetadata_airflow_password_secret_key = "openmetadata-airflow-password"
openmetadata_opensearch_host = resource.aws_opensearch_domain.openmetadata.endpoint
openmetadata_opensearch_user = "openmetadata"
openmetadata_opensearch_password_secret = kubernetes_secret.openmetadata_opensearch_credentials.metadata[0].name
openmetadata_opensearch_password_secret_key = "password"
openmetadata_rds_host = module.openmetadata_rds.db_instance_address
openmetadata_rds_user = module.openmetadata_rds.db_instance_username
openmetadata_rds_dbname = module.openmetadata_rds.db_instance_name
openmetadata_rds_password_secret = kubernetes_secret.openmetadata_rds_credentials.metadata[0].name
openmetadata_rds_password_secret_key = "password"
}
)
]
wait = true
timeout = 600
# resource "helm_release" "openmetadata" {
# name = "openmetadata"
# repository = "https://helm.open-metadata.org"
# chart = "openmetadata"
# version = "1.2.1"
# namespace = kubernetes_namespace.openmetadata.metadata[0].name
# values = [
# templatefile(
# "${path.module}/src/helm/openmetadata/values.yml.tftpl",
# {
# host = "catalogue.${local.environment_configuration.route53_zone}"
# eks_role_arn = module.openmetadata_iam_role.iam_role_arn
# client_id = data.aws_secretsmanager_secret_version.openmetadata_entra_id_client_id.secret_string
# tenant_id = data.aws_secretsmanager_secret_version.openmetadata_entra_id_tenant_id.secret_string
# jwt_key_id = random_uuid.openmetadata_jwt.result
# openmetadata_airflow_username = "${local.environment_configuration.airflow_mail_from_address}@${local.environment_configuration.ses_domain_identity}"
# openmetadata_airflow_password_secret = kubernetes_secret.openmetadata_airflow.metadata[0].name
# #checkov:skip=CKV_SECRET_6:Reference to Kubernetes secret not a sensitive value
# openmetadata_airflow_password_secret_key = "openmetadata-airflow-password"
# openmetadata_opensearch_host = resource.aws_opensearch_domain.openmetadata.endpoint
# openmetadata_opensearch_user = "openmetadata"
# openmetadata_opensearch_password_secret = kubernetes_secret.openmetadata_opensearch_credentials.metadata[0].name
# openmetadata_opensearch_password_secret_key = "password"
# openmetadata_rds_host = module.openmetadata_rds.db_instance_address
# openmetadata_rds_user = module.openmetadata_rds.db_instance_username
# openmetadata_rds_dbname = module.openmetadata_rds.db_instance_name
# openmetadata_rds_password_secret = kubernetes_secret.openmetadata_rds_credentials.metadata[0].name
# openmetadata_rds_password_secret_key = "password"
# }
# )
# ]
# wait = true
# timeout = 600

depends_on = [helm_release.openmetadata_dependencies]
}
# depends_on = [helm_release.openmetadata_dependencies]
# }
154 changes: 77 additions & 77 deletions terraform/environments/data-platform-apps-and-tools/opensearch.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,90 +1,90 @@
resource "aws_opensearch_domain" "openmetadata" {
domain_name = "openmetadata"
engine_version = "OpenSearch_2.7"
# resource "aws_opensearch_domain" "openmetadata" {
# domain_name = "openmetadata"
# engine_version = "OpenSearch_2.7"

vpc_options {
subnet_ids = module.vpc.private_subnets
security_group_ids = [module.opensearch_security_group.security_group_id]
}
# vpc_options {
# subnet_ids = module.vpc.private_subnets
# security_group_ids = [module.opensearch_security_group.security_group_id]
# }

cluster_config {
dedicated_master_enabled = true
dedicated_master_count = 3
dedicated_master_type = "m6g.large.search"
instance_count = 6
instance_type = "r6g.large.search"
zone_awareness_enabled = true
zone_awareness_config {
availability_zone_count = 3
}
}
# cluster_config {
# dedicated_master_enabled = true
# dedicated_master_count = 3
# dedicated_master_type = "m6g.large.search"
# instance_count = 6
# instance_type = "r6g.large.search"
# zone_awareness_enabled = true
# zone_awareness_config {
# availability_zone_count = 3
# }
# }

log_publishing_options {
enabled = true
cloudwatch_log_group_arn = module.openmetadata_opensearch_cloudwatch_log_group.cloudwatch_log_group_arn
log_type = "AUDIT_LOGS"
}
# log_publishing_options {
# enabled = true
# cloudwatch_log_group_arn = module.openmetadata_opensearch_cloudwatch_log_group.cloudwatch_log_group_arn
# log_type = "AUDIT_LOGS"
# }

log_publishing_options {
cloudwatch_log_group_arn = module.openmetadata_opensearch_cloudwatch_log_group.cloudwatch_log_group_arn
log_type = "ES_APPLICATION_LOGS"
}
# log_publishing_options {
# cloudwatch_log_group_arn = module.openmetadata_opensearch_cloudwatch_log_group.cloudwatch_log_group_arn
# log_type = "ES_APPLICATION_LOGS"
# }

log_publishing_options {
cloudwatch_log_group_arn = module.openmetadata_opensearch_cloudwatch_log_group.cloudwatch_log_group_arn
log_type = "SEARCH_SLOW_LOGS"
}
# log_publishing_options {
# cloudwatch_log_group_arn = module.openmetadata_opensearch_cloudwatch_log_group.cloudwatch_log_group_arn
# log_type = "SEARCH_SLOW_LOGS"
# }

log_publishing_options {
cloudwatch_log_group_arn = module.openmetadata_opensearch_cloudwatch_log_group.cloudwatch_log_group_arn
log_type = "INDEX_SLOW_LOGS"
}
# log_publishing_options {
# cloudwatch_log_group_arn = module.openmetadata_opensearch_cloudwatch_log_group.cloudwatch_log_group_arn
# log_type = "INDEX_SLOW_LOGS"
# }

encrypt_at_rest {
enabled = true
kms_key_id = module.openmetadata_opensearch_kms.key_id
}
# encrypt_at_rest {
# enabled = true
# kms_key_id = module.openmetadata_opensearch_kms.key_id
# }

domain_endpoint_options {
enforce_https = true
tls_security_policy = "Policy-Min-TLS-1-2-2019-07"
}
# domain_endpoint_options {
# enforce_https = true
# tls_security_policy = "Policy-Min-TLS-1-2-2019-07"
# }

advanced_security_options {
enabled = true
anonymous_auth_enabled = false
internal_user_database_enabled = true
master_user_options {
master_user_name = "openmetadata"
master_user_password = random_password.opensearch.result
}
}
# advanced_security_options {
# enabled = true
# anonymous_auth_enabled = false
# internal_user_database_enabled = true
# master_user_options {
# master_user_name = "openmetadata"
# master_user_password = random_password.opensearch.result
# }
# }

node_to_node_encryption {
enabled = true
}
# node_to_node_encryption {
# enabled = true
# }

ebs_options {
ebs_enabled = true
volume_size = 100
}
}
# ebs_options {
# ebs_enabled = true
# volume_size = 100
# }
# }

data "aws_iam_policy_document" "opensearch_domain" {
// TODO: Find source for this policy @jacobwoffenden
#checkov:skip=CKV_AWS_283:
statement {
effect = "Allow"
actions = ["es:ESHttp*"]
principals {
type = "AWS"
identifiers = ["*"]
}
resources = ["${aws_opensearch_domain.openmetadata.arn}/*"]
}
}
# data "aws_iam_policy_document" "opensearch_domain" {
# // TODO: Find source for this policy @jacobwoffenden
# #checkov:skip=CKV_AWS_283:
# statement {
# effect = "Allow"
# actions = ["es:ESHttp*"]
# principals {
# type = "AWS"
# identifiers = ["*"]
# }
# resources = ["${aws_opensearch_domain.openmetadata.arn}/*"]
# }
# }

resource "aws_opensearch_domain_policy" "openmetadata" {
domain_name = aws_opensearch_domain.openmetadata.domain_name
access_policies = data.aws_iam_policy_document.opensearch_domain.json
}
# resource "aws_opensearch_domain_policy" "openmetadata" {
# domain_name = aws_opensearch_domain.openmetadata.domain_name
# access_policies = data.aws_iam_policy_document.opensearch_domain.json
# }