From a4a86b6e0d5729f2c42237ef6930fe1da43d20e7 Mon Sep 17 00:00:00 2001 From: Felix Date: Wed, 14 Sep 2022 12:31:06 +0200 Subject: [PATCH 01/21] Initial commit --- .gitattributes | 2 + LICENSE | 201 +++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 2 + 3 files changed, 205 insertions(+) create mode 100644 .gitattributes create mode 100644 LICENSE create mode 100644 README.md diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..dfe077042 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# Auto detect text files and perform LF normalization +* text=auto diff --git a/LICENSE b/LICENSE new file mode 100644 index 000000000..b09cd7856 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 000000000..755224762 --- /dev/null +++ b/README.md @@ -0,0 +1,2 @@ +# elastic-terraform-examples + From 3f8b96f0886e2d2232ac4c7a9fdf7c0b511eb9a9 Mon Sep 17 00:00:00 2001 From: Felix Date: Wed, 14 Sep 2022 12:37:06 +0200 Subject: [PATCH 02/21] Initial commit --- .gitignore | 7 + json_templates/default-policy.json | 1 + json_templates/es_api_key.json | 3 + json_templates/gcp_integration.json | 400 ++++++++++++++++++++++++ scripts/es_api_key.sh | 10 + scripts/es_create_ilm_policy.sh | 12 + scripts/es_create_mapping.sh | 16 + scripts/kb_add_integration_to_policy.sh | 16 + scripts/kb_create_agent_policy.sh | 21 ++ scripts/kb_upload_saved_objects.sh | 15 + terraform/ec_gc_deployment.tf | 78 +++++ terraform/gc_dataflow.tf | 20 ++ terraform/gc_elastic_agent.tf | 80 +++++ terraform/gc_logging.tf | 164 ++++++++++ terraform/gc_provider.tf | 6 + terraform/main.tf | 149 +++++++++ 16 files changed, 998 insertions(+) create mode 100644 .gitignore create mode 100644 json_templates/default-policy.json create mode 100644 json_templates/es_api_key.json create mode 100644 json_templates/gcp_integration.json create mode 100644 scripts/es_api_key.sh create mode 100644 scripts/es_create_ilm_policy.sh create mode 100644 scripts/es_create_mapping.sh create mode 100644 scripts/kb_add_integration_to_policy.sh create mode 100644 scripts/kb_create_agent_policy.sh create mode 100644 scripts/kb_upload_saved_objects.sh create mode 100644 terraform/ec_gc_deployment.tf create mode 100644 terraform/gc_dataflow.tf create mode 100644 terraform/gc_elastic_agent.tf create mode 100644 terraform/gc_logging.tf create mode 100644 terraform/gc_provider.tf create mode 100644 terraform/main.tf diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..f71942805 --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ + +.vs/slnx.sqlite +*.tfstate +*.backup +terraform/.terraform +*.hcl +local_env \ No newline at end of file diff --git a/json_templates/default-policy.json b/json_templates/default-policy.json new file mode 100644 index 000000000..bab414ae3 --- /dev/null +++ b/json_templates/default-policy.json @@ -0,0 +1 @@ +{ "name": "${policy_name}", "description": "Terraformed policy", "namespace": "default", "monitoring_enabled": [ "logs", "metrics" ]} \ No newline at end of file diff --git a/json_templates/es_api_key.json b/json_templates/es_api_key.json new file mode 100644 index 000000000..dccd71b19 --- /dev/null +++ b/json_templates/es_api_key.json @@ -0,0 +1,3 @@ +{ + "name": "${elastic-api-key-name}" +} diff --git a/json_templates/gcp_integration.json b/json_templates/gcp_integration.json new file mode 100644 index 000000000..dafe15b84 --- /dev/null +++ b/json_templates/gcp_integration.json @@ -0,0 +1,400 @@ +{ + "name": "gcp", + "description": "Terraformed GCP integration", + "namespace": "default", + "policy_id": "${policy_id}", + "enabled": true, + "output_id": "", + "inputs": [ + { + "type": "gcp-pubsub", + "policy_template": "audit", + "enabled": %{ if audit_log_topic != ""}true%{ else }false%{ endif }, + "streams": [ + { + "enabled": true, + "data_stream": { + "type": "logs", + "dataset": "gcp.audit" + }, + "vars": { + "topic": { + "value": "${audit_log_topic}", + "type": "text" + }, + "subscription_name": { + "value": "${audit_log_topic}", + "type": "text" + }, + "subscription_create": { + "value": true, + "type": "bool" + }, + "alternative_host": { + "type": "text" + }, + "tags": { + "value": [ + "forwarded", + "gcp-audit" + ], + "type": "text" + }, + "preserve_original_event": { + "value": false, + "type": "bool" + }, + "processors": { + "type": "yaml" + } + } + } + ] + }, + { + "type": "gcp-pubsub", + "policy_template": "firewall", + "enabled": %{ if firewall_log_topic != ""}true%{ else }false%{ endif }, + "streams": [ + { + "enabled": true, + "data_stream": { + "type": "logs", + "dataset": "gcp.firewall" + }, + "vars": { + "topic": { + "value": "${firewall_log_topic}", + "type": "text" + }, + "subscription_name": { + "value": "${firewall_log_topic}", + "type": "text" + }, + "subscription_create": { + "value": true, + "type": "bool" + }, + "alternative_host": { + "type": "text" + }, + "tags": { + "value": [ + "forwarded", + "gcp-firewall" + ], + "type": "text" + }, + "preserve_original_event": { + "value": false, + "type": "bool" + }, + "processors": { + "type": "yaml" + } + } + } + ] + }, + { + "type": "gcp-pubsub", + "policy_template": "vpcflow", + "enabled": %{ if vpcflow_log_topic != ""}true%{ else }false%{ endif }, + "streams": [ + { + "enabled": true, + "data_stream": { + "type": "logs", + "dataset": "gcp.vpcflow" + }, + "vars": { + "topic": { + "value": "${vpcflow_log_topic}", + "type": "text" + }, + "subscription_name": { + "value": "${vpcflow_log_topic}", + "type": "text" + }, + "subscription_create": { + "value": true, + "type": "bool" + }, + "alternative_host": { + "type": "text" + }, + "tags": { + "value": [ + "forwarded", + "gcp-vpcflow" + ], + "type": "text" + }, + "preserve_original_event": { + "value": false, + "type": "bool" + }, + "processors": { + "type": "yaml" + } + } + } + ] + }, + { + "type": "gcp-pubsub", + "policy_template": "dns", + "enabled": %{ if dns_log_topic != ""}true%{ else }false%{ endif }, + "streams": [ + { + "enabled": true, + "data_stream": { + "type": "logs", + "dataset": "gcp.dns" + }, + "vars": { + "topic": { + "value": "${dns_log_topic}", + "type": "text" + }, + "subscription_name": { + "value": "${dns_log_topic}", + "type": "text" + }, + "subscription_create": { + "value": true, + "type": "bool" + }, + "alternative_host": { + "type": "text" + }, + "tags": { + "value": [ + "forwarded", + "gcp-dns" + ], + "type": "text" + }, + "preserve_original_event": { + "value": false, + "type": "bool" + }, + "processors": { + "type": "yaml" + } + } + } + ] + }, + { + "type": "gcp/metrics", + "policy_template": "billing", + "enabled": false, + "streams": [ + { + "enabled": false, + "data_stream": { + "type": "metrics", + "dataset": "gcp.billing" + }, + "vars": { + "period": { + "value": "24h", + "type": "text" + }, + "dataset_id": { + "type": "text" + }, + "table_pattern": { + "value": "gcp_billing_export_v1", + "type": "text" + }, + "cost_type": { + "value": "regular", + "type": "text" + } + } + } + ] + }, + { + "type": "gcp/metrics", + "policy_template": "compute", + "enabled": true, + "streams": [ + { + "enabled": true, + "data_stream": { + "type": "metrics", + "dataset": "gcp.compute" + }, + "vars": { + "zone": { + "type": "text" + }, + "region": { + "type": "text" + }, + "period": { + "value": "60s", + "type": "text" + }, + "exclude_labels": { + "type": "bool" + } + } + } + ] + }, + { + "type": "gcp/metrics", + "policy_template": "firestore", + "enabled": true, + "streams": [ + { + "enabled": true, + "data_stream": { + "type": "metrics", + "dataset": "gcp.firestore" + }, + "vars": { + "zone": { + "type": "text" + }, + "region": { + "type": "text" + }, + "period": { + "value": "60s", + "type": "text" + }, + "exclude_labels": { + "type": "bool" + } + } + } + ] + }, + { + "type": "gcp/metrics", + "policy_template": "loadbalancing", + "enabled": true, + "streams": [ + { + "enabled": true, + "data_stream": { + "type": "metrics", + "dataset": "gcp.loadbalancing_metrics" + }, + "vars": { + "zone": { + "type": "text" + }, + "region": { + "type": "text" + }, + "period": { + "value": "60s", + "type": "text" + }, + "exclude_labels": { + "type": "bool" + } + } + } + ] + }, + { + "type": "gcp-pubsub", + "policy_template": "loadbalancing", + "enabled": %{ if lb_log_topic != ""}true%{ else }false%{ endif }, + "streams": [ + { + "enabled": true, + "data_stream": { + "type": "logs", + "dataset": "gcp.loadbalancing_logs" + }, + "vars": { + "topic": { + "value": "${lb_log_topic}", + "type": "text" + }, + "subscription_name": { + "value": "${lb_log_topic}", + "type": "text" + }, + "subscription_create": { + "value": true, + "type": "bool" + }, + "alternative_host": { + "type": "text" + }, + "tags": { + "value": [ + "forwarded", + "gcp-loadbalancing_logs" + ], + "type": "text" + }, + "preserve_original_event": { + "value": false, + "type": "bool" + }, + "processors": { + "type": "yaml" + } + } + } + ] + }, + { + "type": "gcp/metrics", + "policy_template": "storage", + "enabled": true, + "streams": [ + { + "enabled": true, + "data_stream": { + "type": "metrics", + "dataset": "gcp.storage" + }, + "vars": { + "zone": { + "type": "text" + }, + "region": { + "type": "text" + }, + "period": { + "value": "60s", + "type": "text" + }, + "exclude_labels": { + "type": "bool" + } + } + } + ] + } + ], + "package": { + "name": "gcp", + "title": "Google Cloud Platform", + "version": "2.7.0" + }, + "vars": { + "project_id": { + "value": "${gcp_project}", + "type": "text" + }, + "credentials_file": { + "type": "text" + }, + "credentials_json": { + "value": ${gcp_credentials_json}, + "type": "text" + } + } +} \ No newline at end of file diff --git a/scripts/es_api_key.sh b/scripts/es_api_key.sh new file mode 100644 index 000000000..d4ac79962 --- /dev/null +++ b/scripts/es_api_key.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +eval "$(jq -r '@sh "ELASTIC_ENDPOINT=\(.elastic_endpoint) ELASTIC_USERNAME=\(.elastic_username) ELASTIC_PASSWORD=\(.elastic_password) API_KEY_BODY=\(.api_key_body)"')" + +output=$(curl -s -X POST -u "$ELASTIC_USERNAME:$ELASTIC_PASSWORD" \ + -H 'Content-Type:application/json' -d "$API_KEY_BODY" \ + ${ELASTIC_ENDPOINT}/_security/api_key | jq '.') + +ENCODED=$( echo $output | jq -r '.encoded' ) +jq -n --arg encoded "$ENCODED" '{"encoded" : $encoded}' diff --git a/scripts/es_create_ilm_policy.sh b/scripts/es_create_ilm_policy.sh new file mode 100644 index 000000000..5ac8fda54 --- /dev/null +++ b/scripts/es_create_ilm_policy.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +eval "$(jq -r '@sh "ELASTIC_HTTP_METHOD=\(.elastic_http_method) ELASTIC_ENDPOINT=\(.elastic_endpoint) ELASTIC_USERNAME=\(.elastic_username) ELASTIC_PASSWORD=\(.elastic_password) ELASTIC_JSON_BODY=\(.elastic_json_body)"')" + +# Define mapping +output=$(curl -s -X ${ELASTIC_HTTP_METHOD} -u "$ELASTIC_USERNAME:$ELASTIC_PASSWORD" \ + -H 'Content-Type:application/json' -d "$ELASTIC_JSON_BODY" \ + ${ELASTIC_ENDPOINT}/_ilm/policy/patent_search_rollover_policy | jq '.') + +# Return response +ACKNOWLEDGED=$( echo $output | jq -r '.acknowledged' ) +jq -n --arg acknowledged "$ACKNOWLEDGED" '{"acknowledged" : $acknowledged}' diff --git a/scripts/es_create_mapping.sh b/scripts/es_create_mapping.sh new file mode 100644 index 000000000..b6ffd3432 --- /dev/null +++ b/scripts/es_create_mapping.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +eval "$(jq -r '@sh "ELASTIC_HTTP_METHOD=\(.elastic_http_method) ELASTIC_ENDPOINT=\(.elastic_endpoint) ELASTIC_USERNAME=\(.elastic_username) ELASTIC_PASSWORD=\(.elastic_password) ELASTIC_JSON_BODY=\(.elastic_json_body) ELASTIC_INDEX_NAME=\(.elastic_index_name)"')" + +# Create index +# output=$(curl -s -X ${ELASTIC_HTTP_METHOD} -u "$ELASTIC_USERNAME:$ELASTIC_PASSWORD" \ +# ${ELASTIC_ENDPOINT}/${ELASTIC_INDEX_NAME}) + +# Define mapping +output=$(curl -s -X ${ELASTIC_HTTP_METHOD} -u "$ELASTIC_USERNAME:$ELASTIC_PASSWORD" \ + -H 'Content-Type:application/json' -d "$ELASTIC_JSON_BODY" \ + ${ELASTIC_ENDPOINT}/${ELASTIC_INDEX_NAME} | jq '.') + +# Return response +ACKNOWLEDGED=$( echo $output | jq -r '.acknowledged' ) +jq -n --arg acknowledged "$ACKNOWLEDGED" '{"acknowledged" : $acknowledged}' diff --git a/scripts/kb_add_integration_to_policy.sh b/scripts/kb_add_integration_to_policy.sh new file mode 100644 index 000000000..96ee8b9a4 --- /dev/null +++ b/scripts/kb_add_integration_to_policy.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +eval "$(jq -r '@sh "ELASTIC_HTTP_METHOD=\(.elastic_http_method) ELASTIC_ENDPOINT=\(.kibana_endpoint) ELASTIC_USERNAME=\(.elastic_username) ELASTIC_PASSWORD=\(.elastic_password) ELASTIC_JSON_BODY=\(.elastic_json_body)"')" + +# Define mapping +output=$(curl -s -X POST -u "$ELASTIC_USERNAME:$ELASTIC_PASSWORD" \ + -H "kbn-xsrf: true" -H 'Content-Type:application/json' -d "$ELASTIC_JSON_BODY" \ + ${ELASTIC_ENDPOINT}/api/fleet/package_policies | jq '.') + +# Return response +ID=$( echo $output | jq -r '.item.id' ) +SUCCESS=$( echo $output | jq -r '.success' ) +ERROR=$( echo $output | jq -r '.error' ) +MESSAGE=$( echo $output | jq -r '.message' ) + +jq -n --arg id "$ID" --arg success "$SUCCESS" --arg error "$ERROR" --arg message "$MESSAGE" '{"id": $id, "success" : $success, "error": $error, "message": $message}' \ No newline at end of file diff --git a/scripts/kb_create_agent_policy.sh b/scripts/kb_create_agent_policy.sh new file mode 100644 index 000000000..10eee3467 --- /dev/null +++ b/scripts/kb_create_agent_policy.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +eval "$(jq -r '@sh "ELASTIC_HTTP_METHOD=\(.elastic_http_method) ELASTIC_ENDPOINT=\(.kibana_endpoint) ELASTIC_USERNAME=\(.elastic_username) ELASTIC_PASSWORD=\(.elastic_password) ELASTIC_JSON_BODY=\(.elastic_json_body)"')" + +# Define mapping +output=$(curl -s -X POST -u "$ELASTIC_USERNAME:$ELASTIC_PASSWORD" \ + -H "kbn-xsrf: true" -H 'Content-Type:application/json' -d "$ELASTIC_JSON_BODY" \ + ${ELASTIC_ENDPOINT}/api/fleet/agent_policies?sys_monitoring=true | jq '.') + +# Return response +ID=$( echo $output | jq -r '.item.id' ) +SUCCESS=$( echo $output | jq -r '.success' ) +ERROR=$( echo $output | jq -r '.error' ) +MESSAGE=$( echo $output | jq -r '.message' ) + +if [ "${ERROR}" = "Conflict" ] ;then + ##Regex to extract agent ID from the error message + ID=$( echo $MESSAGE | grep -P "'[0-9a-z\-]+'" -o | grep -P "[0-9a-z\-]+" -o ) +fi + +jq -n --arg id "$ID" --arg success "$SUCCESS" --arg error "$ERROR" --arg message "$MESSAGE" '{"id": $id, "success" : $success, "error": $error, "message": $message}' \ No newline at end of file diff --git a/scripts/kb_upload_saved_objects.sh b/scripts/kb_upload_saved_objects.sh new file mode 100644 index 000000000..215348ae4 --- /dev/null +++ b/scripts/kb_upload_saved_objects.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +eval "$(jq -r '@sh "ELASTIC_HTTP_METHOD=\(.elastic_http_method) ELASTIC_ENDPOINT=\(.kibana_endpoint) ELASTIC_USERNAME=\(.elastic_username) ELASTIC_PASSWORD=\(.elastic_password) SO_FILE=\(.so_file)"')" + +# Define mapping +output=$(curl -s -X ${ELASTIC_HTTP_METHOD} -u "$ELASTIC_USERNAME:$ELASTIC_PASSWORD" \ + -H "kbn-xsrf: true" --form file=@${SO_FILE} \ + ${ELASTIC_ENDPOINT}/api/saved_objects/_import | jq '.') + +# Return response +SUCCESS=$( echo $output | jq -r '.success' ) +ERROR=$( echo $output | jq -r '.error' ) +MESSAGE=$( echo $output | jq -r '.message' ) + +jq -n --arg success "$SUCCESS" --arg error "$ERROR" --arg message "$MESSAGE" '{"success" : $success, "error": $error, "message": $message}' \ No newline at end of file diff --git a/terraform/ec_gc_deployment.tf b/terraform/ec_gc_deployment.tf new file mode 100644 index 000000000..e4879c618 --- /dev/null +++ b/terraform/ec_gc_deployment.tf @@ -0,0 +1,78 @@ +# ------------------------------------------------------------- +# Deploy Elastic Cloud +# ------------------------------------------------------------- +resource "ec_deployment" "elastic_gc_deployment" { + name = var.elastic_gc_deployment_name + region = var.elastic_gc_region + version = var.elastic_version + deployment_template_id = var.elastic_gc_deployment_template_id + elasticsearch { + autoscale = "true" + } + kibana {} + integrations_server {} +} + +output "elastic_endpoint" { + value = ec_deployment.elastic_gc_deployment.elasticsearch[0].https_endpoint +} + +output "elastic_password" { + value = ec_deployment.elastic_gc_deployment.elasticsearch_password + sensitive=true +} + +output "elastic_cloud_id" { + value = ec_deployment.elastic_gc_deployment.elasticsearch[0].cloud_id +} + +output "elastic_username" { + value = ec_deployment.elastic_gc_deployment.elasticsearch_username +} + +# ------------------------------------------------------------- +# Load Policy +# ------------------------------------------------------------- + +data "external" "elastic_create_gcp_policy" { + query = { + kibana_endpoint = ec_deployment.elastic_gc_deployment.kibana[0].https_endpoint + elastic_username = ec_deployment.elastic_gc_deployment.elasticsearch_username + elastic_password = ec_deployment.elastic_gc_deployment.elasticsearch_password + elastic_json_body = templatefile("../json_templates/default-policy.json", {"policy_name": "GCP"}) + } + program = ["sh", "../scripts/kb_create_agent_policy.sh" ] + depends_on = [ec_deployment.elastic_gc_deployment] +} + +output "elastic_create_gcp_policy" { + value = data.external.elastic_create_gcp_policy.result + depends_on = [data.external.elastic_create_gcp_policy] +} + +data "external" "elastic_add_gcp_integration" { + query = { + kibana_endpoint = ec_deployment.elastic_gc_deployment.kibana[0].https_endpoint + elastic_username = ec_deployment.elastic_gc_deployment.elasticsearch_username + elastic_password = ec_deployment.elastic_gc_deployment.elasticsearch_password + elastic_json_body = templatefile("../json_templates/gcp_integration.json", + { + "policy_id": data.external.elastic_create_gcp_policy.result.id, + "gcp_project": var.google_cloud_project, + "gcp_credentials_json": jsonencode(file(var.google_cloud_service_account_path)), + "audit_log_topic": var.google_pubsub_audit_topic, + "firewall_log_topic": var.google_pubsub_firewall_topic, + "vpcflow_log_topic": var.google_pubsub_vpcflow_topic, + "dns_log_topic": var.google_pubsub_dns_topic, + "lb_log_topic": var.google_pubsub_lb_topic + } + ) + } + program = ["sh", "../scripts/kb_add_integration_to_policy.sh" ] + depends_on = [data.external.elastic_create_gcp_policy] +} + +output "elastic_add_gcp_integration" { + value = data.external.elastic_add_gcp_integration.result + depends_on = [data.external.elastic_add_gcp_integration] +} \ No newline at end of file diff --git a/terraform/gc_dataflow.tf b/terraform/gc_dataflow.tf new file mode 100644 index 000000000..d2089ec1b --- /dev/null +++ b/terraform/gc_dataflow.tf @@ -0,0 +1,20 @@ +# ------------------------------------------------------------- +# Create a Dataflow job to read from BigQuery and write to Elastic +# ------------------------------------------------------------- +# resource "google_dataflow_flex_template_job" "read_from_bigquery_to_elasticserach" { +# project = var.google_cloud_project +# provider = google-beta +# name = var.google_cloud_dataflow_job_name +# region = var.google_cloud_region +# container_spec_gcs_path = var.google_cloud_container_spec_gcs_path +# parameters = { +# connectionUrl = ec_deployment.elastic_gc_deployment.elasticsearch[0].cloud_id +# apiKey = data.external.elastic_generate_api_key.result.encoded +# index = var.elastic_index_name +# inputTableSpec = var.google_cloud_inputTableSpec +# maxNumWorkers = var.google_cloud_maxNumWorkers +# maxRetryAttempts = var.google_cloud_maxRetryAttempts +# maxRetryDuration = var.google_cloud_maxRetryDuration +# } +# depends_on = [data.external.elastic_generate_api_key] +# } \ No newline at end of file diff --git a/terraform/gc_elastic_agent.tf b/terraform/gc_elastic_agent.tf new file mode 100644 index 000000000..56ec367bc --- /dev/null +++ b/terraform/gc_elastic_agent.tf @@ -0,0 +1,80 @@ +# ------------------------------------------------------------- +# Create Compute VM + Elastic Agent +# ------------------------------------------------------------- + +resource "google_compute_instance" "vm_instance" { + depends_on = [ec_deployment.elastic_gc_deployment, data.external.elastic_create_gcp_policy] ## We want to have the elastic deployment before we install the agent + + name = "elastic-agent" + machine_type = "e2-standard-2" + tags = ["terraformed"] + + + boot_disk { + initialize_params { + image = "debian-cloud/debian-11" + size = 20 + } + } + + network_interface { + network = "default" + access_config { + // Ephemeral public IP + } + } + + metadata_startup_script = <