From ecacdcb4a00c2b0a7178ef94ef73f0e5f84cefde Mon Sep 17 00:00:00 2001 From: Sam Levenick Date: Mon, 26 Aug 2019 09:34:59 -0700 Subject: [PATCH] Inspec org log sink (#2243) Merged PR #2243. --- build/ansible | 2 +- build/inspec | 2 +- build/terraform | 2 +- build/terraform-beta | 2 +- build/terraform-mapper | 2 +- products/logging/api.yaml | 40 +++++++++++++++++++ products/logging/inspec.yaml | 21 ++++++++++ products/logging/terraform.yaml | 2 + templates/inspec/doc_template.md.erb | 4 +- .../google_logging_organization_log_sink.erb | 10 +++++ ...gging_organization_log_sink_attributes.erb | 3 ++ .../google_logging_organization_log_sinks.erb | 5 +++ .../inspec/tests/integration/build/gcp-mm.tf | 18 ++++++++- .../configuration/mm-attributes.yml | 6 ++- 14 files changed, 110 insertions(+), 9 deletions(-) create mode 100644 products/logging/inspec.yaml create mode 100644 templates/inspec/examples/google_logging_organization_log_sink/google_logging_organization_log_sink.erb create mode 100644 templates/inspec/examples/google_logging_organization_log_sink/google_logging_organization_log_sink_attributes.erb create mode 100644 templates/inspec/examples/google_logging_organization_log_sink/google_logging_organization_log_sinks.erb diff --git a/build/ansible b/build/ansible index 63358435e17b..d979b11fbe02 160000 --- a/build/ansible +++ b/build/ansible @@ -1 +1 @@ -Subproject commit 63358435e17b41b2f68dbcdd30111877d34082e2 +Subproject commit d979b11fbe025cd91f7396a39d7f7720f288c494 diff --git a/build/inspec b/build/inspec index 955339f7c5ca..f40d22c78870 160000 --- a/build/inspec +++ b/build/inspec @@ -1 +1 @@ -Subproject commit 955339f7c5ca57e6c866bc85b8ee5b03d6e87f90 +Subproject commit f40d22c7887067f6bf5b8096b0ad73f79ad8678e diff --git a/build/terraform b/build/terraform index 1ed5f94e1a49..b5462e6dc85c 160000 --- a/build/terraform +++ b/build/terraform @@ -1 +1 @@ -Subproject commit 1ed5f94e1a499ec9a1af33f7f777ed449c51f736 +Subproject commit b5462e6dc85c2d3a3b0f0b7f20eebea040a196c2 diff --git a/build/terraform-beta b/build/terraform-beta index c2b754ec32c9..1f182c35df50 160000 --- a/build/terraform-beta +++ b/build/terraform-beta @@ -1 +1 @@ -Subproject commit c2b754ec32c9ea2e69f710e8dcc9ccd1c753ca40 +Subproject commit 1f182c35df505add2ddbfefbac6eaa21891ac525 diff --git a/build/terraform-mapper b/build/terraform-mapper index aaebe37f5b93..791ccb94729b 160000 --- a/build/terraform-mapper +++ b/build/terraform-mapper @@ -1 +1 @@ -Subproject commit aaebe37f5b93273cf5820abcfea0881362489474 +Subproject commit 791ccb94729b4293e2af0db7993d7406e907d244 diff --git a/products/logging/api.yaml b/products/logging/api.yaml index 90cf53c0ac6d..6f76f7cc22dc 100644 --- a/products/logging/api.yaml +++ b/products/logging/api.yaml @@ -190,3 +190,43 @@ objects: item_type: Api::Type::String description: | The values must be monotonically increasing. + - !ruby/object:Api::Resource + name: "OrganizationLogSink" + base_url: organizations/{{organization}}/sinks + self_link: organizations/{{organization}}/sinks/{{name}} + collection_url_key: 'sinks' + description: | + Describes a sink used to export log entries + properties: + - !ruby/object:Api::Type::String + name: organization + description: | + Id of the organization that this sink belongs to. + required: true + - !ruby/object:Api::Type::String + name: name + description: | + Name of the log sink. + required: true + - !ruby/object:Api::Type::String + name: filter + description: | + An advanced logs filter. The only exported log entries are those that are in the + resource owning the sink and that match the filter. + - !ruby/object:Api::Type::String + name: destination + description: | + The export destination. + - !ruby/object:Api::Type::String + name: writerIdentity + description: | + An IAM identity—a service account or group—under which Logging writes the exported + log entries to the sink's destination. This field is set by sinks.create and sinks.update + based on the value of uniqueWriterIdentity in those methods. + - !ruby/object:Api::Type::Boolean + name: includeChildren + description: | + If the field is false, the default, only the logs owned by the sink's parent resource are + available for export. If the field is true, then logs from all the projects, folders, and + billing accounts contained in the sink's parent resource are also available for export. + Whether a particular log entry from the children is exported depends on the sink's filter expression. diff --git a/products/logging/inspec.yaml b/products/logging/inspec.yaml new file mode 100644 index 000000000000..f6bd4b5070c1 --- /dev/null +++ b/products/logging/inspec.yaml @@ -0,0 +1,21 @@ +# Copyright 2017 Google Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- !ruby/object:Provider::Inspec::Config +overrides: !ruby/object:Overrides::ResourceOverrides + # Handwritten managed zone resources exist in inspec-gcp already + Metric: !ruby/object:Overrides::Inspec::ResourceOverride + exclude: true + OrganizationLogSink: !ruby/object:Overrides::Inspec::ResourceOverride + # Creating a log sink requires organization level privileges + privileged: true \ No newline at end of file diff --git a/products/logging/terraform.yaml b/products/logging/terraform.yaml index 8e3a027b7349..1172b7c09802 100644 --- a/products/logging/terraform.yaml +++ b/products/logging/terraform.yaml @@ -38,6 +38,8 @@ overrides: !ruby/object:Overrides::ResourceOverrides properties: metricDescriptor.labels.valueType: !ruby/object:Overrides::Terraform::PropertyOverride custom_flatten: 'templates/terraform/custom_flatten/default_if_empty.erb' + OrganizationLogSink: !ruby/object:Overrides::Terraform::ResourceOverride + exclude: true files: !ruby/object:Provider::Config::Files # These files have templating (ERB) code that will be run. diff --git a/templates/inspec/doc_template.md.erb b/templates/inspec/doc_template.md.erb index 937478a24ea5..887d2c720c06 100644 --- a/templates/inspec/doc_template.md.erb +++ b/templates/inspec/doc_template.md.erb @@ -50,11 +50,11 @@ with `where` as a block or a method. <% end -%> <% end # if plural -%> -<% unless @api.apis_required.empty? -%> +<% unless @api.apis_required&.empty? -%> ## GCP Permissions -<% @api.apis_required.each do |api| -%> +<% @api.apis_required&.each do |api| -%> Ensure the [<%= api.name -%>](<%= api.url -%>) is enabled for the current project. <% end # @api.apis_required.each -%> <% end # unless @api.apis_required.empty? -%> \ No newline at end of file diff --git a/templates/inspec/examples/google_logging_organization_log_sink/google_logging_organization_log_sink.erb b/templates/inspec/examples/google_logging_organization_log_sink/google_logging_organization_log_sink.erb new file mode 100644 index 000000000000..0210d6976c93 --- /dev/null +++ b/templates/inspec/examples/google_logging_organization_log_sink/google_logging_organization_log_sink.erb @@ -0,0 +1,10 @@ +<% gcp_organization_id = "#{external_attribute('gcp_organization_id', doc_generation)}" -%> +<% org_sink = grab_attributes['org_sink'] -%> +describe google_logging_organization_log_sink(organization: <%= gcp_organization_id -%>, name: <%= doc_generation ? "'#{org_sink['name']}'" : "org_sink['name']" -%>) do + it { should exist } + its('filter') { should cmp <%= doc_generation ? "'#{org_sink['filter']}'" : "org_sink['filter']" -%> } +end + +describe google_logging_organization_log_sink(organization: <%= gcp_organization_id -%>, name: 'nonexistent') do + it { should_not exist } +end \ No newline at end of file diff --git a/templates/inspec/examples/google_logging_organization_log_sink/google_logging_organization_log_sink_attributes.erb b/templates/inspec/examples/google_logging_organization_log_sink/google_logging_organization_log_sink_attributes.erb new file mode 100644 index 000000000000..082ea57f55a3 --- /dev/null +++ b/templates/inspec/examples/google_logging_organization_log_sink/google_logging_organization_log_sink_attributes.erb @@ -0,0 +1,3 @@ +org_sink = attribute('org_sink', default: <%= grab_attributes['org_sink'] -%>) +gcp_organization_id = attribute(:gcp_organization_id, default: <%= external_attribute('gcp_organization_id') -%>, description: 'The identifier of the organization that is the parent of this folder') +gcp_enable_privileged_resources = attribute(:gcp_enable_privileged_resources, default:0, description:'Flag to enable privileged resources requiring elevated privileges in GCP.') \ No newline at end of file diff --git a/templates/inspec/examples/google_logging_organization_log_sink/google_logging_organization_log_sinks.erb b/templates/inspec/examples/google_logging_organization_log_sink/google_logging_organization_log_sinks.erb new file mode 100644 index 000000000000..d3d3c21ebd1a --- /dev/null +++ b/templates/inspec/examples/google_logging_organization_log_sink/google_logging_organization_log_sinks.erb @@ -0,0 +1,5 @@ +<% gcp_organization_id = "#{external_attribute('gcp_organization_id', doc_generation)}" -%> +<% org_sink = grab_attributes['org_sink'] -%> +describe google_logging_organization_log_sinks(organization: <%= gcp_organization_id -%>) do + its('names') { should include <%= doc_generation ? "'#{org_sink['name']}'" : "org_sink['name']" -%> } +end \ No newline at end of file diff --git a/templates/inspec/tests/integration/build/gcp-mm.tf b/templates/inspec/tests/integration/build/gcp-mm.tf index 09a08f096b24..86211476e9dd 100644 --- a/templates/inspec/tests/integration/build/gcp-mm.tf +++ b/templates/inspec/tests/integration/build/gcp-mm.tf @@ -141,6 +141,10 @@ variable "region_backend_service" { type = "map" } +variable "org_sink" { + type = "map" +} + resource "google_compute_ssl_policy" "custom-ssl-policy" { name = "${var.ssl_policy["name"]}" min_tls_version = "${var.ssl_policy["min_tls_version"]}" @@ -511,7 +515,7 @@ resource "google_sourcerepo_repository" "gcp-inspec-sourcerepo-repository" { resource "google_folder" "inspec-gcp-folder" { count = "${var.gcp_organization_id == "none" ? 0 : var.gcp_enable_privileged_resources}" display_name = "${var.folder["display_name"]}" - parent = "${var.gcp_organization_id}" + parent = "organizations/${var.gcp_organization_id}" } resource "google_storage_bucket_object" "archive" { @@ -552,4 +556,16 @@ resource "google_container_node_pool" "inspec-gcp-regional-node-pool" { region = "${var.gcp_location}" cluster = "${google_container_cluster.gcp-inspec-regional-cluster.name}" node_count = "${var.regional_node_pool["node_count"]}" +} + +resource "google_logging_organization_sink" "my-sink" { + count = "${var.gcp_organization_id == "none" ? 0 : var.gcp_enable_privileged_resources}" + name = "${var.org_sink.name}" + org_id = "${var.gcp_organization_id}" + + # Can export to pubsub, cloud storage, or bigquery + destination = "storage.googleapis.com/${google_storage_bucket.generic-storage-bucket.name}" + + # Log all WARN or higher severity messages relating to instances + filter = "${var.org_sink.filter}" } \ No newline at end of file diff --git a/templates/inspec/tests/integration/configuration/mm-attributes.yml b/templates/inspec/tests/integration/configuration/mm-attributes.yml index d9ba8968bc81..e422a48d55f0 100644 --- a/templates/inspec/tests/integration/configuration/mm-attributes.yml +++ b/templates/inspec/tests/integration/configuration/mm-attributes.yml @@ -220,4 +220,8 @@ regional_node_pool: name: inspec-gcp-regional-node-pool cluster_name: inspec-gcp-regional-node-pool-cluster node_count: 1 - initial_node_count: 1 \ No newline at end of file + initial_node_count: 1 + +org_sink: + name: inspec-gcp-org-sink + filter: resource.type = gce_instance \ No newline at end of file