From e94ab947589bae340633a575d4e9f319bf76f5a4 Mon Sep 17 00:00:00 2001 From: Sam Levenick Date: Mon, 17 Sep 2018 14:26:29 -0700 Subject: [PATCH 1/8] Update compute for chef to 0.3.1 --- products/compute/chef.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/products/compute/chef.yaml b/products/compute/chef.yaml index 264ca20aa9e9..6d2a2a315965 100644 --- a/products/compute/chef.yaml +++ b/products/compute/chef.yaml @@ -16,7 +16,7 @@ # edit InstanceGroup --- !ruby/object:Provider::Chef::Config manifest: !ruby/object:Provider::Chef::Manifest - version: '0.3.0' + version: '0.3.1' source: 'https://github.com/GoogleCloudPlatform/chef-google-compute' issues: 'https://github.com/GoogleCloudPlatform/chef-google-compute/issues' summary: 'A Chef cookbook to manage Google Cloud Compute resources' From 2be468e98b0db91157cc21737ecfb419af3db250 Mon Sep 17 00:00:00 2001 From: Sam Levenick Date: Tue, 18 Sep 2018 12:38:42 -0700 Subject: [PATCH 2/8] Starting bigquery work for chef --- products/bigquery/api.yaml | 9 ---- products/bigquery/chef.yaml | 43 +++++++++++++++++++ products/bigquery/examples/chef/dataset.rb | 31 +++++++++++++ .../bigquery/examples/chef/delete_dataset.rb | 27 ++++++++++++ products/bigquery/examples/chef/readme.rb | 31 +++++++++++++ products/bigquery/examples/chef/table.rb | 43 +++++++++++++++++++ 6 files changed, 175 insertions(+), 9 deletions(-) create mode 100644 products/bigquery/chef.yaml create mode 100644 products/bigquery/examples/chef/dataset.rb create mode 100644 products/bigquery/examples/chef/delete_dataset.rb create mode 100644 products/bigquery/examples/chef/readme.rb create mode 100644 products/bigquery/examples/chef/table.rb diff --git a/products/bigquery/api.yaml b/products/bigquery/api.yaml index b30bd6d7054c..024c8938b6db 100644 --- a/products/bigquery/api.yaml +++ b/products/bigquery/api.yaml @@ -29,9 +29,6 @@ objects: description: > Datasets allow you to organize and control access to your tables. properties: - - !ruby/object:Api::Type::String - name: 'name' - description: 'Dataset name' - !ruby/object:Api::Type::Array name: 'access' description: 'Access controls on the bucket.' @@ -148,12 +145,6 @@ objects: base_url: projects/{{project}}/datasets/{{dataset}}/tables description: > A Table that belongs to a Dataset - parameters: - # TODO(alexstephen): Remove once we have support for placing - # nested object fields in URL - - !ruby/object:Api::Type::String - name: 'dataset' - description: Name of the dataset properties: - !ruby/object:Api::Type::NestedObject name: tableReference diff --git a/products/bigquery/chef.yaml b/products/bigquery/chef.yaml new file mode 100644 index 000000000000..0c23d0453d07 --- /dev/null +++ b/products/bigquery/chef.yaml @@ -0,0 +1,43 @@ +# Copyright 2017 Google Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- !ruby/object:Provider::Chef::Config +manifest: !ruby/object:Provider::Chef::Manifest + version: '0.1.0' + source: 'https://github.com/GoogleCloudPlatform/chef-google-bigquery' + issues: 'https://github.com/GoogleCloudPlatform/chef-google-bigquery/issues' + summary: 'A Chef cookbook to manage Google Cloud BigQuery resources' + description: | + This cookbook provides the built-in types and services for Chef to manage + Google Cloud BigQuery resources, as native Chef types. + depends: +<%= indent(compile('provider/chef/common~depends.yaml'), 4) %> + operating_systems: +<%= indent(include('provider/chef/common~operating_systems.yaml'), 4) %> +overrides: !ruby/object:Provider::ResourceOverrides +examples: !ruby/object:Api::Resource::HashArray + Dataset: + - dataset.rb +files: !ruby/object:Provider::Config::Files + copy: +<%= indent(compile('provider/chef/common~copy.yaml'), 4) %> + compile: +<%= indent(include('provider/chef/common~compile~before.yaml'), 4) %> +<%= indent(include('provider/chef/common~compile~after.yaml'), 4) %> +<% # common~compile~after.yaml should be the last line of compile: section -%> +style: +changelog: + - !ruby/object:Provider::Config::Changelog + version: '0.1.0' + date: 2018-09-15T09:00:00-0700 + general: 'Initial release' diff --git a/products/bigquery/examples/chef/dataset.rb b/products/bigquery/examples/chef/dataset.rb new file mode 100644 index 000000000000..5a70393e92a4 --- /dev/null +++ b/products/bigquery/examples/chef/dataset.rb @@ -0,0 +1,31 @@ +<%# The license inside this block applies to this file +# Copyright 2017 Google Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +-%> +<% unless name == 'README.md' -%> +<%= compile 'templates/license.erb' -%> + +<%= lines(autogen_notice :chef) -%> + +<% end -%> + +<%= compile 'templates/chef/example~auth.rb.erb' -%> + +gbigquery_dataset <%= example_resource_name('example_dataset') -%> do + action :create + dataset_reference ({ + dataset_id: 'example_dataset' + }) + project ENV['PROJECT'] # e.g. 'my-test-project' + credential 'mycred' +end \ No newline at end of file diff --git a/products/bigquery/examples/chef/delete_dataset.rb b/products/bigquery/examples/chef/delete_dataset.rb new file mode 100644 index 000000000000..d75b0dd64b1d --- /dev/null +++ b/products/bigquery/examples/chef/delete_dataset.rb @@ -0,0 +1,27 @@ +<%# The license inside this block applies to this file +# Copyright 2017 Google Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +-%> +<% unless name == "README.md" -%> +<%= compile 'templates/license.erb' -%> + +<%= lines(autogen_notice :chef) -%> + +<%= compile 'templates/chef/example~auth.rb.erb' -%> + +<% end # name == README.md -%> +gbigquery_dataset '<%= example_resource_name('example_dataset') -%>' do + action :delete + project ENV['PROJECT'] # e.g. 'my-test-project' + credential 'mycred' +end diff --git a/products/bigquery/examples/chef/readme.rb b/products/bigquery/examples/chef/readme.rb new file mode 100644 index 000000000000..09988069c987 --- /dev/null +++ b/products/bigquery/examples/chef/readme.rb @@ -0,0 +1,31 @@ +<%# The license inside this block applies to this file +# Copyright 2017 Google Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +-%> +<% unless name == 'README.md' -%> + +<%= compile 'templates/license.erb' -%> + +<%= lines(autogen_notice :chef) -%> + +<% end -%> +<%= compile 'templates/chef/example~auth.rb.erb' -%> + +gbigquery_dataset { <%= example_resource_name('example_dataset') -%>: + action :create + dataset_reference { + dataset_id: 'example_dataset' + } + project ENV['PROJECT'] # e.g. 'my-test-project' + credential 'mycred' +} \ No newline at end of file diff --git a/products/bigquery/examples/chef/table.rb b/products/bigquery/examples/chef/table.rb new file mode 100644 index 000000000000..36d690186a10 --- /dev/null +++ b/products/bigquery/examples/chef/table.rb @@ -0,0 +1,43 @@ +<%# The license inside this block applies to this file +# Copyright 2017 Google Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +-%> +<% unless name == 'README.md' -%> +<%= compile 'templates/license.erb' -%> + +<%= lines(autogen_notice :chef) -%> + +<% end -%> + +<%= compile 'templates/chef/example~auth.rb.erb' -%> + +gbigquery_dataset <%= example_resource_name('example_dataset') -%> do + action :create + dataset_reference ({ + dataset_id: 'example_dataset' + }) + project ENV['PROJECT'] # e.g. 'my-test-project' + credential 'mycred' +end + +gbigquery_table <%= example_resource_name('example_table') -%> do + action :create + # dataset <%= example_resource_name('example_dataset') -%> + table_reference ({ + dataset_id: <%= example_resource_name('example_dataset') -%>, + project_id: 'sam-tutorial-6', + table_id: <%= example_resource_name('example_table') -%> + }) + project ENV['PROJECT'] # e.g. 'my-test-project' + credential 'mycred' +end From 25f0a85a512d9cdb6060e4a4b4e1707a7a44ff9b Mon Sep 17 00:00:00 2001 From: Sam Levenick Date: Tue, 18 Sep 2018 16:35:31 -0700 Subject: [PATCH 3/8] Re add duplicate properties for url construction --- products/bigquery/api.yaml | 6 ++++++ products/bigquery/examples/chef/table.rb | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/products/bigquery/api.yaml b/products/bigquery/api.yaml index 024c8938b6db..a18fc2880fd3 100644 --- a/products/bigquery/api.yaml +++ b/products/bigquery/api.yaml @@ -145,6 +145,12 @@ objects: base_url: projects/{{project}}/datasets/{{dataset}}/tables description: > A Table that belongs to a Dataset + parameters: + # TODO(alexstephen): Remove once we have support for placing + # nested object fields in URL + - !ruby/object:Api::Type::String + name: 'dataset' + description: Name of the dataset properties: - !ruby/object:Api::Type::NestedObject name: tableReference diff --git a/products/bigquery/examples/chef/table.rb b/products/bigquery/examples/chef/table.rb index 36d690186a10..97fb7b7cc237 100644 --- a/products/bigquery/examples/chef/table.rb +++ b/products/bigquery/examples/chef/table.rb @@ -32,10 +32,10 @@ gbigquery_table <%= example_resource_name('example_table') -%> do action :create - # dataset <%= example_resource_name('example_dataset') -%> + dataset <%= example_resource_name('example_dataset') -%> table_reference ({ dataset_id: <%= example_resource_name('example_dataset') -%>, - project_id: 'sam-tutorial-6', + project_id: ENV['PROJECT'], table_id: <%= example_resource_name('example_table') -%> }) project ENV['PROJECT'] # e.g. 'my-test-project' From a148fb21828c5b5dce69e8c8f792a11253e355c6 Mon Sep 17 00:00:00 2001 From: Sam Levenick Date: Thu, 27 Sep 2018 17:06:23 -0700 Subject: [PATCH 4/8] Revert bigquery api --- products/bigquery/api.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/products/bigquery/api.yaml b/products/bigquery/api.yaml index a18fc2880fd3..b30bd6d7054c 100644 --- a/products/bigquery/api.yaml +++ b/products/bigquery/api.yaml @@ -29,6 +29,9 @@ objects: description: > Datasets allow you to organize and control access to your tables. properties: + - !ruby/object:Api::Type::String + name: 'name' + description: 'Dataset name' - !ruby/object:Api::Type::Array name: 'access' description: 'Access controls on the bucket.' From 1e12626eaee995c8e9fa55855df3c8e46c4bf2a4 Mon Sep 17 00:00:00 2001 From: Sam Levenick Date: Thu, 27 Sep 2018 17:19:34 -0700 Subject: [PATCH 5/8] Fix chef bigquery readme --- products/bigquery/examples/chef/readme.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/products/bigquery/examples/chef/readme.rb b/products/bigquery/examples/chef/readme.rb index 09988069c987..b2f24e83edf4 100644 --- a/products/bigquery/examples/chef/readme.rb +++ b/products/bigquery/examples/chef/readme.rb @@ -21,11 +21,11 @@ <% end -%> <%= compile 'templates/chef/example~auth.rb.erb' -%> -gbigquery_dataset { <%= example_resource_name('example_dataset') -%>: +gbigquery_dataset <%= example_resource_name('example_dataset') -%> do action :create - dataset_reference { + dataset_reference ({ dataset_id: 'example_dataset' - } + }) project ENV['PROJECT'] # e.g. 'my-test-project' credential 'mycred' -} \ No newline at end of file +end \ No newline at end of file From bc9d75fe7c69d7738d06654e43915c585998d0dd Mon Sep 17 00:00:00 2001 From: Sam Levenick Date: Thu, 27 Sep 2018 17:23:37 -0700 Subject: [PATCH 6/8] Fix readme formatting for chef bigquery --- products/bigquery/examples/chef/readme.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/products/bigquery/examples/chef/readme.rb b/products/bigquery/examples/chef/readme.rb index b2f24e83edf4..62407ba43280 100644 --- a/products/bigquery/examples/chef/readme.rb +++ b/products/bigquery/examples/chef/readme.rb @@ -28,4 +28,4 @@ }) project ENV['PROJECT'] # e.g. 'my-test-project' credential 'mycred' -end \ No newline at end of file +end From 9ba03ad5c6810593c0ccea253e92e4fb9d99ecab Mon Sep 17 00:00:00 2001 From: Sam Levenick Date: Fri, 28 Sep 2018 09:26:38 -0700 Subject: [PATCH 7/8] Add bigquery submodule for chef --- .gitmodules | 3 +++ build/chef/bigquery | 1 + 2 files changed, 4 insertions(+) create mode 160000 build/chef/bigquery diff --git a/.gitmodules b/.gitmodules index 515df3d2c747..4a11ff8e1358 100644 --- a/.gitmodules +++ b/.gitmodules @@ -81,3 +81,6 @@ [submodule "build/chef/iam"] path = build/chef/iam url = git@github.com:GoogleCloudPlatform/chef-google-iam.git +[submodule "build/chef/bigquery"] + path = build/chef/bigquery + url = https://github.com/GoogleCloudPlatform/chef-google-bigquery diff --git a/build/chef/bigquery b/build/chef/bigquery new file mode 160000 index 000000000000..582fa4064583 --- /dev/null +++ b/build/chef/bigquery @@ -0,0 +1 @@ +Subproject commit 582fa40645833b06d965c410e0ef2fc897de7734 From 2f7ff6ff89c2e9a840e0cfe223dae07220d7df31 Mon Sep 17 00:00:00 2001 From: Sam Levenick Date: Fri, 28 Sep 2018 10:24:26 -0700 Subject: [PATCH 8/8] Add chef bigquery to ci pipeline --- .ci/ci.yml.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/ci.yml.tmpl b/.ci/ci.yml.tmpl index d76d8652433d..20508baaa07c 100644 --- a/.ci/ci.yml.tmpl +++ b/.ci/ci.yml.tmpl @@ -1,4 +1,4 @@ -{% set chef_modules = ['_bundle', 'auth', 'compute', 'sql', 'storage', 'spanner', 'container', 'dns', 'iam'] %} +{% set chef_modules = ['_bundle', 'auth', 'bigquery', 'compute', 'sql', 'storage', 'spanner', 'container', 'dns', 'iam'] %} {% set puppet_modules = ['_bundle', 'auth', 'bigquery', 'compute', 'sql', 'storage', 'spanner', 'container', 'dns', 'pubsub', 'resourcemanager'] %} {% set terraform_enabled = true %} {% set ansible_enabled = true %}