diff --git a/.gitignore b/.gitignore index f2ac5f40dd..01b747b916 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,4 @@ fast/stages/**/globals.auto.tfvars.json cloud_sql_proxy examples/cloud-operations/binauthz/tenant-setup.yaml examples/cloud-operations/binauthz/app/app.yaml +env/ diff --git a/examples/data-solutions/README.md b/examples/data-solutions/README.md index ec2cfd08d6..23fabdcc27 100644 --- a/examples/data-solutions/README.md +++ b/examples/data-solutions/README.md @@ -32,4 +32,10 @@ This [example](./data-platform-foundations/) implements SQL Server Always On Ava This [example](./cloudsql-multiregion/) creates a [Cloud SQL instance](https://cloud.google.com/sql) with multi-region read replicas as described in the [Cloud SQL for PostgreSQL disaster recovery](https://cloud.google.com/architecture/cloud-sql-postgres-disaster-recovery-complete-failover-fallback) article. +
+ +### Data Playground starter with Cloud Vertex AI Notebook and GCS + + +This [example](./data-playground/) creates a [Vertex AI Notebook](https://cloud.google.com/vertex-ai/docs/workbench/introduction) running under a VPC network and a starter GCS bucket to store inputs and outputs of data experiments.
\ No newline at end of file diff --git a/examples/data-solutions/data-playground/README.md b/examples/data-solutions/data-playground/README.md new file mode 100644 index 0000000000..fb75969505 --- /dev/null +++ b/examples/data-solutions/data-playground/README.md @@ -0,0 +1,43 @@ +# Data Playground + +This example creates a minimum viable template for a data experimentation project with the needed APIs enabled, basic VPC and Firewall set in place, GCS bucket and an AI notebook to get started. + +This is the high level diagram: + +![High-level diagram](diagram.png "High-level diagram") + +## Managed resources and services + +This sample creates several distinct groups of resources: + +- projects + - Service Project configured for GCE instances and GCS buckets +- networking + - VPC network + - One default subnet + - Firewall rules for [SSH access via IAP](https://cloud.google.com/iap/docs/using-tcp-forwarding) and open communication within the VPC +- Vertex AI notebook + - One Jupyter lab notebook instance with public access +- GCS + - One bucket initial bucket + + + +## Variables +| name | description | type | required | default | +| ------------------------- | --------------------------------------------------------------------------------------------------------------------------------------- | ----------- | -------- | ------------ | +| project\_id | Project id, references existing project if \`project\_create\` is null. | string | ✓ | | +| location | The location where resources will be deployed | string | | europe | +| region | The region where resources will be deployed. | string | | europe-west1 | +| project\_create | Provide values if project creation is needed, uses existing project if null. Parent format: folders/folder\_id or organizations/org\_id | object({…}) | | null | +| prefix | Unique prefix used for resource names. Not used for project if 'project\_create' is null. | string | | dp | +| service\_encryption\_keys | Cloud KMS to use to encrypt different services. Key location should match service region. | object({…}) | | null | +| vpc\_config | Parameters to create a simple VPC for the Data Playground | object({…}) | | {...} | + +## Outputs +| Name | Description | +| ----------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------- | +| bucket | GCS Bucket URL. | +| project | Project id | +| vpc | VPC Network name | +| notebook | Vertex AI notebook name | diff --git a/examples/data-solutions/data-playground/diagram.png b/examples/data-solutions/data-playground/diagram.png new file mode 100644 index 0000000000..9da71fd09e Binary files /dev/null and b/examples/data-solutions/data-playground/diagram.png differ diff --git a/examples/data-solutions/data-playground/main.tf b/examples/data-solutions/data-playground/main.tf new file mode 100644 index 0000000000..43453f5ae3 --- /dev/null +++ b/examples/data-solutions/data-playground/main.tf @@ -0,0 +1,113 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +############################################################################### +# Project # +############################################################################### +locals { + service_encryption_keys = var.service_encryption_keys +} + +module "project" { + source = "../../../modules/project" + name = var.project_id + parent = try(var.project_create.parent, null) + billing_account = try(var.project_create.billing_account_id, null) + project_create = var.project_create != null + prefix = var.project_create == null ? null : var.prefix + services = [ + "stackdriver.googleapis.com", + "compute.googleapis.com", + "storage-component.googleapis.com", + "storage.googleapis.com", + "servicenetworking.googleapis.com", + "bigquery.googleapis.com", + "bigquerystorage.googleapis.com", + "bigqueryreservation.googleapis.com", + "dataflow.googleapis.com", + "notebooks.googleapis.com", + "composer.googleapis.com" + ] + policy_boolean = { + # "constraints/compute.requireOsLogin" = false + # Example of applying a project wide policy, mainly useful for Composer + } + service_encryption_key_ids = { + storage = [try(local.service_encryption_keys.storage, null)] + } +} + +############################################################################### +# Networking # +############################################################################### + +module "vpc" { + source = "../../../modules/net-vpc" + project_id = module.project.project_id + name = var.vpc_config.vpc_name + subnets = [ + { + ip_cidr_range = var.vpc_config.ip_cidr_range + name = var.vpc_config.subnet_name + region = var.region + secondary_ip_range = {} + } + ] +} + +module "vpc-firewall" { + source = "../../../modules/net-vpc-firewall" + project_id = module.project.project_id + network = module.vpc.name + admin_ranges = [var.vpc_config.ip_cidr_range] +} + +############################################################################### +# GCS # +############################################################################### + +module "base-gcs-bucket" { + source = "../../../modules/gcs" + project_id = module.project.project_id + prefix = module.project.project_id + name = "base" + encryption_key = try(local.service_encryption_keys.storage, null) # Example assignment of an encryption key +} + +############################################################################### +# Vertex AI Notebook # +############################################################################### +# TODO: Add encryption_key to Vertex AI notebooks as well +# TODO: Add shared VPC support +resource "google_notebooks_instance" "playground" { + name = "data-play-notebook" + location = format("%s-%s", var.region, "b") + machine_type = "e2-medium" + project = module.project.project_id + + container_image { + repository = "gcr.io/deeplearning-platform-release/base-cpu" + tag = "latest" + } + + install_gpu_driver = true + boot_disk_type = "PD_SSD" + boot_disk_size_gb = 110 + + no_public_ip = false + no_proxy_access = false + + network = module.vpc.network.id + subnet = module.vpc.subnets[format("%s/%s", var.region, var.vpc_config.subnet_name)].id +} diff --git a/examples/data-solutions/data-playground/outputs.tf b/examples/data-solutions/data-playground/outputs.tf new file mode 100644 index 0000000000..3c47229fb9 --- /dev/null +++ b/examples/data-solutions/data-playground/outputs.tf @@ -0,0 +1,33 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "bucket" { + description = "GCS Bucket URL." + value = module.base-gcs-bucket.url +} + +output "notebook" { + description = "Vertex AI notebook" + value = resource.google_notebooks_instance.playground.name +} + +output "project" { + description = "Project id" + value = module.project.project_id +} + +output "vpc" { + description = "VPC Network" + value = module.vpc.name +} \ No newline at end of file diff --git a/examples/data-solutions/data-playground/variables.tf b/examples/data-solutions/data-playground/variables.tf new file mode 100644 index 0000000000..92f63e2d99 --- /dev/null +++ b/examples/data-solutions/data-playground/variables.tf @@ -0,0 +1,68 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +variable "location" { + description = "The location where resources will be deployed." + type = string + default = "europe" +} + +variable "project_id" { + description = "Project id, references existing project if `project_create` is null." + type = string +} + +variable "project_create" { + description = "Provide values if project creation is needed, uses existing project if null. Parent format: folders/folder_id or organizations/org_id" + type = object({ + billing_account_id = string + parent = string + }) + default = null +} + +variable "prefix" { + description = "Unique prefix used for resource names. Not used for project if 'project_create' is null." + type = string + default = "dp" +} + +variable "region" { + description = "The region where resources will be deployed." + type = string + default = "europe-west1" +} + +variable "service_encryption_keys" { # service encription key + description = "Cloud KMS to use to encrypt different services. Key location should match service region." + type = object({ + storage = string + }) + default = null +} + +variable "vpc_config" { + description = "Parameters to create a simple VPC for the Data Playground" + type = object({ + ip_cidr_range = string + subnet_name = string + vpc_name = string + }) + default = { + ip_cidr_range = "10.0.0.0/20" + subnet_name = "default-subnet" + vpc_name = "data-playground-vpc" + } +} \ No newline at end of file diff --git a/examples/data-solutions/data-playground/versions.tf b/examples/data-solutions/data-playground/versions.tf new file mode 100644 index 0000000000..32e9ab8acc --- /dev/null +++ b/examples/data-solutions/data-playground/versions.tf @@ -0,0 +1,27 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.1.0" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.17.0" + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.17.0" + } + } +} diff --git a/tests/examples/data_solutions/data_playground/__init__.py b/tests/examples/data_solutions/data_playground/__init__.py new file mode 100644 index 0000000000..6d6d1266c3 --- /dev/null +++ b/tests/examples/data_solutions/data_playground/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/examples/data_solutions/data_playground/fixture/main.tf b/tests/examples/data_solutions/data_playground/fixture/main.tf new file mode 100644 index 0000000000..8082f997d8 --- /dev/null +++ b/tests/examples/data_solutions/data_playground/fixture/main.tf @@ -0,0 +1,24 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "test" { + source = "../../../../../examples/data-solutions/data-playground/" + project_id = "sampleproject" + project_create = { + billing_account_id = "123456-123456-123456", + parent = "folders/467898377" + } +} diff --git a/tests/examples/data_solutions/data_playground/test_plan.py b/tests/examples/data_solutions/data_playground/test_plan.py new file mode 100644 index 0000000000..1807e3de2b --- /dev/null +++ b/tests/examples/data_solutions/data_playground/test_plan.py @@ -0,0 +1,26 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +import pytest + + +FIXTURES_DIR = os.path.join(os.path.dirname(__file__), 'fixture') + +def test_resources(e2e_plan_runner): + "Test that plan works and the numbers of resources is as expected." + modules, resources = e2e_plan_runner(FIXTURES_DIR) + assert len(modules) == 4 + assert len(resources) == 23