Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

POC for resumable GKE creates #5197

Closed
wants to merge 1 commit into from
Closed

POC for resumable GKE creates #5197

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
78 changes: 39 additions & 39 deletions google/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ type Config struct {
RequestTimeout time.Duration

client *http.Client
context context.Context
terraformVersion string
userAgent string

Expand Down Expand Up @@ -243,7 +244,7 @@ var defaultClientScopes = []string{
"https://www.googleapis.com/auth/userinfo.email",
}

func (c *Config) LoadAndValidate() error {
func (c *Config) LoadAndValidate(ctx context.Context) error {
if len(c.Scopes) == 0 {
c.Scopes = defaultClientScopes
}
Expand All @@ -264,18 +265,17 @@ func (c *Config) LoadAndValidate() error {
userAgent := fmt.Sprintf("%s %s", tfUserAgent, providerVersion)

c.client = client
c.context = ctx
c.userAgent = userAgent

context := context.Background()

// This base path and some others below need the version and possibly more of the path
// set on them. The client libraries are inconsistent about which values they need;
// while most only want the host URL, some older ones also want the version and some
// of those "projects" as well. You can find out if this is required by looking at
// the basePath value in the client library file.
computeClientBasePath := c.ComputeBasePath + "projects/"
log.Printf("[INFO] Instantiating GCE client for path %s", computeClientBasePath)
c.clientCompute, err = compute.NewService(context, option.WithHTTPClient(client))
c.clientCompute, err = compute.NewService(ctx, option.WithHTTPClient(client))
if err != nil {
return err
}
Expand All @@ -284,7 +284,7 @@ func (c *Config) LoadAndValidate() error {

computeBetaClientBasePath := c.ComputeBetaBasePath + "projects/"
log.Printf("[INFO] Instantiating GCE Beta client for path %s", computeBetaClientBasePath)
c.clientComputeBeta, err = computeBeta.NewService(context, option.WithHTTPClient(client))
c.clientComputeBeta, err = computeBeta.NewService(ctx, option.WithHTTPClient(client))
if err != nil {
return err
}
Expand All @@ -293,7 +293,7 @@ func (c *Config) LoadAndValidate() error {

containerClientBasePath := removeBasePathVersion(c.ContainerBasePath)
log.Printf("[INFO] Instantiating GKE client for path %s", containerClientBasePath)
c.clientContainer, err = container.NewService(context, option.WithHTTPClient(client))
c.clientContainer, err = container.NewService(ctx, option.WithHTTPClient(client))
if err != nil {
return err
}
Expand All @@ -302,7 +302,7 @@ func (c *Config) LoadAndValidate() error {

containerBetaClientBasePath := removeBasePathVersion(c.ContainerBetaBasePath)
log.Printf("[INFO] Instantiating GKE Beta client for path %s", containerBetaClientBasePath)
c.clientContainerBeta, err = containerBeta.NewService(context, option.WithHTTPClient(client))
c.clientContainerBeta, err = containerBeta.NewService(ctx, option.WithHTTPClient(client))
if err != nil {
return err
}
Expand All @@ -311,7 +311,7 @@ func (c *Config) LoadAndValidate() error {

dnsClientBasePath := c.DNSBasePath + "projects/"
log.Printf("[INFO] Instantiating Google Cloud DNS client for path %s", dnsClientBasePath)
c.clientDns, err = dns.NewService(context, option.WithHTTPClient(client))
c.clientDns, err = dns.NewService(ctx, option.WithHTTPClient(client))
if err != nil {
return err
}
Expand All @@ -320,7 +320,7 @@ func (c *Config) LoadAndValidate() error {

dnsBetaClientBasePath := c.DnsBetaBasePath + "projects/"
log.Printf("[INFO] Instantiating Google Cloud DNS Beta client for path %s", dnsBetaClientBasePath)
c.clientDnsBeta, err = dnsBeta.NewService(context, option.WithHTTPClient(client))
c.clientDnsBeta, err = dnsBeta.NewService(ctx, option.WithHTTPClient(client))
if err != nil {
return err
}
Expand All @@ -329,7 +329,7 @@ func (c *Config) LoadAndValidate() error {

kmsClientBasePath := removeBasePathVersion(c.KMSBasePath)
log.Printf("[INFO] Instantiating Google Cloud KMS client for path %s", kmsClientBasePath)
c.clientKms, err = cloudkms.NewService(context, option.WithHTTPClient(client))
c.clientKms, err = cloudkms.NewService(ctx, option.WithHTTPClient(client))
if err != nil {
return err
}
Expand All @@ -338,7 +338,7 @@ func (c *Config) LoadAndValidate() error {

loggingClientBasePath := removeBasePathVersion(c.LoggingBasePath)
log.Printf("[INFO] Instantiating Google Stackdriver Logging client for path %s", loggingClientBasePath)
c.clientLogging, err = cloudlogging.NewService(context, option.WithHTTPClient(client))
c.clientLogging, err = cloudlogging.NewService(ctx, option.WithHTTPClient(client))
if err != nil {
return err
}
Expand All @@ -347,7 +347,7 @@ func (c *Config) LoadAndValidate() error {

storageClientBasePath := c.StorageBasePath
log.Printf("[INFO] Instantiating Google Storage client for path %s", storageClientBasePath)
c.clientStorage, err = storage.NewService(context, option.WithHTTPClient(client))
c.clientStorage, err = storage.NewService(ctx, option.WithHTTPClient(client))
if err != nil {
return err
}
Expand All @@ -356,7 +356,7 @@ func (c *Config) LoadAndValidate() error {

sqlClientBasePath := c.SQLBasePath
log.Printf("[INFO] Instantiating Google SqlAdmin client for path %s", sqlClientBasePath)
c.clientSqlAdmin, err = sqladmin.NewService(context, option.WithHTTPClient(client))
c.clientSqlAdmin, err = sqladmin.NewService(ctx, option.WithHTTPClient(client))
if err != nil {
return err
}
Expand All @@ -365,7 +365,7 @@ func (c *Config) LoadAndValidate() error {

pubsubClientBasePath := removeBasePathVersion(c.PubsubBasePath)
log.Printf("[INFO] Instantiating Google Pubsub client for path %s", pubsubClientBasePath)
c.clientPubsub, err = pubsub.NewService(context, option.WithHTTPClient(client))
c.clientPubsub, err = pubsub.NewService(ctx, option.WithHTTPClient(client))
if err != nil {
return err
}
Expand All @@ -374,7 +374,7 @@ func (c *Config) LoadAndValidate() error {

dataflowClientBasePath := removeBasePathVersion(c.DataflowBasePath)
log.Printf("[INFO] Instantiating Google Dataflow client for path %s", dataflowClientBasePath)
c.clientDataflow, err = dataflow.NewService(context, option.WithHTTPClient(client))
c.clientDataflow, err = dataflow.NewService(ctx, option.WithHTTPClient(client))
if err != nil {
return err
}
Expand All @@ -383,7 +383,7 @@ func (c *Config) LoadAndValidate() error {

resourceManagerBasePath := removeBasePathVersion(c.ResourceManagerBasePath)
log.Printf("[INFO] Instantiating Google Cloud ResourceManager client for path %s", resourceManagerBasePath)
c.clientResourceManager, err = cloudresourcemanager.NewService(context, option.WithHTTPClient(client))
c.clientResourceManager, err = cloudresourcemanager.NewService(ctx, option.WithHTTPClient(client))
if err != nil {
return err
}
Expand All @@ -392,7 +392,7 @@ func (c *Config) LoadAndValidate() error {

resourceManagerV2Beta1BasePath := removeBasePathVersion(c.ResourceManagerV2Beta1BasePath)
log.Printf("[INFO] Instantiating Google Cloud ResourceManager V client for path %s", resourceManagerV2Beta1BasePath)
c.clientResourceManagerV2Beta1, err = resourceManagerV2Beta1.NewService(context, option.WithHTTPClient(client))
c.clientResourceManagerV2Beta1, err = resourceManagerV2Beta1.NewService(ctx, option.WithHTTPClient(client))
if err != nil {
return err
}
Expand All @@ -401,7 +401,7 @@ func (c *Config) LoadAndValidate() error {

runtimeConfigClientBasePath := removeBasePathVersion(c.RuntimeConfigBasePath)
log.Printf("[INFO] Instantiating Google Cloud Runtimeconfig client for path %s", runtimeConfigClientBasePath)
c.clientRuntimeconfig, err = runtimeconfig.NewService(context, option.WithHTTPClient(client))
c.clientRuntimeconfig, err = runtimeconfig.NewService(ctx, option.WithHTTPClient(client))
if err != nil {
return err
}
Expand All @@ -410,7 +410,7 @@ func (c *Config) LoadAndValidate() error {

iamClientBasePath := removeBasePathVersion(c.IAMBasePath)
log.Printf("[INFO] Instantiating Google Cloud IAM client for path %s", iamClientBasePath)
c.clientIAM, err = iam.NewService(context, option.WithHTTPClient(client))
c.clientIAM, err = iam.NewService(ctx, option.WithHTTPClient(client))
if err != nil {
return err
}
Expand All @@ -419,7 +419,7 @@ func (c *Config) LoadAndValidate() error {

iamCredentialsClientBasePath := removeBasePathVersion(c.IamCredentialsBasePath)
log.Printf("[INFO] Instantiating Google Cloud IAMCredentials client for path %s", iamCredentialsClientBasePath)
c.clientIamCredentials, err = iamcredentials.NewService(context, option.WithHTTPClient(client))
c.clientIamCredentials, err = iamcredentials.NewService(ctx, option.WithHTTPClient(client))
if err != nil {
return err
}
Expand All @@ -428,7 +428,7 @@ func (c *Config) LoadAndValidate() error {

serviceManagementClientBasePath := removeBasePathVersion(c.ServiceManagementBasePath)
log.Printf("[INFO] Instantiating Google Cloud Service Management client for path %s", serviceManagementClientBasePath)
c.clientServiceMan, err = servicemanagement.NewService(context, option.WithHTTPClient(client))
c.clientServiceMan, err = servicemanagement.NewService(ctx, option.WithHTTPClient(client))
if err != nil {
return err
}
Expand All @@ -437,7 +437,7 @@ func (c *Config) LoadAndValidate() error {

serviceUsageClientBasePath := removeBasePathVersion(c.ServiceUsageBasePath)
log.Printf("[INFO] Instantiating Google Cloud Service Usage client for path %s", serviceUsageClientBasePath)
c.clientServiceUsage, err = serviceusage.NewService(context, option.WithHTTPClient(client))
c.clientServiceUsage, err = serviceusage.NewService(ctx, option.WithHTTPClient(client))
if err != nil {
return err
}
Expand All @@ -446,7 +446,7 @@ func (c *Config) LoadAndValidate() error {

cloudBillingClientBasePath := removeBasePathVersion(c.CloudBillingBasePath)
log.Printf("[INFO] Instantiating Google Cloud Billing client for path %s", cloudBillingClientBasePath)
c.clientBilling, err = cloudbilling.NewService(context, option.WithHTTPClient(client))
c.clientBilling, err = cloudbilling.NewService(ctx, option.WithHTTPClient(client))
if err != nil {
return err
}
Expand All @@ -455,7 +455,7 @@ func (c *Config) LoadAndValidate() error {

cloudBuildClientBasePath := removeBasePathVersion(c.CloudBuildBasePath)
log.Printf("[INFO] Instantiating Google Cloud Build client for path %s", cloudBuildClientBasePath)
c.clientBuild, err = cloudbuild.NewService(context, option.WithHTTPClient(client))
c.clientBuild, err = cloudbuild.NewService(ctx, option.WithHTTPClient(client))
if err != nil {
return err
}
Expand All @@ -464,7 +464,7 @@ func (c *Config) LoadAndValidate() error {

bigQueryClientBasePath := c.BigQueryBasePath
log.Printf("[INFO] Instantiating Google Cloud BigQuery client for path %s", bigQueryClientBasePath)
c.clientBigQuery, err = bigquery.NewService(context, option.WithHTTPClient(client))
c.clientBigQuery, err = bigquery.NewService(ctx, option.WithHTTPClient(client))
if err != nil {
return err
}
Expand All @@ -473,7 +473,7 @@ func (c *Config) LoadAndValidate() error {

cloudFunctionsClientBasePath := removeBasePathVersion(c.CloudFunctionsBasePath)
log.Printf("[INFO] Instantiating Google Cloud CloudFunctions Client for path %s", cloudFunctionsClientBasePath)
c.clientCloudFunctions, err = cloudfunctions.NewService(context, option.WithHTTPClient(client))
c.clientCloudFunctions, err = cloudfunctions.NewService(ctx, option.WithHTTPClient(client))
if err != nil {
return err
}
Expand All @@ -488,7 +488,7 @@ func (c *Config) LoadAndValidate() error {
bigtableAdminBasePath := removeBasePathVersion(c.BigtableAdminBasePath)
log.Printf("[INFO] Instantiating Google Cloud BigtableAdmin for path %s", bigtableAdminBasePath)

clientBigtable, err := bigtableadmin.NewService(context, option.WithHTTPClient(client))
clientBigtable, err := bigtableadmin.NewService(ctx, option.WithHTTPClient(client))
if err != nil {
return err
}
Expand All @@ -498,7 +498,7 @@ func (c *Config) LoadAndValidate() error {

sourceRepoClientBasePath := removeBasePathVersion(c.SourceRepoBasePath)
log.Printf("[INFO] Instantiating Google Cloud Source Repo client for path %s", sourceRepoClientBasePath)
c.clientSourceRepo, err = sourcerepo.NewService(context, option.WithHTTPClient(client))
c.clientSourceRepo, err = sourcerepo.NewService(ctx, option.WithHTTPClient(client))
if err != nil {
return err
}
Expand All @@ -507,7 +507,7 @@ func (c *Config) LoadAndValidate() error {

spannerClientBasePath := removeBasePathVersion(c.SpannerBasePath)
log.Printf("[INFO] Instantiating Google Cloud Spanner client for path %s", spannerClientBasePath)
c.clientSpanner, err = spanner.NewService(context, option.WithHTTPClient(client))
c.clientSpanner, err = spanner.NewService(ctx, option.WithHTTPClient(client))
if err != nil {
return err
}
Expand All @@ -516,7 +516,7 @@ func (c *Config) LoadAndValidate() error {

dataprocClientBasePath := removeBasePathVersion(c.DataprocBasePath)
log.Printf("[INFO] Instantiating Google Cloud Dataproc client for path %s", dataprocClientBasePath)
c.clientDataproc, err = dataproc.NewService(context, option.WithHTTPClient(client))
c.clientDataproc, err = dataproc.NewService(ctx, option.WithHTTPClient(client))
if err != nil {
return err
}
Expand All @@ -525,7 +525,7 @@ func (c *Config) LoadAndValidate() error {

dataprocBetaClientBasePath := removeBasePathVersion(c.DataprocBetaBasePath)
log.Printf("[INFO] Instantiating Google Cloud Dataproc Beta client for path %s", dataprocBetaClientBasePath)
c.clientDataprocBeta, err = dataprocBeta.NewService(context, option.WithHTTPClient(client))
c.clientDataprocBeta, err = dataprocBeta.NewService(ctx, option.WithHTTPClient(client))
if err != nil {
return err
}
Expand All @@ -534,7 +534,7 @@ func (c *Config) LoadAndValidate() error {

filestoreClientBasePath := removeBasePathVersion(c.FilestoreBasePath)
log.Printf("[INFO] Instantiating Filestore client for path %s", filestoreClientBasePath)
c.clientFilestore, err = file.NewService(context, option.WithHTTPClient(client))
c.clientFilestore, err = file.NewService(ctx, option.WithHTTPClient(client))
if err != nil {
return err
}
Expand All @@ -543,7 +543,7 @@ func (c *Config) LoadAndValidate() error {

cloudIoTClientBasePath := removeBasePathVersion(c.CloudIoTBasePath)
log.Printf("[INFO] Instantiating Google Cloud IoT Core client for path %s", cloudIoTClientBasePath)
c.clientCloudIoT, err = cloudiot.NewService(context, option.WithHTTPClient(client))
c.clientCloudIoT, err = cloudiot.NewService(ctx, option.WithHTTPClient(client))
if err != nil {
return err
}
Expand All @@ -552,7 +552,7 @@ func (c *Config) LoadAndValidate() error {

appEngineClientBasePath := removeBasePathVersion(c.AppEngineBasePath)
log.Printf("[INFO] Instantiating App Engine client for path %s", appEngineClientBasePath)
c.clientAppEngine, err = appengine.NewService(context, option.WithHTTPClient(client))
c.clientAppEngine, err = appengine.NewService(ctx, option.WithHTTPClient(client))
if err != nil {
return err
}
Expand All @@ -561,7 +561,7 @@ func (c *Config) LoadAndValidate() error {

composerClientBasePath := removeBasePathVersion(c.ComposerBasePath)
log.Printf("[INFO] Instantiating Cloud Composer client for path %s", composerClientBasePath)
c.clientComposer, err = composer.NewService(context, option.WithHTTPClient(client))
c.clientComposer, err = composer.NewService(ctx, option.WithHTTPClient(client))
if err != nil {
return err
}
Expand All @@ -570,7 +570,7 @@ func (c *Config) LoadAndValidate() error {

serviceNetworkingClientBasePath := removeBasePathVersion(c.ServiceNetworkingBasePath)
log.Printf("[INFO] Instantiating Service Networking client for path %s", serviceNetworkingClientBasePath)
c.clientServiceNetworking, err = servicenetworking.NewService(context, option.WithHTTPClient(client))
c.clientServiceNetworking, err = servicenetworking.NewService(ctx, option.WithHTTPClient(client))
if err != nil {
return err
}
Expand All @@ -579,7 +579,7 @@ func (c *Config) LoadAndValidate() error {

storageTransferClientBasePath := removeBasePathVersion(c.StorageTransferBasePath)
log.Printf("[INFO] Instantiating Google Cloud Storage Transfer client for path %s", storageTransferClientBasePath)
c.clientStorageTransfer, err = storagetransfer.NewService(context, option.WithHTTPClient(client))
c.clientStorageTransfer, err = storagetransfer.NewService(ctx, option.WithHTTPClient(client))
if err != nil {
return err
}
Expand All @@ -588,8 +588,8 @@ func (c *Config) LoadAndValidate() error {

c.Region = GetRegionFromRegionSelfLink(c.Region)

c.requestBatcherServiceUsage = NewRequestBatcher("Service Usage", context, c.BatchingConfig)
c.requestBatcherIam = NewRequestBatcher("IAM", context, c.BatchingConfig)
c.requestBatcherServiceUsage = NewRequestBatcher("Service Usage", ctx, c.BatchingConfig)
c.requestBatcherIam = NewRequestBatcher("IAM", ctx, c.BatchingConfig)

return nil
}
Expand Down
12 changes: 12 additions & 0 deletions google/container_operation.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,17 @@
package google

import (
"context"
"errors"
"fmt"
"log"

container "google.golang.org/api/container/v1beta1"
)

type ContainerOperationWaiter struct {
Service *container.Service
Context context.Context
Op *container.Operation
Project string
Location string
Expand Down Expand Up @@ -62,6 +66,13 @@ func (w *ContainerOperationWaiter) QueryOp() (interface{}, error) {
w.Project, w.Location, w.Op.Name)

var op *container.Operation
select {
case <-w.Context.Done():
log.Println("[WARN] request has been cancelled early")
return op, errors.New("unable to finish polling, context has been cancelled")
default:
// default must be here to keep the previous case from blocking
}
err := retryTimeDuration(func() (opErr error) {
op, opErr = w.Service.Projects.Locations.Operations.Get(name).Do()
return opErr
Expand All @@ -88,6 +99,7 @@ func (w *ContainerOperationWaiter) TargetStates() []string {
func containerOperationWait(config *Config, op *container.Operation, project, location, activity string, timeoutMinutes int) error {
w := &ContainerOperationWaiter{
Service: config.clientContainerBeta,
Context: config.context,
Op: op,
Project: project,
Location: location,
Expand Down
Loading