From 5c36af9988031e0df6c98b8cc39cdc016a3e917d Mon Sep 17 00:00:00 2001 From: Jingyi Hu Date: Fri, 13 Sep 2024 23:43:15 +0000 Subject: [PATCH 01/12] feat: generate CRD for BigQueryDataTransferConfig --- .../bigquerydatatransferconfig_types.go | 6 +- .../v1alpha1/zz_generated.deepcopy.go | 18 +- ...erydatatransfer.cnrm.cloud.google.com.yaml | 361 +++++++++++------- dev/tasks/generate-crds | 2 +- .../testdata/exceptions/acronyms.txt | 2 - 5 files changed, 238 insertions(+), 151 deletions(-) diff --git a/apis/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig_types.go b/apis/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig_types.go index 8e3fa5e854..a0b59f0566 100644 --- a/apis/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig_types.go +++ b/apis/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig_types.go @@ -25,7 +25,7 @@ var BigQueryDataTransferConfigGVK = GroupVersion.WithKind("BigQueryDataTransferC // +kcc:proto=google.cloud.bigquery.datatransfer.v1.EncryptionConfiguration type EncryptionConfiguration struct { // The KMS key used for encrypting BigQuery data. - KmsKeyRef refv1beta1.KMSCryptoKeyRef `json:"kmsKeyRef,omitempty"` + KmsKeyRef *refv1beta1.KMSCryptoKeyRef `json:"kmsKeyRef,omitempty"` } // BigQueryDataTransferConfigSpec defines the desired state of BigQueryDataTransferConfig @@ -47,7 +47,7 @@ type BigQueryDataTransferConfigSpec struct { DataSourceID *string `json:"dataSourceID,omitempty"` // The BigQuery target dataset id. - DatasetRef refv1beta1.BigQueryDatasetRef `json:"datasetRef,omitempty"` + DatasetRef *refv1beta1.BigQueryDatasetRef `json:"datasetRef,omitempty"` // Is this config disabled. When set to true, no runs will be scheduled for // this transfer config. @@ -69,7 +69,7 @@ type BigQueryDataTransferConfigSpec struct { // Pub/Sub topic where notifications will be sent after transfer runs // associated with this transfer config finish. - PubSubTopicRef refv1beta1.PubSubTopicRef `json:"pubSubTopicRef,omitempty"` + PubSubTopicRef *refv1beta1.PubSubTopicRef `json:"pubSubTopicRef,omitempty"` // Parameters specific to each data source. For more information see the // bq tab in the 'Setting up a data transfer' section for each data source. diff --git a/apis/bigquerydatatransfer/v1alpha1/zz_generated.deepcopy.go b/apis/bigquerydatatransfer/v1alpha1/zz_generated.deepcopy.go index a864183af8..53d726a785 100644 --- a/apis/bigquerydatatransfer/v1alpha1/zz_generated.deepcopy.go +++ b/apis/bigquerydatatransfer/v1alpha1/zz_generated.deepcopy.go @@ -127,7 +127,11 @@ func (in *BigQueryDataTransferConfigSpec) DeepCopyInto(out *BigQueryDataTransfer *out = new(string) **out = **in } - out.DatasetRef = in.DatasetRef + if in.DatasetRef != nil { + in, out := &in.DatasetRef, &out.DatasetRef + *out = new(v1beta1.BigQueryDatasetRef) + **out = **in + } if in.Disabled != nil { in, out := &in.Disabled, &out.Disabled *out = new(bool) @@ -146,9 +150,13 @@ func (in *BigQueryDataTransferConfigSpec) DeepCopyInto(out *BigQueryDataTransfer if in.EncryptionConfiguration != nil { in, out := &in.EncryptionConfiguration, &out.EncryptionConfiguration *out = new(EncryptionConfiguration) + (*in).DeepCopyInto(*out) + } + if in.PubSubTopicRef != nil { + in, out := &in.PubSubTopicRef, &out.PubSubTopicRef + *out = new(v1beta1.PubSubTopicRef) **out = **in } - out.PubSubTopicRef = in.PubSubTopicRef if in.Params != nil { in, out := &in.Params, &out.Params *out = make(map[string]string, len(*in)) @@ -247,7 +255,11 @@ func (in *EmailPreferences) DeepCopy() *EmailPreferences { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *EncryptionConfiguration) DeepCopyInto(out *EncryptionConfiguration) { *out = *in - out.KmsKeyRef = in.KmsKeyRef + if in.KmsKeyRef != nil { + in, out := &in.KmsKeyRef, &out.KmsKeyRef + *out = new(v1beta1.KMSCryptoKeyRef) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EncryptionConfiguration. diff --git a/config/crds/resources/apiextensions.k8s.io_v1_customresourcedefinition_bigquerydatatransferconfigs.bigquerydatatransfer.cnrm.cloud.google.com.yaml b/config/crds/resources/apiextensions.k8s.io_v1_customresourcedefinition_bigquerydatatransferconfigs.bigquerydatatransfer.cnrm.cloud.google.com.yaml index 327c87eb84..cf6de8dece 100644 --- a/config/crds/resources/apiextensions.k8s.io_v1_customresourcedefinition_bigquerydatatransferconfigs.bigquerydatatransfer.cnrm.cloud.google.com.yaml +++ b/config/crds/resources/apiextensions.k8s.io_v1_customresourcedefinition_bigquerydatatransferconfigs.bigquerydatatransfer.cnrm.cloud.google.com.yaml @@ -6,9 +6,7 @@ metadata: creationTimestamp: null labels: cnrm.cloud.google.com/managed-by-kcc: "true" - cnrm.cloud.google.com/stability-level: alpha cnrm.cloud.google.com/system: "true" - cnrm.cloud.google.com/tf2crd: "true" name: bigquerydatatransferconfigs.bigquerydatatransfer.cnrm.cloud.google.com spec: group: bigquerydatatransfer.cnrm.cloud.google.com @@ -16,10 +14,8 @@ spec: categories: - gcp kind: BigQueryDataTransferConfig + listKind: BigQueryDataTransferConfigList plural: bigquerydatatransferconfigs - shortNames: - - gcpbigquerydatatransferconfig - - gcpbigquerydatatransferconfigs singular: bigquerydatatransferconfig preserveUnknownFields: false scope: Namespaced @@ -43,70 +39,139 @@ spec: name: v1alpha1 schema: openAPIV3Schema: + description: BigQueryDataTransferConfig is the Schema for the BigQueryDataTransferConfig + API properties: apiVersion: - description: 'apiVersion defines the versioned schema of this representation + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: - description: 'kind is a string value representing the REST resource this + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object spec: + description: BigQueryDataTransferConfigSpec defines the desired state + of BigQueryDataTransferConfig properties: dataRefreshWindowDays: - description: |- - The number of days to look back to automatically refresh the data. - For example, if dataRefreshWindowDays = 10, then every day BigQuery - reingests data for [today-10, today-1], rather than ingesting data for - just [today-1]. Only valid if the data source supports the feature. - Set the value to 0 to use the default value. + description: The number of days to look back to automatically refresh + the data. For example, if `data_refresh_window_days = 10`, then + every day BigQuery reingests data for [today-10, today-1], rather + than ingesting data for just [today-1]. Only valid if the data source + supports the feature. Set the value to 0 to use the default value. + format: int32 type: integer - dataSourceId: - description: Immutable. The data source id. Cannot be changed once - the transfer config is created. + dataSourceID: + description: 'Immutable. Data source ID. This cannot be changed once + data transfer is created. The full list of available data source + IDs can be returned through an API call: https://cloud.google.com/bigquery-transfer/docs/reference/datatransfer/rest/v1/projects.locations.dataSources/list' type: string - destinationDatasetId: + x-kubernetes-validations: + - message: DataSourceID field is immutable + rule: self == oldSelf + datasetRef: description: The BigQuery target dataset id. - type: string + oneOf: + - not: + required: + - external + required: + - name + - not: + anyOf: + - required: + - name + - required: + - namespace + required: + - external + properties: + external: + description: If provided must be in the format `projects/[project_id]/datasets/[dataset_id]`. + type: string + name: + description: The `metadata.name` field of a `BigQueryDataset` + resource. + type: string + namespace: + description: The `metadata.namespace` field of a `BigQueryDataset` + resource. + type: string + type: object disabled: - description: When set to true, no runs are scheduled for a given transfer. + description: Is this config disabled. When set to true, no runs will + be scheduled for this transfer config. type: boolean displayName: - description: The user specified display name for the transfer config. + description: User specified display name for the data transfer. type: string emailPreferences: - description: |- - Email notifications will be sent according to these preferences to the - email address of the user who owns this transfer config. + description: Email notifications will be sent according to these preferences + to the email address of the user who owns this transfer config. properties: enableFailureEmail: description: If true, email notifications will be sent on transfer run failures. type: boolean - required: - - enableFailureEmail + type: object + encryptionConfiguration: + description: The encryption configuration part. Currently, it is only + used for the optional KMS key name. The BigQuery service account + of your project must be granted permissions to use the key. Read + methods will return the key name applied in effect. Write methods + will apply the key if it is present, or otherwise try to apply project + default keys if it is absent. + properties: + kmsKeyRef: + description: The KMS key used for encrypting BigQuery data. + oneOf: + - not: + required: + - external + required: + - name + - not: + anyOf: + - required: + - name + - required: + - namespace + required: + - external + properties: + external: + description: A reference to an externally managed KMSCryptoKey. + Should be in the format `projects/[kms_project_id]/locations/[region]/keyRings/[key_ring_id]/cryptoKeys/[key]`. + type: string + name: + description: The `name` of a `KMSCryptoKey` resource. + type: string + namespace: + description: The `namespace` of a `KMSCryptoKey` resource. + type: string + type: object type: object location: - description: |- - Immutable. The geographic location where the transfer config should reside. - Examples: US, EU, asia-northeast1. The default value is US. - type: string - notificationPubsubTopic: - description: |- - Pub/Sub topic where notifications will be sent after transfer runs - associated with this transfer config finish. + description: Immutable. type: string + x-kubernetes-validations: + - message: Location field is immutable + rule: self == oldSelf params: additionalProperties: type: string + description: 'Parameters specific to each data source. For more information + see the bq tab in the ''Setting up a data transfer'' section for + each data source. For example the parameters for Cloud Storage transfers + are listed here: https://cloud.google.com/bigquery-transfer/docs/cloud-storage-transfer#bq' type: object projectRef: - description: The project that this resource belongs to. + description: The Project that this resource belongs to. oneOf: - not: required: @@ -123,128 +188,112 @@ spec: - external properties: external: - description: 'Allowed value: The `name` field of a `Project` resource.' + description: The `projectID` field of a project, when not managed + by KCC. + type: string + kind: + description: The kind of the Project resource; optional but must + be `Project` if provided. type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: The `name` field of a `Project` resource. type: string namespace: - description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + description: The `namespace` field of a `Project` resource. + type: string + type: object + pubSubTopicRef: + description: Pub/Sub topic where notifications will be sent after + transfer runs associated with this transfer config finish. + oneOf: + - not: + required: + - external + required: + - name + - not: + anyOf: + - required: + - name + - required: + - namespace + required: + - external + properties: + external: + description: If provided must be in the format `projects/[project_id]/topics/[topic_id]`. + type: string + name: + description: The `metadata.name` field of a `PubSubTopic` resource. + type: string + namespace: + description: The `metadata.namespace` field of a `PubSubTopic` + resource. type: string type: object resourceID: - description: Immutable. Optional. The service-generated name of the - resource. Used for acquisition only. Leave unset to create a new - resource. + description: Immutable. The BigQueryDataTransferConfig name. If not + given, the metadata.name will be used. type: string + x-kubernetes-validations: + - message: ResourceID field is immutable + rule: self == oldSelf schedule: description: |- - Data transfer schedule. If the data source does not support a custom - schedule, this should be empty. If it is empty, the default value for - the data source will be used. The specified times are in UTC. Examples - of valid format: 1st,3rd monday of month 15:30, every wed,fri of jan, - jun 13:15, and first sunday of quarter 00:00. See more explanation - about the format here: - https://cloud.google.com/appengine/docs/flexible/python/scheduling-jobs-with-cron-yaml#the_schedule_format - NOTE: the granularity should be at least 8 hours, or less frequent. + Data transfer schedule. + If the data source does not support a custom schedule, this should be + empty. If it is empty, the default value for the data source will be used. + The specified times are in UTC. + Examples of valid format: + `1st,3rd monday of month 15:30`, + `every wed,fri of jan,jun 13:15`, and + `first sunday of quarter 00:00`. + See more explanation about the format here: + https://cloud.google.com/appengine/docs/flexible/python/scheduling-jobs-with-cron-yaml#the_schedule_format + + NOTE: The minimum interval time between recurring transfers depends on the + data source; refer to the documentation for your data source. type: string scheduleOptions: description: Options customizing the data transfer schedule. properties: disableAutoScheduling: - description: |- - If true, automatic scheduling of data transfer runs for this - configuration will be disabled. The runs can be started on ad-hoc - basis using transferConfigs.startManualRuns API. When automatic + description: If true, automatic scheduling of data transfer runs + for this configuration will be disabled. The runs can be started + on ad-hoc basis using StartManualTransferRuns API. When automatic scheduling is disabled, the TransferConfig.schedule field will be ignored. type: boolean endTime: - description: |- - Defines time to stop scheduling transfer runs. A transfer run cannot be - scheduled at or after the end time. The end time can be changed at any - moment. The time when a data transfer can be triggered manually is not - limited by this option. + description: Defines time to stop scheduling transfer runs. A + transfer run cannot be scheduled at or after the end time. The + end time can be changed at any moment. The time when a data + transfer can be triggered manually is not limited by this option. type: string startTime: - description: |- - Specifies time to start scheduling transfer runs. The first run will be - scheduled at or after the start time according to a recurrence pattern - defined in the schedule string. The start time can be changed at any - moment. The time when a data transfer can be triggered manually is not - limited by this option. + description: Specifies time to start scheduling transfer runs. + The first run will be scheduled at or after the start time according + to a recurrence pattern defined in the schedule string. The + start time can be changed at any moment. The time when a data + transfer can be triggered manually is not limited by this option. type: string type: object - sensitiveParams: - description: |- - Different parameters are configured primarily using the the 'params' field on this - resource. This block contains the parameters which contain secrets or passwords so that they can be marked - sensitive and hidden from plan output. The name of the field, eg: secret_access_key, will be the key - in the 'params' map in the api request. - - Credentials may not be specified in both locations and will cause an error. Changing from one location - to a different credential configuration in the config will require an apply to update state. - properties: - secretAccessKey: - description: The Secret Access Key of the AWS account transferring - data from. - oneOf: - - not: - required: - - valueFrom - required: - - value - - not: - required: - - value - required: - - valueFrom - properties: - value: - description: Value of the field. Cannot be used if 'valueFrom' - is specified. - type: string - valueFrom: - description: Source for the field's value. Cannot be used - if 'value' is specified. - properties: - secretKeyRef: - description: Reference to a value with the given key in - the given Secret in the resource's namespace. - properties: - key: - description: Key that identifies the value to be extracted. - type: string - name: - description: Name of the Secret to extract a value - from. - type: string - required: - - name - - key - type: object - type: object - type: object - required: - - secretAccessKey - type: object - serviceAccountName: - description: |- - Service account email. If this field is set, transfer config will - be created with this service account credentials. It requires that - requesting user calling this API has permissions to act as this service account. - type: string + userID: + description: Deprecated. Unique ID of the user on whose behalf transfer + is done. + format: int64 + type: integer required: - - dataSourceId - - displayName - - params + - location - projectRef type: object status: + description: BigQueryDataTransferConfigStatus defines the config connector + machine state of BigQueryDataTransferConfig properties: conditions: - description: Conditions represent the latest available observation - of the resource's current state. + description: Conditions represent the latest available observations + of the object's current state. items: properties: lastTransitionTime: @@ -268,13 +317,9 @@ spec: type: string type: object type: array - name: - description: |- - The resource name of the transfer config. Transfer config names have the - form projects/{projectId}/locations/{location}/transferConfigs/{configId} - or projects/{projectId}/transferConfigs/{configId}, - where configId is usually a uuid, but this is not required. - The name is ignored when creating a transfer config. + externalRef: + description: A unique specifier for the BigQueryDataTransferConfig + resource in GCP. type: string observedGeneration: description: ObservedGeneration is the generation of the resource @@ -282,7 +327,45 @@ spec: If this is equal to metadata.generation, then that means that the current reported status reflects the most recent desired state of the resource. + format: int64 type: integer + observedState: + description: ObservedState is the state of the resource as most recently + observed in GCP. + properties: + datasetRegion: + description: Output only. Region in which BigQuery dataset is + located. + type: string + name: + description: Identifier. The resource name of the transfer config. + Transfer config names have the form either `projects/{project_id}/locations/{region}/transferConfigs/{config_id}` + or `projects/{project_id}/transferConfigs/{config_id}`, where + `config_id` is usually a UUID, even though it is not guaranteed + or required. The name is ignored when creating a transfer config. + type: string + nextRunTime: + description: Output only. Next time when data transfer will run. + type: string + ownerInfo: + description: Output only. Information about the user whose credentials + are used to transfer data. Populated only for `transferConfigs.get` + requests. In case the user information is not available, this + field will not be populated. + properties: + email: + description: E-mail address of the user. + type: string + type: object + state: + description: Output only. State of the most recently updated transfer + run. + type: string + updateTime: + description: Output only. Data transfer modification time. Ignored + by server on input. + type: string + type: object type: object required: - spec @@ -291,9 +374,3 @@ spec: storage: true subresources: status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/dev/tasks/generate-crds b/dev/tasks/generate-crds index 6a058010e9..5a5881d62a 100755 --- a/dev/tasks/generate-crds +++ b/dev/tasks/generate-crds @@ -55,7 +55,7 @@ go run ./scripts/crd-tools reflow-descriptions --dir apis/config/crd/ # excluded_resources are resources that are under development for a direct conversion # we don't modify the CRD just yet for those but will in the future -excluded_resources=("bigquerydatatransferconfig" "securesourcemanagerinstance") +excluded_resources=("securesourcemanagerinstance") cd ${REPO_ROOT} cd apis/config/crd/ diff --git a/tests/apichecks/testdata/exceptions/acronyms.txt b/tests/apichecks/testdata/exceptions/acronyms.txt index 73cbf308a6..a39298b9b1 100644 --- a/tests/apichecks/testdata/exceptions/acronyms.txt +++ b/tests/apichecks/testdata/exceptions/acronyms.txt @@ -51,8 +51,6 @@ [acronyms] crd=bigquerydatasets.bigquery.cnrm.cloud.google.com version=v1beta1: field ".spec.access[].view.datasetId" should be ".spec.access[].view.datasetID" [acronyms] crd=bigquerydatasets.bigquery.cnrm.cloud.google.com version=v1beta1: field ".spec.access[].view.projectId" should be ".spec.access[].view.projectID" [acronyms] crd=bigquerydatasets.bigquery.cnrm.cloud.google.com version=v1beta1: field ".spec.access[].view.tableId" should be ".spec.access[].view.tableID" -[acronyms] crd=bigquerydatatransferconfigs.bigquerydatatransfer.cnrm.cloud.google.com version=v1alpha1: field ".spec.dataSourceId" should be ".spec.dataSourceID" -[acronyms] crd=bigquerydatatransferconfigs.bigquerydatatransfer.cnrm.cloud.google.com version=v1alpha1: field ".spec.destinationDatasetId" should be ".spec.destinationDatasetID" [acronyms] crd=bigquerytables.bigquery.cnrm.cloud.google.com version=v1beta1: field ".spec.externalDataConfiguration.connectionId" should be ".spec.externalDataConfiguration.connectionID" [acronyms] crd=bigquerytables.bigquery.cnrm.cloud.google.com version=v1beta1: field ".spec.tableConstraints.foreignKeys[].referencedTable.datasetId" should be ".spec.tableConstraints.foreignKeys[].referencedTable.datasetID" [acronyms] crd=bigquerytables.bigquery.cnrm.cloud.google.com version=v1beta1: field ".spec.tableConstraints.foreignKeys[].referencedTable.projectId" should be ".spec.tableConstraints.foreignKeys[].referencedTable.projectID" From 57701b9603bf44c057590a4f1443c8d07e69ab4c Mon Sep 17 00:00:00 2001 From: Jingyi Hu Date: Thu, 12 Sep 2024 06:58:34 +0000 Subject: [PATCH 02/12] fix: guard against nil pointer in mappings --- .../bigquerydatatransfer_mappings.go | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/pkg/controller/direct/bigquerydatatransfer/bigquerydatatransfer_mappings.go b/pkg/controller/direct/bigquerydatatransfer/bigquerydatatransfer_mappings.go index 60b72dc612..08e554fc33 100644 --- a/pkg/controller/direct/bigquerydatatransfer/bigquerydatatransfer_mappings.go +++ b/pkg/controller/direct/bigquerydatatransfer/bigquerydatatransfer_mappings.go @@ -54,7 +54,9 @@ func BigQueryDataTransferConfigSpec_FromProto(mapCtx *direct.MapContext, in *pb. return nil } out := &krm.BigQueryDataTransferConfigSpec{} - out.DatasetRef = refv1beta1.BigQueryDatasetRef{External: in.GetDestinationDatasetId()} + if in.GetDestinationDatasetId() != "" { + out.DatasetRef = &refv1beta1.BigQueryDatasetRef{External: in.GetDestinationDatasetId()} + } out.DisplayName = direct.LazyPtr(in.GetDisplayName()) out.DataSourceID = direct.LazyPtr(in.GetDataSourceId()) out.Params = Params_FromProto(mapCtx, in.GetParams()) @@ -63,7 +65,9 @@ func BigQueryDataTransferConfigSpec_FromProto(mapCtx *direct.MapContext, in *pb. out.DataRefreshWindowDays = direct.LazyPtr(in.GetDataRefreshWindowDays()) out.Disabled = direct.LazyPtr(in.GetDisabled()) out.UserID = direct.LazyPtr(in.GetUserId()) - out.PubSubTopicRef = refv1beta1.PubSubTopicRef{External: in.GetNotificationPubsubTopic()} + if in.GetNotificationPubsubTopic() != "" { + out.PubSubTopicRef = &refv1beta1.PubSubTopicRef{External: in.GetNotificationPubsubTopic()} + } out.EmailPreferences = EmailPreferences_FromProto(mapCtx, in.GetEmailPreferences()) out.EncryptionConfiguration = EncryptionConfiguration_FromProto(mapCtx, in.GetEncryptionConfiguration()) return out @@ -73,7 +77,9 @@ func BigQueryDataTransferConfigSpec_ToProto(mapCtx *direct.MapContext, in *krm.B return nil } out := &pb.TransferConfig{} - out.Destination = &pb.TransferConfig_DestinationDatasetId{DestinationDatasetId: in.DatasetRef.External} + if in.DatasetRef != nil { + out.Destination = &pb.TransferConfig_DestinationDatasetId{DestinationDatasetId: in.DatasetRef.External} + } out.DisplayName = direct.ValueOf(in.DisplayName) out.DataSourceId = direct.ValueOf(in.DataSourceID) out.Params = Params_ToProto(mapCtx, in.Params) @@ -82,7 +88,9 @@ func BigQueryDataTransferConfigSpec_ToProto(mapCtx *direct.MapContext, in *krm.B out.DataRefreshWindowDays = direct.ValueOf(in.DataRefreshWindowDays) out.Disabled = direct.ValueOf(in.Disabled) out.UserId = direct.ValueOf(in.UserID) - out.NotificationPubsubTopic = in.PubSubTopicRef.External + if in.PubSubTopicRef != nil { + out.NotificationPubsubTopic = in.PubSubTopicRef.External + } out.EmailPreferences = EmailPreferences_ToProto(mapCtx, in.EmailPreferences) out.EncryptionConfiguration = EncryptionConfiguration_ToProto(mapCtx, in.EncryptionConfiguration) return out @@ -92,7 +100,9 @@ func EncryptionConfiguration_FromProto(mapCtx *direct.MapContext, in *pb.Encrypt return nil } out := &krm.EncryptionConfiguration{} - out.KmsKeyRef = refv1beta1.KMSCryptoKeyRef{External: in.GetKmsKeyName().String()} + if in.GetKmsKeyName() != nil { + out.KmsKeyRef = &refv1beta1.KMSCryptoKeyRef{External: in.GetKmsKeyName().String()} + } return out } func EncryptionConfiguration_ToProto(mapCtx *direct.MapContext, in *krm.EncryptionConfiguration) *pb.EncryptionConfiguration { @@ -100,7 +110,9 @@ func EncryptionConfiguration_ToProto(mapCtx *direct.MapContext, in *krm.Encrypti return nil } out := &pb.EncryptionConfiguration{} - out.KmsKeyName = &wrapperspb.StringValue{Value: in.KmsKeyRef.External} + if in.KmsKeyRef != nil { + out.KmsKeyName = &wrapperspb.StringValue{Value: in.KmsKeyRef.External} + } return out } func Params_FromProto(mapCtx *direct.MapContext, in *structpb.Struct) map[string]string { From a2d2ea3490d6cab3f3d1b01708396344d6cc039e Mon Sep 17 00:00:00 2001 From: Jingyi Hu Date: Fri, 13 Sep 2024 22:33:24 +0000 Subject: [PATCH 03/12] chore: mark additional required fields in BigQueryDataTransferConfig --- .../v1alpha1/bigquerydatatransferconfig_types.go | 3 +++ ...erconfigs.bigquerydatatransfer.cnrm.cloud.google.com.yaml | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/apis/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig_types.go b/apis/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig_types.go index a0b59f0566..01053b7f58 100644 --- a/apis/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig_types.go +++ b/apis/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig_types.go @@ -44,9 +44,11 @@ type BigQueryDataTransferConfigSpec struct { // Data source ID. This cannot be changed once data transfer is created. The // full list of available data source IDs can be returned through an API call: // https://cloud.google.com/bigquery-transfer/docs/reference/datatransfer/rest/v1/projects.locations.dataSources/list + // +required DataSourceID *string `json:"dataSourceID,omitempty"` // The BigQuery target dataset id. + // +required DatasetRef *refv1beta1.BigQueryDatasetRef `json:"datasetRef,omitempty"` // Is this config disabled. When set to true, no runs will be scheduled for @@ -75,6 +77,7 @@ type BigQueryDataTransferConfigSpec struct { // bq tab in the 'Setting up a data transfer' section for each data source. // For example the parameters for Cloud Storage transfers are listed here: // https://cloud.google.com/bigquery-transfer/docs/cloud-storage-transfer#bq + // +required Params map[string]string `json:"params,omitempty"` Parent `json:",inline"` diff --git a/config/crds/resources/apiextensions.k8s.io_v1_customresourcedefinition_bigquerydatatransferconfigs.bigquerydatatransfer.cnrm.cloud.google.com.yaml b/config/crds/resources/apiextensions.k8s.io_v1_customresourcedefinition_bigquerydatatransferconfigs.bigquerydatatransfer.cnrm.cloud.google.com.yaml index cf6de8dece..521b989cbd 100644 --- a/config/crds/resources/apiextensions.k8s.io_v1_customresourcedefinition_bigquerydatatransferconfigs.bigquerydatatransfer.cnrm.cloud.google.com.yaml +++ b/config/crds/resources/apiextensions.k8s.io_v1_customresourcedefinition_bigquerydatatransferconfigs.bigquerydatatransfer.cnrm.cloud.google.com.yaml @@ -189,7 +189,7 @@ spec: properties: external: description: The `projectID` field of a project, when not managed - by KCC. + by Config Connector. type: string kind: description: The kind of the Project resource; optional but must @@ -284,7 +284,10 @@ spec: format: int64 type: integer required: + - dataSourceID + - datasetRef - location + - params - projectRef type: object status: From 31530234af7704af5c1899ac4ef423dc86df9c4b Mon Sep 17 00:00:00 2001 From: Jingyi Hu Date: Fri, 13 Sep 2024 22:40:32 +0000 Subject: [PATCH 04/12] feat: implement direct controller for BigQueryDataTransferConfig --- apis/refs/v1beta1/bigqueryref.go | 91 +++- apis/refs/v1beta1/pubsubref.go | 80 ++++ .../bigquerydatatransferconfig_controller.go | 394 ++++++++++++++++++ ...uerydatatransferconfig_externalresource.go | 96 +++++ pkg/controller/direct/register/register.go | 1 + 5 files changed, 659 insertions(+), 3 deletions(-) create mode 100644 pkg/controller/direct/bigquerydatatransfer/bigquerydatatransferconfig_controller.go create mode 100644 pkg/controller/direct/bigquerydatatransfer/bigquerydatatransferconfig_externalresource.go diff --git a/apis/refs/v1beta1/bigqueryref.go b/apis/refs/v1beta1/bigqueryref.go index 6565a7f0df..006c79daf8 100644 --- a/apis/refs/v1beta1/bigqueryref.go +++ b/apis/refs/v1beta1/bigqueryref.go @@ -14,11 +14,96 @@ package v1beta1 +import ( + "context" + "fmt" + "strings" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" +) + type BigQueryDatasetRef struct { - // If provided must be in the format `bigquery.googleapis.com/projects/[project_id]/datasets/[dataset_id]`. + // If provided must be in the format `projects/[project_id]/datasets/[dataset_id]`. External string `json:"external,omitempty"` - // The `metadata.name` field of a `PubSubTopic` resource. + // The `metadata.name` field of a `BigQueryDataset` resource. Name string `json:"name,omitempty"` - // The `metadata.namespace` field of a `PubSubTopic` resource. + // The `metadata.namespace` field of a `BigQueryDataset` resource. Namespace string `json:"namespace,omitempty"` } + +type BigQueryDataset struct { + projectID string + datasetID string +} + +func ResolveBigQueryDataset(ctx context.Context, reader client.Reader, src client.Object, ref *BigQueryDatasetRef) (*BigQueryDataset, error) { + if ref == nil { + return nil, nil + } + + if ref.Name == "" && ref.External == "" { + return nil, fmt.Errorf("must specify either name or external on BigQueryDatasetRef") + } + if ref.Name != "" && ref.External != "" { + return nil, fmt.Errorf("cannot specify both name and external on BigQueryDatasetRef") + } + + // External is provided. + if ref.External != "" { + // External should be in the `projects/[project_id]/datasets/[dataset_id]` format. + tokens := strings.Split(ref.External, "/") + if len(tokens) == 4 && tokens[0] == "projects" && tokens[2] == "datasets" { + return &BigQueryDataset{ + projectID: tokens[1], + datasetID: tokens[3], + }, nil + } + return nil, fmt.Errorf("format of BigQueryDatasetRef external=%q was not known (use projects/[project_id]/datasets/[dataset_id])", ref.External) + + } + + // Fetch BigQueryDataset object to construct the external form. + dataset := &unstructured.Unstructured{} + dataset.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "bigquery.cnrm.cloud.google.com", + Version: "v1beta1", + Kind: "BigQueryDataset", + }) + nn := types.NamespacedName{ + Namespace: ref.Namespace, + Name: ref.Name, + } + if nn.Namespace == "" { + nn.Namespace = src.GetNamespace() + } + if err := reader.Get(ctx, nn, dataset); err != nil { + if apierrors.IsNotFound(err) { + return nil, fmt.Errorf("referenced BigQueryDataset %v not found", nn) + } + return nil, fmt.Errorf("error reading referenced BigQueryDataset %v: %w", nn, err) + } + projectID, err := ResolveProjectID(ctx, reader, dataset) + if err != nil { + return nil, err + } + datasetID, err := GetResourceID(dataset) + if err != nil { + return nil, err + } + return &BigQueryDataset{ + projectID: projectID, + datasetID: datasetID, + }, nil +} + +func (d *BigQueryDataset) String() string { + return fmt.Sprintf("projects/%s/datasets/%s", d.projectID, d.datasetID) +} + +func (d *BigQueryDataset) GetDatasetID() string { + return d.datasetID +} diff --git a/apis/refs/v1beta1/pubsubref.go b/apis/refs/v1beta1/pubsubref.go index f2f495e334..5995255325 100644 --- a/apis/refs/v1beta1/pubsubref.go +++ b/apis/refs/v1beta1/pubsubref.go @@ -14,6 +14,18 @@ package v1beta1 +import ( + "context" + "fmt" + "strings" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" +) + type PubSubTopicRef struct { // If provided must be in the format `projects/[project_id]/topics/[topic_id]`. External string `json:"external,omitempty"` @@ -22,3 +34,71 @@ type PubSubTopicRef struct { // The `metadata.namespace` field of a `PubSubTopic` resource. Namespace string `json:"namespace,omitempty"` } + +type PubSubTopic struct { + projectID string + topicID string +} + +func ResolvePubSubTopic(ctx context.Context, reader client.Reader, src client.Object, ref *PubSubTopicRef) (*PubSubTopic, error) { + if ref == nil { + return nil, nil + } + + if ref.Name == "" && ref.External == "" { + return nil, fmt.Errorf("must specify either name or external on PubSubTopicRef") + } + if ref.Name != "" && ref.External != "" { + return nil, fmt.Errorf("cannot specify both name and external on PubSubTopicRef") + } + + // External is provided. + if ref.External != "" { + // External should be in the `projects/[project_id]/topics/[topic_id]` format. + tokens := strings.Split(ref.External, "/") + if len(tokens) == 4 && tokens[0] == "projects" && tokens[2] == "topics" { + return &PubSubTopic{ + projectID: tokens[1], + topicID: tokens[3], + }, nil + } + return nil, fmt.Errorf("format of PubSubTopicRef external=%q was not known (use projects/[project_id]/topics/[topic_id])", ref.External) + } + + // Fetch PubSubTopic object to construct the external form. + pubSubTopic := &unstructured.Unstructured{} + pubSubTopic.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "pubsub.cnrm.cloud.google.com", + Version: "v1beta1", + Kind: "PubSubTopic", + }) + nn := types.NamespacedName{ + Namespace: ref.Namespace, + Name: ref.Name, + } + if nn.Namespace == "" { + nn.Namespace = src.GetNamespace() + } + if err := reader.Get(ctx, nn, pubSubTopic); err != nil { + if apierrors.IsNotFound(err) { + return nil, fmt.Errorf("referenced PubSubTopic %v not found", nn) + } + return nil, fmt.Errorf("error reading referenced PubSubTopic %v: %w", nn, err) + } + projectID, err := ResolveProjectID(ctx, reader, pubSubTopic) + if err != nil { + return nil, err + } + topicID, err := GetResourceID(pubSubTopic) + if err != nil { + return nil, err + } + return &PubSubTopic{ + projectID: projectID, + topicID: topicID, + }, nil +} + +func (t *PubSubTopic) String() string { + return fmt.Sprintf("projects/%s/topics/%s", t.projectID, t.topicID) +} diff --git a/pkg/controller/direct/bigquerydatatransfer/bigquerydatatransferconfig_controller.go b/pkg/controller/direct/bigquerydatatransfer/bigquerydatatransferconfig_controller.go new file mode 100644 index 0000000000..f488d316eb --- /dev/null +++ b/pkg/controller/direct/bigquerydatatransfer/bigquerydatatransferconfig_controller.go @@ -0,0 +1,394 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bigquerydatatransfer + +import ( + "context" + "fmt" + "reflect" + + krm "github.com/GoogleCloudPlatform/k8s-config-connector/apis/bigquerydatatransfer/v1alpha1" + refv1beta1 "github.com/GoogleCloudPlatform/k8s-config-connector/apis/refs/v1beta1" + "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/config" + "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/direct" + "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/direct/directbase" + "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/direct/registry" + + gcp "cloud.google.com/go/bigquery/datatransfer/apiv1" + bigquerydatatransferpb "cloud.google.com/go/bigquery/datatransfer/apiv1/datatransferpb" + "google.golang.org/api/option" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/fieldmaskpb" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + ctrlName = "bigquerydatatransfer-controller" + serviceDomain = "//bigquerydatatransfer.googleapis.com" +) + +func init() { + registry.RegisterModel(krm.BigQueryDataTransferConfigGVK, NewModel) +} + +func NewModel(ctx context.Context, config *config.ControllerConfig) (directbase.Model, error) { + return &model{config: *config}, nil +} + +var _ directbase.Model = &model{} + +type model struct { + config config.ControllerConfig +} + +func (m *model) client(ctx context.Context) (*gcp.Client, error) { + var opts []option.ClientOption + if m.config.UserAgent != "" { + opts = append(opts, option.WithUserAgent(m.config.UserAgent)) + } + if m.config.HTTPClient != nil { + opts = append(opts, option.WithHTTPClient(m.config.HTTPClient)) + } + if m.config.UserProjectOverride && m.config.BillingProject != "" { + opts = append(opts, option.WithQuotaProject(m.config.BillingProject)) + } + + gcpClient, err := gcp.NewRESTClient(ctx, opts...) + if err != nil { + return nil, fmt.Errorf("building bigquerydatatransfer client: %w", err) + } + return gcpClient, err +} + +func (m *model) AdapterForObject(ctx context.Context, reader client.Reader, u *unstructured.Unstructured) (directbase.Adapter, error) { + obj := &krm.BigQueryDataTransferConfig{} + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(u.Object, &obj); err != nil { + return nil, fmt.Errorf("error converting to %T: %w", obj, err) + } + + // Get ResourceID + resourceID := direct.ValueOf(obj.Spec.ResourceID) + if resourceID == "" { + serviceGeneratedID, err := parseServiceGeneratedIDFromExternalRef(obj) + if err != nil { + return nil, err + } + resourceID = serviceGeneratedID + } + + location := obj.Spec.Location + if location == "" { + return nil, fmt.Errorf("cannot resolve location") + } + + // Resolve PubSubTopic Ref + if obj.Spec.PubSubTopicRef != nil { + topic, err := refv1beta1.ResolvePubSubTopic(ctx, reader, obj, obj.Spec.PubSubTopicRef) + if err != nil { + return nil, err + } + obj.Spec.PubSubTopicRef.External = topic.String() + } + + // Resolve BigQueryDataSet Ref + if obj.Spec.DatasetRef != nil { + dataset, err := refv1beta1.ResolveBigQueryDataset(ctx, reader, obj, obj.Spec.DatasetRef) + if err != nil { + return nil, err + } + obj.Spec.DatasetRef.External = dataset.GetDatasetID() // the GCP API only takes datasetID + } + + // Resolve KMSCryptoKey Ref + if obj.Spec.EncryptionConfiguration != nil { + kmsCryptoKeyRef, err := refv1beta1.ResolveKMSCryptoKeyRef(ctx, reader, obj, obj.Spec.EncryptionConfiguration.KmsKeyRef) + if err != nil { + return nil, err + } + obj.Spec.EncryptionConfiguration.KmsKeyRef = kmsCryptoKeyRef + } + + // Resolve Project Ref + projectRef, err := refv1beta1.ResolveProject(ctx, reader, obj, obj.Spec.ProjectRef) + if err != nil { + return nil, err + } + projectID := projectRef.ProjectID + if projectID == "" { + return nil, fmt.Errorf("cannot resolve project") + } + + var id *BigQueryDataTransferConfigIdentity + externalRef := direct.ValueOf(obj.Status.ExternalRef) + if externalRef == "" { + id = BuildID(projectID, location, resourceID) + } else { + id, err = asID(externalRef) + if err != nil { + return nil, err + } + + if id.projectID != projectID { + return nil, fmt.Errorf("BigQueryDataTransferConfig %s/%s has spec.projectRef changed, expect %s, got %s", + u.GetNamespace(), u.GetName(), id.projectID, projectID) + } + if id.location != location { + return nil, fmt.Errorf("BigQueryDataTransferConfig %s/%s has spec.location changed, expect %s, got %s", + u.GetNamespace(), u.GetName(), id.location, location) + } + if id.transferConfigID != resourceID { + return nil, fmt.Errorf("BigQueryDataTransferConfig %s/%s spec.resourceID changed or does not match the service generated ID, expect %s, got %s", + u.GetNamespace(), u.GetName(), id.transferConfigID, resourceID) + } + } + + // Get bigquerydatatransfer GCP client + gcpClient, err := m.client(ctx) + if err != nil { + return nil, err + } + return &Adapter{ + id: id, + gcpClient: gcpClient, + desired: obj, + }, nil +} + +func (m *model) AdapterForURL(ctx context.Context, url string) (directbase.Adapter, error) { + // TODO: Support URLs + return nil, nil +} + +type Adapter struct { + id *BigQueryDataTransferConfigIdentity + gcpClient *gcp.Client + desired *krm.BigQueryDataTransferConfig + actual *bigquerydatatransferpb.TransferConfig +} + +var _ directbase.Adapter = &Adapter{} + +func (a *Adapter) Find(ctx context.Context) (bool, error) { + log := klog.FromContext(ctx).WithName(ctrlName) + + if a.id.transferConfigID == "" { // resource ID is not yet generated by the GCP service + return false, nil + } + + log.V(2).Info("getting BigQueryDataTransferConfig", "name", a.id.FullyQualifiedName()) + req := &bigquerydatatransferpb.GetTransferConfigRequest{Name: a.id.FullyQualifiedName()} + transferconfigpb, err := a.gcpClient.GetTransferConfig(ctx, req) + if err != nil { + if direct.IsNotFound(err) { + return false, nil + } + return false, fmt.Errorf("getting BigQueryDataTransferConfig %q: %w", a.id.FullyQualifiedName(), err) + } + + a.actual = transferconfigpb + return true, nil +} + +func (a *Adapter) Create(ctx context.Context, createOp *directbase.CreateOperation) error { + u := createOp.GetUnstructured() + + log := klog.FromContext(ctx).WithName(ctrlName) + log.V(2).Info("creating BigQueryDataTransferConfig", "name", a.id.FullyQualifiedName()) + mapCtx := &direct.MapContext{} + + projectID := a.id.projectID + if projectID == "" { + return fmt.Errorf("project is empty") + } + + desired := a.desired.DeepCopy() + resource := BigQueryDataTransferConfigSpec_ToProto(mapCtx, &desired.Spec) + if mapCtx.Err() != nil { + return mapCtx.Err() + } + + req := &bigquerydatatransferpb.CreateTransferConfigRequest{ + Parent: a.id.Parent(), + TransferConfig: resource, + } + created, err := a.gcpClient.CreateTransferConfig(ctx, req) + if err != nil { + return fmt.Errorf("creating BigQueryDataTransferConfig %s: %w", a.id.FullyQualifiedName(), err) + } + log.V(2).Info("successfully created BigQueryDataTransferConfig", "name", a.id.FullyQualifiedName()) + + status := &krm.BigQueryDataTransferConfigStatus{} + status.ObservedState = BigQueryDataTransferConfigObservedState_FromProto(mapCtx, created) + if mapCtx.Err() != nil { + return mapCtx.Err() + } + // The UUID is service generated. e.g. "projects/{project_id}/locations/{region}/transferConfigs/{config_id}" + serviceGeneratedID, err := parseServiceGeneratedIDFromName(created.Name) + if err != nil { + return fmt.Errorf("error converting %s to BigQueryDataTransferConfigIdentity: %w", created.Name, err) + } + a.id.transferConfigID = serviceGeneratedID + status.ExternalRef = a.id.AsExternalRef() + + return setStatus(u, status) +} + +func (a *Adapter) Update(ctx context.Context, updateOp *directbase.UpdateOperation) error { + u := updateOp.GetUnstructured() + + log := klog.FromContext(ctx).WithName(ctrlName) + log.V(2).Info("updating BigQueryDataTransferConfig", "name", a.id.FullyQualifiedName()) + mapCtx := &direct.MapContext{} + + // Convert KRM object to proto message + desiredKRM := a.desired.DeepCopy() + desired := BigQueryDataTransferConfigSpec_ToProto(mapCtx, &desiredKRM.Spec) + if mapCtx.Err() != nil { + return mapCtx.Err() + } + actual := a.actual + resource := proto.Clone(a.actual).(*bigquerydatatransferpb.TransferConfig) // this is the proto resource we are passing to GCP API update call. + + // Check for immutable fields + if !reflect.DeepEqual(desired.DataSourceId, a.actual.DataSourceId) { + return fmt.Errorf("BigQueryDataTransferConfig %s/%s data source ID cannot be changed", u.GetNamespace(), u.GetName()) + } + if !reflect.DeepEqual(desired.Destination, a.actual.Destination) { + return fmt.Errorf("BigQueryDataTransferConfig %s/%s destination dataset cannot be changed", u.GetNamespace(), u.GetName()) + } + + // Find diff + updateMask := &fieldmaskpb.FieldMask{} + if !reflect.DeepEqual(desired.DataRefreshWindowDays, actual.DataRefreshWindowDays) { + resource.DataRefreshWindowDays = desired.DataRefreshWindowDays + updateMask.Paths = append(updateMask.Paths, "data_refresh_window_days") + } + if !reflect.DeepEqual(desired.Disabled, actual.Disabled) { + resource.Disabled = desired.Disabled + updateMask.Paths = append(updateMask.Paths, "disabled") + } + if !reflect.DeepEqual(desired.DisplayName, actual.DisplayName) { + resource.DisplayName = desired.DisplayName + updateMask.Paths = append(updateMask.Paths, "display_name") + } + if desired.EmailPreferences != nil && !reflect.DeepEqual(desired.EmailPreferences, actual.EmailPreferences) { + resource.EmailPreferences = desired.EmailPreferences + updateMask.Paths = append(updateMask.Paths, "email_preferences") + } + if desired.EncryptionConfiguration != nil && !reflect.DeepEqual(desired.EncryptionConfiguration, actual.EncryptionConfiguration) { + resource.EncryptionConfiguration = desired.EncryptionConfiguration + updateMask.Paths = append(updateMask.Paths, "encryption_configuration") + } + if !reflect.DeepEqual(desired.NotificationPubsubTopic, actual.NotificationPubsubTopic) { + resource.NotificationPubsubTopic = desired.NotificationPubsubTopic + updateMask.Paths = append(updateMask.Paths, "notification_pubsub_topic") + } + if desired.Params != nil && !reflect.DeepEqual(desired.Params, actual.Params) { + resource.Params = desired.Params + updateMask.Paths = append(updateMask.Paths, "params") + } + if !reflect.DeepEqual(desired.Schedule, actual.Schedule) { + resource.Schedule = desired.Schedule + updateMask.Paths = append(updateMask.Paths, "schedule") + } + if desired.ScheduleOptions != nil && !reflect.DeepEqual(desired.ScheduleOptions, actual.ScheduleOptions) { + resource.ScheduleOptions = desired.ScheduleOptions + updateMask.Paths = append(updateMask.Paths, "schedule_options") + } + + if len(updateMask.Paths) == 0 { + return nil + } + + resource.Name = a.id.FullyQualifiedName() // need to pass service generated ID to GCP API + req := &bigquerydatatransferpb.UpdateTransferConfigRequest{ + TransferConfig: resource, + UpdateMask: updateMask, + } + updated, err := a.gcpClient.UpdateTransferConfig(ctx, req) + if err != nil { + return fmt.Errorf("updating BigQueryDataTransferConfig %s: %w", a.id.FullyQualifiedName(), err) + } + log.V(2).Info("successfully updated BigQueryDataTransferConfig", "name", a.id.FullyQualifiedName()) + + status := &krm.BigQueryDataTransferConfigStatus{} + status.ObservedState = BigQueryDataTransferConfigObservedState_FromProto(mapCtx, updated) + if mapCtx.Err() != nil { + return mapCtx.Err() + } + return setStatus(u, status) +} + +func (a *Adapter) Export(ctx context.Context) (*unstructured.Unstructured, error) { + if a.actual == nil { + return nil, fmt.Errorf("Find() not called") + } + u := &unstructured.Unstructured{} + + obj := &krm.BigQueryDataTransferConfig{} + mapCtx := &direct.MapContext{} + obj.Spec = *BigQueryDataTransferConfigSpec_FromProto(mapCtx, a.actual) + if mapCtx.Err() != nil { + return nil, mapCtx.Err() + } + + obj.Spec.ProjectRef = &refv1beta1.ProjectRef{Name: a.id.projectID} + obj.Spec.Location = a.id.location + uObj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(obj) + if err != nil { + return nil, err + } + u.Object = uObj + return u, nil +} + +// Delete implements the Adapter interface. +func (a *Adapter) Delete(ctx context.Context, deleteOp *directbase.DeleteOperation) (bool, error) { + log := klog.FromContext(ctx).WithName(ctrlName) + log.V(2).Info("deleting BigQueryDataTransferConfig", "name", a.id.FullyQualifiedName()) + + if a.id.transferConfigID == "" { + return false, nil + } + req := &bigquerydatatransferpb.DeleteTransferConfigRequest{Name: a.id.FullyQualifiedName()} + err := a.gcpClient.DeleteTransferConfig(ctx, req) + if err != nil { + return false, fmt.Errorf("deleting BigQueryDataTransferConfig %s: %w", a.id.FullyQualifiedName(), err) + } + log.V(2).Info("successfully deleted BigQueryDataTransferConfig", "name", a.id.FullyQualifiedName()) + + return true, nil +} + +func setStatus(u *unstructured.Unstructured, typedStatus any) error { + status, err := runtime.DefaultUnstructuredConverter.ToUnstructured(typedStatus) + if err != nil { + return fmt.Errorf("error converting status to unstructured: %w", err) + } + + old, _, _ := unstructured.NestedMap(u.Object, "status") + if old != nil { + status["conditions"] = old["conditions"] + status["observedGeneration"] = old["observedGeneration"] + status["externalRef"] = old["externalRef"] + } + + u.Object["status"] = status + + return nil +} diff --git a/pkg/controller/direct/bigquerydatatransfer/bigquerydatatransferconfig_externalresource.go b/pkg/controller/direct/bigquerydatatransfer/bigquerydatatransferconfig_externalresource.go new file mode 100644 index 0000000000..1df1fadf2d --- /dev/null +++ b/pkg/controller/direct/bigquerydatatransfer/bigquerydatatransferconfig_externalresource.go @@ -0,0 +1,96 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bigquerydatatransfer + +import ( + "fmt" + "strings" + + krm "github.com/GoogleCloudPlatform/k8s-config-connector/apis/bigquerydatatransfer/v1alpha1" + "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/direct" +) + +type BigQueryDataTransferConfigIdentity struct { + projectID string + location string + transferConfigID string +} + +// Parent builds a BigQueryDataTransferConfig parent +func (c *BigQueryDataTransferConfigIdentity) Parent() string { + return fmt.Sprintf("projects/%s/locations/%s", c.projectID, c.location) +} + +// FullyQualifiedName builds a BigQueryDataTransferConfig resource fully qualified name +func (c *BigQueryDataTransferConfigIdentity) FullyQualifiedName() string { + return fmt.Sprintf("projects/%s/locations/%s/transferConfigs/%s", c.projectID, c.location, c.transferConfigID) +} + +// AsExternalRef builds a externalRef from a BigQueryDataTransferConfig +func (c *BigQueryDataTransferConfigIdentity) AsExternalRef() *string { + e := serviceDomain + "/" + c.FullyQualifiedName() + return &e +} + +// asID builds a BigQueryDataTransferConfigIdentity from a external reference +func asID(externalRef string) (*BigQueryDataTransferConfigIdentity, error) { + if !strings.HasPrefix(externalRef, serviceDomain) { + return nil, fmt.Errorf("externalRef should have prefix %s, got %s", serviceDomain, externalRef) + } + path := strings.TrimPrefix(externalRef, serviceDomain+"/") + tokens := strings.Split(path, "/") + if len(tokens) != 6 || tokens[0] != "projects" || tokens[2] != "locations" || tokens[4] != "transferConfigs" { + return nil, fmt.Errorf("externalRef should be %s/projects//locations//transferConfigs/, got %s", + serviceDomain, externalRef) + } + return &BigQueryDataTransferConfigIdentity{ + projectID: tokens[1], + location: tokens[3], + transferConfigID: tokens[5], + }, nil +} + +// BuildID builds a unique identifier BigQueryDataTransferConfigIdentity from resource components +func BuildID(projectID, location, transferConfigID string) *BigQueryDataTransferConfigIdentity { + return &BigQueryDataTransferConfigIdentity{ + projectID: projectID, + location: location, + transferConfigID: transferConfigID, + } +} + +// parseServiceGeneratedIDFromName extracts the service generated UUID from the name field of the resource. e.g. "projects/{project_id}/locations/{region}/transferConfigs/{config_id}" +func parseServiceGeneratedIDFromName(s string) (string, error) { + tokens := strings.Split(s, "/") + if len(tokens) != 6 || tokens[0] != "projects" || tokens[2] != "locations" || tokens[4] != "transferConfigs" { + return "", fmt.Errorf("service generated name should have format projects//locations//transferConfigs/, got %s", s) + } + return tokens[5], nil +} + +// parseServiceGeneratedIDFromExternalRef extracts the service generated UUID from the externalRef. +func parseServiceGeneratedIDFromExternalRef(obj *krm.BigQueryDataTransferConfig) (string, error) { + if obj == nil || obj.Status.ExternalRef == nil { + return "", nil // it is OK to have "" resource ID prior to resource creation. + } + s := direct.ValueOf(obj.Status.ExternalRef) + s = strings.TrimPrefix(s, serviceDomain+"/") + tokens := strings.Split(s, "/") + if len(tokens) != 6 || tokens[0] != "projects" || tokens[2] != "locations" || tokens[4] != "transferConfigs" { + return "", fmt.Errorf("externalRef should be %s/projects//locations//transferConfigs/, got %s", + serviceDomain, s) + } + return tokens[5], nil +} diff --git a/pkg/controller/direct/register/register.go b/pkg/controller/direct/register/register.go index a4976b2963..69d0cd709a 100644 --- a/pkg/controller/direct/register/register.go +++ b/pkg/controller/direct/register/register.go @@ -18,6 +18,7 @@ import ( _ "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/direct/alloydb" _ "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/direct/apikeys" _ "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/direct/bigqueryconnection" + _ "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/direct/bigquerydatatransfer" _ "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/direct/cloudbuild" _ "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/direct/compute" _ "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/direct/dataflow" From 9f7bb1a8062b56b09ac9edafb4875072fd3924ed Mon Sep 17 00:00:00 2001 From: Jingyi Hu Date: Fri, 13 Sep 2024 22:43:44 +0000 Subject: [PATCH 05/12] test: add a basic test for BigQueryDataTransferConfig --- ...ydatatransferconfig-salesforce.golden.yaml | 42 ++ .../_http.log | 427 ++++++++++++++++++ .../create.yaml | 31 ++ .../dependencies.yaml | 20 + .../update.yaml | 32 ++ tests/e2e/normalize.go | 9 + tests/e2e/replacements.go | 2 + tests/e2e/unified_test.go | 18 + 8 files changed, 581 insertions(+) create mode 100644 pkg/test/resourcefixture/testdata/basic/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig/bigquerydatatransferconfig-salesforce/_generated_object_bigquerydatatransferconfig-salesforce.golden.yaml create mode 100644 pkg/test/resourcefixture/testdata/basic/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig/bigquerydatatransferconfig-salesforce/_http.log create mode 100644 pkg/test/resourcefixture/testdata/basic/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig/bigquerydatatransferconfig-salesforce/create.yaml create mode 100644 pkg/test/resourcefixture/testdata/basic/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig/bigquerydatatransferconfig-salesforce/dependencies.yaml create mode 100644 pkg/test/resourcefixture/testdata/basic/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig/bigquerydatatransferconfig-salesforce/update.yaml diff --git a/pkg/test/resourcefixture/testdata/basic/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig/bigquerydatatransferconfig-salesforce/_generated_object_bigquerydatatransferconfig-salesforce.golden.yaml b/pkg/test/resourcefixture/testdata/basic/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig/bigquerydatatransferconfig-salesforce/_generated_object_bigquerydatatransferconfig-salesforce.golden.yaml new file mode 100644 index 0000000000..6385e523c3 --- /dev/null +++ b/pkg/test/resourcefixture/testdata/basic/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig/bigquerydatatransferconfig-salesforce/_generated_object_bigquerydatatransferconfig-salesforce.golden.yaml @@ -0,0 +1,42 @@ +apiVersion: bigquerydatatransfer.cnrm.cloud.google.com/v1alpha1 +kind: BigQueryDataTransferConfig +metadata: + annotations: + cnrm.cloud.google.com/management-conflict-prevention-policy: none + finalizers: + - cnrm.cloud.google.com/finalizer + - cnrm.cloud.google.com/deletion-defender + generation: 2 + labels: + cnrm-test: "true" + name: bigquerydatatransferconfig-${uniqueId} + namespace: ${uniqueId} +spec: + dataSourceID: salesforce + datasetRef: + name: bigquerydataset${uniqueId} + disabled: true + displayName: updated example of big query data transfer config + location: us-central1 + params: + assets: asset-b + connector.authentication.oauth.clientId: client-id2 + connector.authentication.oauth.clientSecret: client-secret2 + connector.authentication.oauth.myDomain: MyDomainName2 + projectRef: + external: ${projectId} +status: + conditions: + - lastTransitionTime: "1970-01-01T00:00:00Z" + message: The resource is up to date + reason: UpToDate + status: "True" + type: Ready + externalRef: //bigquerydatatransfer.googleapis.com/projects/${projectId}/locations/us-central1/transferConfigs/${transferConfigID} + observedGeneration: 2 + observedState: + datasetRegion: us-central1 + name: projects/${projectNumber}/locations/us-central1/transferConfigs/${transferConfigID} + ownerInfo: + email: user@google.com + updateTime: "1970-01-01T00:00:00Z" diff --git a/pkg/test/resourcefixture/testdata/basic/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig/bigquerydatatransferconfig-salesforce/_http.log b/pkg/test/resourcefixture/testdata/basic/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig/bigquerydatatransferconfig-salesforce/_http.log new file mode 100644 index 0000000000..b16761ca38 --- /dev/null +++ b/pkg/test/resourcefixture/testdata/basic/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig/bigquerydatatransferconfig-salesforce/_http.log @@ -0,0 +1,427 @@ +GET https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets/${datasetID}?alt=json +Content-Type: application/json +User-Agent: Terraform/ (+https://www.terraform.io) Terraform-Plugin-SDK/2.10.1 terraform-provider-google-beta/kcc/controller-manager + +404 Not Found +Cache-Control: private +Content-Type: application/json; charset=UTF-8 +Server: ESF +Vary: Origin +Vary: X-Origin +Vary: Referer +X-Content-Type-Options: nosniff +X-Frame-Options: SAMEORIGIN +X-Xss-Protection: 0 + +{ + "error": { + "code": 404, + "errors": [ + { + "domain": "global", + "message": "Not found: Dataset ${projectId}:bigquerydataset${uniqueId}", + "reason": "notFound" + } + ], + "message": "Not found: Dataset ${projectId}:bigquerydataset${uniqueId}", + "status": "NOT_FOUND" + } +} + +--- + +POST https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets?alt=json +Content-Type: application/json +User-Agent: Terraform/ (+https://www.terraform.io) Terraform-Plugin-SDK/2.10.1 terraform-provider-google-beta/kcc/controller-manager + +{ + "datasetReference": { + "datasetId": "bigquerydataset${uniqueId}" + }, + "labels": { + "cnrm-test": "true", + "managed-by-cnrm": "true" + }, + "location": "us-central1" +} + +200 OK +Cache-Control: private +Content-Type: application/json; charset=UTF-8 +Server: ESF +Vary: Origin +Vary: X-Origin +Vary: Referer +X-Content-Type-Options: nosniff +X-Frame-Options: SAMEORIGIN +X-Xss-Protection: 0 + +{ + "access": [ + { + "role": "WRITER", + "specialGroup": "projectWriters" + }, + { + "role": "OWNER", + "specialGroup": "projectOwners" + }, + { + "role": "OWNER", + "userByEmail": "user@google.com" + }, + { + "role": "READER", + "specialGroup": "projectReaders" + } + ], + "creationTime": "123456789", + "datasetReference": { + "datasetId": "bigquerydataset${uniqueId}", + "projectId": "${projectId}" + }, + "etag": "abcdef0123A=", + "id": "000000000000000000000", + "kind": "bigquery#dataset", + "labels": { + "cnrm-test": "true", + "managed-by-cnrm": "true" + }, + "lastModifiedTime": "123456789", + "location": "us-central1", + "selfLink": "https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets/bigquerydataset${uniqueId}", + "type": "DEFAULT" +} + +--- + +GET https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets/${datasetID}?alt=json +Content-Type: application/json +User-Agent: Terraform/ (+https://www.terraform.io) Terraform-Plugin-SDK/2.10.1 terraform-provider-google-beta/kcc/controller-manager + +200 OK +Cache-Control: private +Content-Type: application/json; charset=UTF-8 +Server: ESF +Vary: Origin +Vary: X-Origin +Vary: Referer +X-Content-Type-Options: nosniff +X-Frame-Options: SAMEORIGIN +X-Xss-Protection: 0 + +{ + "access": [ + { + "role": "WRITER", + "specialGroup": "projectWriters" + }, + { + "role": "OWNER", + "specialGroup": "projectOwners" + }, + { + "role": "OWNER", + "userByEmail": "user@google.com" + }, + { + "role": "READER", + "specialGroup": "projectReaders" + } + ], + "creationTime": "123456789", + "datasetReference": { + "datasetId": "bigquerydataset${uniqueId}", + "projectId": "${projectId}" + }, + "etag": "abcdef0123A=", + "id": "000000000000000000000", + "kind": "bigquery#dataset", + "labels": { + "cnrm-test": "true", + "managed-by-cnrm": "true" + }, + "lastModifiedTime": "123456789", + "location": "us-central1", + "selfLink": "https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets/bigquerydataset${uniqueId}", + "type": "DEFAULT" +} + +--- + +POST https://bigquerydatatransfer.googleapis.com/v1/projects/${projectId}/locations/us-central1/transferConfigs?%24alt=json%3Benum-encoding%3Dint +Content-Type: application/json +x-goog-request-params: parent=projects%2F${projectId}%2Flocations%2Fus-central1 + +{ + "dataSourceId": "salesforce", + "destinationDatasetId": "bigquerydataset${uniqueId}", + "displayName": "example of big query data transfer config", + "params": { + "assets": "asset-a", + "connector.authentication.oauth.clientId": "client-id", + "connector.authentication.oauth.clientSecret": "client-secret", + "connector.authentication.oauth.myDomain": "MyDomainName" + } +} + +200 OK +Cache-Control: private +Content-Type: application/json; charset=UTF-8 +Server: ESF +Vary: Origin +Vary: X-Origin +Vary: Referer +X-Content-Type-Options: nosniff +X-Frame-Options: SAMEORIGIN +X-Xss-Protection: 0 + +{ + "dataSourceId": "salesforce", + "datasetRegion": "us-central1", + "destinationDatasetId": "bigquerydataset${uniqueId}", + "displayName": "example of big query data transfer config", + "encryptionConfiguration": {}, + "name": "projects/${projectNumber}/locations/us-central1/transferConfigs/${transferConfigID}", + "nextRunTime": "2024-04-01T12:34:56.123456Z", + "ownerInfo": { + "email": "user@google.com" + }, + "params": { + "assets": "asset-a", + "connector.authentication.oauth.clientId": "client-id", + "connector.authentication.oauth.clientSecret": "client-secret", + "connector.authentication.oauth.myDomain": "MyDomainName" + }, + "updateTime": "2024-04-01T12:34:56.123456Z", + "userId": "0000000000000000000" +} + +--- + +GET https://bigquerydatatransfer.googleapis.com/v1/projects/${projectId}/locations/us-central1/transferConfigs/${transferConfigID}?%24alt=json%3Benum-encoding%3Dint +Content-Type: application/json +x-goog-request-params: name=projects%2F${projectId}%2Flocations%2Fus-central1%2FtransferConfigs%2F${transferConfigID} + +200 OK +Cache-Control: private +Content-Type: application/json; charset=UTF-8 +Server: ESF +Vary: Origin +Vary: X-Origin +Vary: Referer +X-Content-Type-Options: nosniff +X-Frame-Options: SAMEORIGIN +X-Xss-Protection: 0 + +{ + "dataSourceId": "salesforce", + "datasetRegion": "us-central1", + "destinationDatasetId": "bigquerydataset${uniqueId}", + "displayName": "example of big query data transfer config", + "emailPreferences": {}, + "name": "projects/${projectNumber}/locations/us-central1/transferConfigs/${transferConfigID}", + "nextRunTime": "2024-04-01T12:34:56.123456Z", + "ownerInfo": { + "email": "user@google.com" + }, + "params": { + "assets": "asset-a", + "connector.authentication.oauth.clientId": "client-id", + "connector.authentication.oauth.clientSecret": "client-secret", + "connector.authentication.oauth.myDomain": "MyDomainName" + }, + "updateTime": "2024-04-01T12:34:56.123456Z", + "userId": "0000000000000000000" +} + +--- + +PATCH https://bigquerydatatransfer.googleapis.com/v1/projects/${projectId}/locations/us-central1/transferConfigs/${transferConfigID}?%24alt=json%3Benum-encoding%3Dint&updateMask=disabled%2CdisplayName%2Cparams +Content-Type: application/json +x-goog-request-params: transfer_config.name=projects%2F${projectId}%2Flocations%2Fus-central1%2FtransferConfigs%2F${transferConfigID} + +{ + "dataSourceId": "salesforce", + "datasetRegion": "us-central1", + "destinationDatasetId": "bigquerydataset${uniqueId}", + "disabled": true, + "displayName": "updated example of big query data transfer config", + "emailPreferences": {}, + "name": "projects/${projectId}/locations/us-central1/transferConfigs/${transferConfigID}", + "nextRunTime": "2024-04-01T12:34:56.123456Z", + "ownerInfo": { + "email": "user@google.com" + }, + "params": { + "assets": "asset-b", + "connector.authentication.oauth.clientId": "client-id", + "connector.authentication.oauth.clientSecret": "client-secret", + "connector.authentication.oauth.myDomain": "MyDomainName2" + }, + "updateTime": "2024-04-01T12:34:56.123456Z", + "userId": "0000000000000000000" +} + +200 OK +Cache-Control: private +Content-Type: application/json; charset=UTF-8 +Server: ESF +Vary: Origin +Vary: X-Origin +Vary: Referer +X-Content-Type-Options: nosniff +X-Frame-Options: SAMEORIGIN +X-Xss-Protection: 0 + +{ + "dataSourceId": "salesforce", + "datasetRegion": "us-central1", + "destinationDatasetId": "bigquerydataset${uniqueId}", + "disabled": true, + "displayName": "updated example of big query data transfer config", + "emailPreferences": {}, + "name": "projects/${projectNumber}/locations/us-central1/transferConfigs/${transferConfigID}", + "ownerInfo": { + "email": "user@google.com" + }, + "params": { + "assets": "asset-b", + "connector.authentication.oauth.clientId": "client-id", + "connector.authentication.oauth.clientSecret": "client-secret", + "connector.authentication.oauth.myDomain": "MyDomainName2" + }, + "updateTime": "2024-04-01T12:34:56.123456Z", + "userId": "0000000000000000000" +} + +--- + +GET https://bigquerydatatransfer.googleapis.com/v1/projects/${projectId}/locations/us-central1/transferConfigs/${transferConfigID}?%24alt=json%3Benum-encoding%3Dint +Content-Type: application/json +x-goog-request-params: name=projects%2F${projectId}%2Flocations%2Fus-central1%2FtransferConfigs%2F${transferConfigID} + +200 OK +Cache-Control: private +Content-Type: application/json; charset=UTF-8 +Server: ESF +Vary: Origin +Vary: X-Origin +Vary: Referer +X-Content-Type-Options: nosniff +X-Frame-Options: SAMEORIGIN +X-Xss-Protection: 0 + +{ + "dataSourceId": "salesforce", + "datasetRegion": "us-central1", + "destinationDatasetId": "bigquerydataset${uniqueId}", + "disabled": true, + "displayName": "updated example of big query data transfer config", + "emailPreferences": {}, + "name": "projects/${projectNumber}/locations/us-central1/transferConfigs/${transferConfigID}", + "ownerInfo": { + "email": "user@google.com" + }, + "params": { + "assets": "asset-b", + "connector.authentication.oauth.clientId": "client-id", + "connector.authentication.oauth.clientSecret": "client-secret", + "connector.authentication.oauth.myDomain": "MyDomainName2" + }, + "updateTime": "2024-04-01T12:34:56.123456Z", + "userId": "0000000000000000000" +} + +--- + +DELETE https://bigquerydatatransfer.googleapis.com/v1/projects/${projectId}/locations/us-central1/transferConfigs/${transferConfigID}?%24alt=json%3Benum-encoding%3Dint +Content-Type: application/json +x-goog-request-params: name=projects%2F${projectId}%2Flocations%2Fus-central1%2FtransferConfigs%2F${transferConfigID} + +200 OK +Cache-Control: private +Content-Type: application/json; charset=UTF-8 +Server: ESF +Vary: Origin +Vary: X-Origin +Vary: Referer +X-Content-Type-Options: nosniff +X-Frame-Options: SAMEORIGIN +X-Xss-Protection: 0 + +{} + +--- + +GET https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets/${datasetID}?alt=json +Content-Type: application/json +User-Agent: Terraform/ (+https://www.terraform.io) Terraform-Plugin-SDK/2.10.1 terraform-provider-google-beta/kcc/controller-manager + +200 OK +Cache-Control: private +Content-Type: application/json; charset=UTF-8 +Server: ESF +Vary: Origin +Vary: X-Origin +Vary: Referer +X-Content-Type-Options: nosniff +X-Frame-Options: SAMEORIGIN +X-Xss-Protection: 0 + +{ + "access": [ + { + "role": "WRITER", + "specialGroup": "projectWriters" + }, + { + "role": "WRITER", + "userByEmail": "user@google.com" + }, + { + "role": "OWNER", + "specialGroup": "projectOwners" + }, + { + "role": "OWNER", + "userByEmail": "user@google.com" + }, + { + "role": "READER", + "specialGroup": "projectReaders" + } + ], + "creationTime": "123456789", + "datasetReference": { + "datasetId": "bigquerydataset${uniqueId}", + "projectId": "${projectId}" + }, + "etag": "abcdef0123A=", + "id": "000000000000000000000", + "kind": "bigquery#dataset", + "labels": { + "cnrm-test": "true", + "managed-by-cnrm": "true" + }, + "lastModifiedTime": "123456789", + "location": "us-central1", + "selfLink": "https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets/bigquerydataset${uniqueId}", + "type": "DEFAULT" +} + +--- + +DELETE https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets/${datasetID}?alt=json&deleteContents=false +Content-Type: application/json +User-Agent: Terraform/ (+https://www.terraform.io) Terraform-Plugin-SDK/2.10.1 terraform-provider-google-beta/kcc/controller-manager + +204 No Content +Content-Type: application/json; charset=UTF-8 +Server: ESF +Vary: Origin +Vary: X-Origin +Vary: Referer +X-Content-Type-Options: nosniff +X-Frame-Options: SAMEORIGIN +X-Xss-Protection: 0 \ No newline at end of file diff --git a/pkg/test/resourcefixture/testdata/basic/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig/bigquerydatatransferconfig-salesforce/create.yaml b/pkg/test/resourcefixture/testdata/basic/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig/bigquerydatatransferconfig-salesforce/create.yaml new file mode 100644 index 0000000000..4b1ab90a44 --- /dev/null +++ b/pkg/test/resourcefixture/testdata/basic/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig/bigquerydatatransferconfig-salesforce/create.yaml @@ -0,0 +1,31 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: bigquerydatatransfer.cnrm.cloud.google.com/v1alpha1 +kind: BigQueryDataTransferConfig +metadata: + name: bigquerydatatransferconfig-${uniqueId} +spec: + projectRef: + external: ${projectId} + location: us-central1 + displayName: "example of big query data transfer config" + dataSourceID: "salesforce" + datasetRef: + name: bigquerydataset${uniqueId} + params: + "connector.authentication.oauth.clientId": "client-id" + "connector.authentication.oauth.clientSecret": "client-secret" + "connector.authentication.oauth.myDomain": "MyDomainName" + "assets": "asset-a" diff --git a/pkg/test/resourcefixture/testdata/basic/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig/bigquerydatatransferconfig-salesforce/dependencies.yaml b/pkg/test/resourcefixture/testdata/basic/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig/bigquerydatatransferconfig-salesforce/dependencies.yaml new file mode 100644 index 0000000000..f6108223d9 --- /dev/null +++ b/pkg/test/resourcefixture/testdata/basic/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig/bigquerydatatransferconfig-salesforce/dependencies.yaml @@ -0,0 +1,20 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: bigquery.cnrm.cloud.google.com/v1beta1 +kind: BigQueryDataset +metadata: + name: bigquerydataset${uniqueId} # "-" is not supported in dataset ID +spec: + location: us-central1 diff --git a/pkg/test/resourcefixture/testdata/basic/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig/bigquerydatatransferconfig-salesforce/update.yaml b/pkg/test/resourcefixture/testdata/basic/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig/bigquerydatatransferconfig-salesforce/update.yaml new file mode 100644 index 0000000000..f185bd5703 --- /dev/null +++ b/pkg/test/resourcefixture/testdata/basic/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig/bigquerydatatransferconfig-salesforce/update.yaml @@ -0,0 +1,32 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: bigquerydatatransfer.cnrm.cloud.google.com/v1alpha1 +kind: BigQueryDataTransferConfig +metadata: + name: bigquerydatatransferconfig-${uniqueId} +spec: + projectRef: + external: ${projectId} + location: us-central1 + displayName: "updated example of big query data transfer config" + dataSourceID: "salesforce" + datasetRef: + name: bigquerydataset${uniqueId} + params: + "connector.authentication.oauth.clientId": "client-id2" + "connector.authentication.oauth.clientSecret": "client-secret2" + "connector.authentication.oauth.myDomain": "MyDomainName2" + "assets": "asset-b" + disabled: true diff --git a/tests/e2e/normalize.go b/tests/e2e/normalize.go index 91ae95eff8..eade0c0f74 100644 --- a/tests/e2e/normalize.go +++ b/tests/e2e/normalize.go @@ -174,6 +174,10 @@ func normalizeKRMObject(t *testing.T, u *unstructured.Unstructured, project test // Specific to BigQueryConnectionConnection. visitor.replacePaths[".status.observedState.cloudResource.serviceAccountID"] = "bqcx-${projectNumber}-abcd@gcp-sa-bigquery-condel.iam.gserviceaccount.com" + // Specific to BigQueryDataTransferConfig + visitor.replacePaths[".status.observedState.nextRunTime"] = "1970-01-01T00:00:00Z" + visitor.replacePaths[".status.observedState.ownerInfo.email"] = "user@google.com" + // TODO: This should not be needed, we want to avoid churning the kube objects visitor.sortSlices.Insert(".spec.access") visitor.sortSlices.Insert(".spec.nodeConfig.oauthScopes") @@ -242,6 +246,11 @@ func normalizeKRMObject(t *testing.T, u *unstructured.Unstructured, project test return strings.ReplaceAll(s, id, "${notificationChannelID}") }) } + if typeName == "transferConfigs" { + visitor.stringTransforms = append(visitor.stringTransforms, func(path string, s string) string { + return strings.ReplaceAll(s, id, "${transferConfigID}") + }) + } } resourceID, _, _ := unstructured.NestedString(u.Object, "spec", "resourceID") diff --git a/tests/e2e/replacements.go b/tests/e2e/replacements.go index b4650ad374..b1ce76d307 100644 --- a/tests/e2e/replacements.go +++ b/tests/e2e/replacements.go @@ -100,6 +100,8 @@ func (r *Replacements) ExtractIDsFromLinks(link string) { case "operations": r.OperationIDs[id] = true r.PathIDs[id] = "${operationID}" + case "transferConfigs": + r.PathIDs[id] = "${transferConfigID}" } tokens = tokens[:n-2] } diff --git a/tests/e2e/unified_test.go b/tests/e2e/unified_test.go index 90b0dc7475..aa6924e0a1 100644 --- a/tests/e2e/unified_test.go +++ b/tests/e2e/unified_test.go @@ -764,6 +764,24 @@ func runScenario(ctx context.Context, t *testing.T, testPause bool, fixture reso } }) + // Specific to BigQueryDataTransferConfig + addReplacement("nextRunTime", "2024-04-01T12:34:56.123456Z") + addReplacement("ownerInfo.email", "user@google.com") + addReplacement("userId", "0000000000000000000") + jsonMutators = append(jsonMutators, func(obj map[string]any) { // special handling because the field includes dot + if _, found, _ := unstructured.NestedString(obj, "params", "connector.authentication.oauth.clientId"); found { + if err := unstructured.SetNestedField(obj, "client-id", "params", "connector.authentication.oauth.clientId"); err != nil { + t.Fatal(err) + } + } + if _, found, _ := unstructured.NestedString(obj, "params", "connector.authentication.oauth.clientSecret"); found { + if err := unstructured.SetNestedField(obj, "client-secret", "params", "connector.authentication.oauth.clientSecret"); err != nil { + t.Fatal(err) + } + } + delete(obj, "state") // data transfer run state, which depends on timing + }) + // Remove error details which can contain confidential information jsonMutators = append(jsonMutators, func(obj map[string]any) { response := obj["error"] From 31bbc17827bc829212c48f9e662f965efb98ab8e Mon Sep 17 00:00:00 2001 From: Jingyi Hu Date: Fri, 13 Sep 2024 22:52:09 +0000 Subject: [PATCH 06/12] chore: make ready-pr --- .../bigquerydatatransferconfig_types.go | 169 +++++++++--------- .../v1alpha1/zz_generated.deepcopy.go | 158 +++++++++------- 2 files changed, 177 insertions(+), 150 deletions(-) diff --git a/pkg/clients/generated/apis/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig_types.go b/pkg/clients/generated/apis/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig_types.go index f9be1a5147..e122f8ed97 100644 --- a/pkg/clients/generated/apis/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig_types.go +++ b/pkg/clients/generated/apis/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig_types.go @@ -37,110 +37,87 @@ import ( type ConfigEmailPreferences struct { /* If true, email notifications will be sent on transfer run failures. */ - EnableFailureEmail bool `json:"enableFailureEmail"` + // +optional + EnableFailureEmail *bool `json:"enableFailureEmail,omitempty"` +} + +type ConfigEncryptionConfiguration struct { + /* The KMS key used for encrypting BigQuery data. */ + // +optional + KmsKeyRef *v1alpha1.ResourceRef `json:"kmsKeyRef,omitempty"` } type ConfigScheduleOptions struct { - /* If true, automatic scheduling of data transfer runs for this - configuration will be disabled. The runs can be started on ad-hoc - basis using transferConfigs.startManualRuns API. When automatic - scheduling is disabled, the TransferConfig.schedule field will - be ignored. */ + /* If true, automatic scheduling of data transfer runs for this configuration will be disabled. The runs can be started on ad-hoc basis using StartManualTransferRuns API. When automatic scheduling is disabled, the TransferConfig.schedule field will be ignored. */ // +optional DisableAutoScheduling *bool `json:"disableAutoScheduling,omitempty"` - /* Defines time to stop scheduling transfer runs. A transfer run cannot be - scheduled at or after the end time. The end time can be changed at any - moment. The time when a data transfer can be triggered manually is not - limited by this option. */ + /* Defines time to stop scheduling transfer runs. A transfer run cannot be scheduled at or after the end time. The end time can be changed at any moment. The time when a data transfer can be triggered manually is not limited by this option. */ // +optional EndTime *string `json:"endTime,omitempty"` - /* Specifies time to start scheduling transfer runs. The first run will be - scheduled at or after the start time according to a recurrence pattern - defined in the schedule string. The start time can be changed at any - moment. The time when a data transfer can be triggered manually is not - limited by this option. */ + /* Specifies time to start scheduling transfer runs. The first run will be scheduled at or after the start time according to a recurrence pattern defined in the schedule string. The start time can be changed at any moment. The time when a data transfer can be triggered manually is not limited by this option. */ // +optional StartTime *string `json:"startTime,omitempty"` } -type ConfigSecretAccessKey struct { - /* Value of the field. Cannot be used if 'valueFrom' is specified. */ - // +optional - Value *string `json:"value,omitempty"` - - /* Source for the field's value. Cannot be used if 'value' is specified. */ - // +optional - ValueFrom *ConfigValueFrom `json:"valueFrom,omitempty"` -} - -type ConfigSensitiveParams struct { - /* The Secret Access Key of the AWS account transferring data from. */ - SecretAccessKey ConfigSecretAccessKey `json:"secretAccessKey"` -} - -type ConfigValueFrom struct { - /* Reference to a value with the given key in the given Secret in the resource's namespace. */ - // +optional - SecretKeyRef *v1alpha1.SecretKeyRef `json:"secretKeyRef,omitempty"` -} - type BigQueryDataTransferConfigSpec struct { - /* The number of days to look back to automatically refresh the data. - For example, if dataRefreshWindowDays = 10, then every day BigQuery - reingests data for [today-10, today-1], rather than ingesting data for - just [today-1]. Only valid if the data source supports the feature. - Set the value to 0 to use the default value. */ + /* The number of days to look back to automatically refresh the data. For example, if `data_refresh_window_days = 10`, then every day BigQuery reingests data for [today-10, today-1], rather than ingesting data for just [today-1]. Only valid if the data source supports the feature. Set the value to 0 to use the default value. */ // +optional - DataRefreshWindowDays *int64 `json:"dataRefreshWindowDays,omitempty"` + DataRefreshWindowDays *int32 `json:"dataRefreshWindowDays,omitempty"` - /* Immutable. The data source id. Cannot be changed once the transfer config is created. */ - DataSourceId string `json:"dataSourceId"` + /* Immutable. Data source ID. This cannot be changed once data transfer is created. The full list of available data source IDs can be returned through an API call: https://cloud.google.com/bigquery-transfer/docs/reference/datatransfer/rest/v1/projects.locations.dataSources/list */ + DataSourceID string `json:"dataSourceID"` /* The BigQuery target dataset id. */ - // +optional - DestinationDatasetId *string `json:"destinationDatasetId,omitempty"` + DatasetRef v1alpha1.ResourceRef `json:"datasetRef"` - /* When set to true, no runs are scheduled for a given transfer. */ + /* Is this config disabled. When set to true, no runs will be scheduled for this transfer config. */ // +optional Disabled *bool `json:"disabled,omitempty"` - /* The user specified display name for the transfer config. */ - DisplayName string `json:"displayName"` + /* User specified display name for the data transfer. */ + // +optional + DisplayName *string `json:"displayName,omitempty"` - /* Email notifications will be sent according to these preferences to the - email address of the user who owns this transfer config. */ + /* Email notifications will be sent according to these preferences to the email address of the user who owns this transfer config. */ // +optional EmailPreferences *ConfigEmailPreferences `json:"emailPreferences,omitempty"` - /* Immutable. The geographic location where the transfer config should reside. - Examples: US, EU, asia-northeast1. The default value is US. */ + /* The encryption configuration part. Currently, it is only used for the optional KMS key name. The BigQuery service account of your project must be granted permissions to use the key. Read methods will return the key name applied in effect. Write methods will apply the key if it is present, or otherwise try to apply project default keys if it is absent. */ // +optional - Location *string `json:"location,omitempty"` + EncryptionConfiguration *ConfigEncryptionConfiguration `json:"encryptionConfiguration,omitempty"` - /* Pub/Sub topic where notifications will be sent after transfer runs - associated with this transfer config finish. */ - // +optional - NotificationPubsubTopic *string `json:"notificationPubsubTopic,omitempty"` + /* Immutable. */ + Location string `json:"location"` + /* Parameters specific to each data source. For more information see the bq tab in the 'Setting up a data transfer' section for each data source. For example the parameters for Cloud Storage transfers are listed here: https://cloud.google.com/bigquery-transfer/docs/cloud-storage-transfer#bq */ Params map[string]string `json:"params"` - /* The project that this resource belongs to. */ + /* The Project that this resource belongs to. */ ProjectRef v1alpha1.ResourceRef `json:"projectRef"` - /* Immutable. Optional. The service-generated name of the resource. Used for acquisition only. Leave unset to create a new resource. */ + /* Pub/Sub topic where notifications will be sent after transfer runs associated with this transfer config finish. */ + // +optional + PubSubTopicRef *v1alpha1.ResourceRef `json:"pubSubTopicRef,omitempty"` + + /* Immutable. The BigQueryDataTransferConfig name. If not given, the metadata.name will be used. */ // +optional ResourceID *string `json:"resourceID,omitempty"` - /* Data transfer schedule. If the data source does not support a custom - schedule, this should be empty. If it is empty, the default value for - the data source will be used. The specified times are in UTC. Examples - of valid format: 1st,3rd monday of month 15:30, every wed,fri of jan, - jun 13:15, and first sunday of quarter 00:00. See more explanation - about the format here: + /* Data transfer schedule. + If the data source does not support a custom schedule, this should be + empty. If it is empty, the default value for the data source will be used. + The specified times are in UTC. + Examples of valid format: + `1st,3rd monday of month 15:30`, + `every wed,fri of jan,jun 13:15`, and + `first sunday of quarter 00:00`. + See more explanation about the format here: https://cloud.google.com/appengine/docs/flexible/python/scheduling-jobs-with-cron-yaml#the_schedule_format - NOTE: the granularity should be at least 8 hours, or less frequent. */ + + NOTE: The minimum interval time between recurring transfers depends on the + data source; refer to the documentation for your data source. */ // +optional Schedule *string `json:"schedule,omitempty"` @@ -148,45 +125,65 @@ type BigQueryDataTransferConfigSpec struct { // +optional ScheduleOptions *ConfigScheduleOptions `json:"scheduleOptions,omitempty"` - /* Different parameters are configured primarily using the the 'params' field on this - resource. This block contains the parameters which contain secrets or passwords so that they can be marked - sensitive and hidden from plan output. The name of the field, eg: secret_access_key, will be the key - in the 'params' map in the api request. + /* Deprecated. Unique ID of the user on whose behalf transfer is done. */ + // +optional + UserID *int64 `json:"userID,omitempty"` +} + +type ConfigObservedStateStatus struct { + /* Output only. Region in which BigQuery dataset is located. */ + // +optional + DatasetRegion *string `json:"datasetRegion,omitempty"` + + /* Identifier. The resource name of the transfer config. Transfer config names have the form either `projects/{project_id}/locations/{region}/transferConfigs/{config_id}` or `projects/{project_id}/transferConfigs/{config_id}`, where `config_id` is usually a UUID, even though it is not guaranteed or required. The name is ignored when creating a transfer config. */ + // +optional + Name *string `json:"name,omitempty"` + + /* Output only. Next time when data transfer will run. */ + // +optional + NextRunTime *string `json:"nextRunTime,omitempty"` + + /* Output only. Information about the user whose credentials are used to transfer data. Populated only for `transferConfigs.get` requests. In case the user information is not available, this field will not be populated. */ + // +optional + OwnerInfo *ConfigOwnerInfoStatus `json:"ownerInfo,omitempty"` - Credentials may not be specified in both locations and will cause an error. Changing from one location - to a different credential configuration in the config will require an apply to update state. */ + /* Output only. State of the most recently updated transfer run. */ // +optional - SensitiveParams *ConfigSensitiveParams `json:"sensitiveParams,omitempty"` + State *string `json:"state,omitempty"` - /* Service account email. If this field is set, transfer config will - be created with this service account credentials. It requires that - requesting user calling this API has permissions to act as this service account. */ + /* Output only. Data transfer modification time. Ignored by server on input. */ // +optional - ServiceAccountName *string `json:"serviceAccountName,omitempty"` + UpdateTime *string `json:"updateTime,omitempty"` +} + +type ConfigOwnerInfoStatus struct { + /* E-mail address of the user. */ + // +optional + Email *string `json:"email,omitempty"` } type BigQueryDataTransferConfigStatus struct { /* Conditions represent the latest available observations of the BigQueryDataTransferConfig's current state. */ Conditions []v1alpha1.Condition `json:"conditions,omitempty"` - /* The resource name of the transfer config. Transfer config names have the - form projects/{projectId}/locations/{location}/transferConfigs/{configId} - or projects/{projectId}/transferConfigs/{configId}, - where configId is usually a uuid, but this is not required. - The name is ignored when creating a transfer config. */ + /* A unique specifier for the BigQueryDataTransferConfig resource in GCP. */ // +optional - Name *string `json:"name,omitempty"` + ExternalRef *string `json:"externalRef,omitempty"` /* ObservedGeneration is the generation of the resource that was most recently observed by the Config Connector controller. If this is equal to metadata.generation, then that means that the current reported status reflects the most recent desired state of the resource. */ // +optional ObservedGeneration *int64 `json:"observedGeneration,omitempty"` + + /* ObservedState is the state of the resource as most recently observed in GCP. */ + // +optional + ObservedState *ConfigObservedStateStatus `json:"observedState,omitempty"` } // +genclient // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object -// +kubebuilder:resource:categories=gcp,shortName=gcpbigquerydatatransferconfig;gcpbigquerydatatransferconfigs +// +kubebuilder:resource:categories=gcp,shortName= // +kubebuilder:subresource:status -// +kubebuilder:metadata:labels="cnrm.cloud.google.com/managed-by-kcc=true";"cnrm.cloud.google.com/stability-level=alpha";"cnrm.cloud.google.com/system=true";"cnrm.cloud.google.com/tf2crd=true" +// +kubebuilder:metadata:labels="cnrm.cloud.google.com/managed-by-kcc=true";"cnrm.cloud.google.com/system=true" // +kubebuilder:printcolumn:name="Age",JSONPath=".metadata.creationTimestamp",type="date" // +kubebuilder:printcolumn:name="Ready",JSONPath=".status.conditions[?(@.type=='Ready')].status",type="string",description="When 'True', the most recent reconcile of the resource succeeded" // +kubebuilder:printcolumn:name="Status",JSONPath=".status.conditions[?(@.type=='Ready')].reason",type="string",description="The reason for the value in 'Ready'" diff --git a/pkg/clients/generated/apis/bigquerydatatransfer/v1alpha1/zz_generated.deepcopy.go b/pkg/clients/generated/apis/bigquerydatatransfer/v1alpha1/zz_generated.deepcopy.go index 1ecfcc2c59..836a32c67e 100644 --- a/pkg/clients/generated/apis/bigquerydatatransfer/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/clients/generated/apis/bigquerydatatransfer/v1alpha1/zz_generated.deepcopy.go @@ -95,33 +95,29 @@ func (in *BigQueryDataTransferConfigSpec) DeepCopyInto(out *BigQueryDataTransfer *out = *in if in.DataRefreshWindowDays != nil { in, out := &in.DataRefreshWindowDays, &out.DataRefreshWindowDays - *out = new(int64) - **out = **in - } - if in.DestinationDatasetId != nil { - in, out := &in.DestinationDatasetId, &out.DestinationDatasetId - *out = new(string) + *out = new(int32) **out = **in } + out.DatasetRef = in.DatasetRef if in.Disabled != nil { in, out := &in.Disabled, &out.Disabled *out = new(bool) **out = **in } + if in.DisplayName != nil { + in, out := &in.DisplayName, &out.DisplayName + *out = new(string) + **out = **in + } if in.EmailPreferences != nil { in, out := &in.EmailPreferences, &out.EmailPreferences *out = new(ConfigEmailPreferences) - **out = **in - } - if in.Location != nil { - in, out := &in.Location, &out.Location - *out = new(string) - **out = **in + (*in).DeepCopyInto(*out) } - if in.NotificationPubsubTopic != nil { - in, out := &in.NotificationPubsubTopic, &out.NotificationPubsubTopic - *out = new(string) - **out = **in + if in.EncryptionConfiguration != nil { + in, out := &in.EncryptionConfiguration, &out.EncryptionConfiguration + *out = new(ConfigEncryptionConfiguration) + (*in).DeepCopyInto(*out) } if in.Params != nil { in, out := &in.Params, &out.Params @@ -131,6 +127,11 @@ func (in *BigQueryDataTransferConfigSpec) DeepCopyInto(out *BigQueryDataTransfer } } out.ProjectRef = in.ProjectRef + if in.PubSubTopicRef != nil { + in, out := &in.PubSubTopicRef, &out.PubSubTopicRef + *out = new(k8sv1alpha1.ResourceRef) + **out = **in + } if in.ResourceID != nil { in, out := &in.ResourceID, &out.ResourceID *out = new(string) @@ -146,14 +147,9 @@ func (in *BigQueryDataTransferConfigSpec) DeepCopyInto(out *BigQueryDataTransfer *out = new(ConfigScheduleOptions) (*in).DeepCopyInto(*out) } - if in.SensitiveParams != nil { - in, out := &in.SensitiveParams, &out.SensitiveParams - *out = new(ConfigSensitiveParams) - (*in).DeepCopyInto(*out) - } - if in.ServiceAccountName != nil { - in, out := &in.ServiceAccountName, &out.ServiceAccountName - *out = new(string) + if in.UserID != nil { + in, out := &in.UserID, &out.UserID + *out = new(int64) **out = **in } return @@ -177,8 +173,8 @@ func (in *BigQueryDataTransferConfigStatus) DeepCopyInto(out *BigQueryDataTransf *out = make([]k8sv1alpha1.Condition, len(*in)) copy(*out, *in) } - if in.Name != nil { - in, out := &in.Name, &out.Name + if in.ExternalRef != nil { + in, out := &in.ExternalRef, &out.ExternalRef *out = new(string) **out = **in } @@ -187,6 +183,11 @@ func (in *BigQueryDataTransferConfigStatus) DeepCopyInto(out *BigQueryDataTransf *out = new(int64) **out = **in } + if in.ObservedState != nil { + in, out := &in.ObservedState, &out.ObservedState + *out = new(ConfigObservedStateStatus) + (*in).DeepCopyInto(*out) + } return } @@ -203,6 +204,11 @@ func (in *BigQueryDataTransferConfigStatus) DeepCopy() *BigQueryDataTransferConf // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ConfigEmailPreferences) DeepCopyInto(out *ConfigEmailPreferences) { *out = *in + if in.EnableFailureEmail != nil { + in, out := &in.EnableFailureEmail, &out.EnableFailureEmail + *out = new(bool) + **out = **in + } return } @@ -217,96 +223,120 @@ func (in *ConfigEmailPreferences) DeepCopy() *ConfigEmailPreferences { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ConfigScheduleOptions) DeepCopyInto(out *ConfigScheduleOptions) { +func (in *ConfigEncryptionConfiguration) DeepCopyInto(out *ConfigEncryptionConfiguration) { *out = *in - if in.DisableAutoScheduling != nil { - in, out := &in.DisableAutoScheduling, &out.DisableAutoScheduling - *out = new(bool) - **out = **in - } - if in.EndTime != nil { - in, out := &in.EndTime, &out.EndTime - *out = new(string) - **out = **in - } - if in.StartTime != nil { - in, out := &in.StartTime, &out.StartTime - *out = new(string) + if in.KmsKeyRef != nil { + in, out := &in.KmsKeyRef, &out.KmsKeyRef + *out = new(k8sv1alpha1.ResourceRef) **out = **in } return } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConfigScheduleOptions. -func (in *ConfigScheduleOptions) DeepCopy() *ConfigScheduleOptions { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConfigEncryptionConfiguration. +func (in *ConfigEncryptionConfiguration) DeepCopy() *ConfigEncryptionConfiguration { if in == nil { return nil } - out := new(ConfigScheduleOptions) + out := new(ConfigEncryptionConfiguration) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ConfigSecretAccessKey) DeepCopyInto(out *ConfigSecretAccessKey) { +func (in *ConfigObservedStateStatus) DeepCopyInto(out *ConfigObservedStateStatus) { *out = *in - if in.Value != nil { - in, out := &in.Value, &out.Value + if in.DatasetRegion != nil { + in, out := &in.DatasetRegion, &out.DatasetRegion + *out = new(string) + **out = **in + } + if in.Name != nil { + in, out := &in.Name, &out.Name *out = new(string) **out = **in } - if in.ValueFrom != nil { - in, out := &in.ValueFrom, &out.ValueFrom - *out = new(ConfigValueFrom) + if in.NextRunTime != nil { + in, out := &in.NextRunTime, &out.NextRunTime + *out = new(string) + **out = **in + } + if in.OwnerInfo != nil { + in, out := &in.OwnerInfo, &out.OwnerInfo + *out = new(ConfigOwnerInfoStatus) (*in).DeepCopyInto(*out) } + if in.State != nil { + in, out := &in.State, &out.State + *out = new(string) + **out = **in + } + if in.UpdateTime != nil { + in, out := &in.UpdateTime, &out.UpdateTime + *out = new(string) + **out = **in + } return } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConfigSecretAccessKey. -func (in *ConfigSecretAccessKey) DeepCopy() *ConfigSecretAccessKey { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConfigObservedStateStatus. +func (in *ConfigObservedStateStatus) DeepCopy() *ConfigObservedStateStatus { if in == nil { return nil } - out := new(ConfigSecretAccessKey) + out := new(ConfigObservedStateStatus) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ConfigSensitiveParams) DeepCopyInto(out *ConfigSensitiveParams) { +func (in *ConfigOwnerInfoStatus) DeepCopyInto(out *ConfigOwnerInfoStatus) { *out = *in - in.SecretAccessKey.DeepCopyInto(&out.SecretAccessKey) + if in.Email != nil { + in, out := &in.Email, &out.Email + *out = new(string) + **out = **in + } return } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConfigSensitiveParams. -func (in *ConfigSensitiveParams) DeepCopy() *ConfigSensitiveParams { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConfigOwnerInfoStatus. +func (in *ConfigOwnerInfoStatus) DeepCopy() *ConfigOwnerInfoStatus { if in == nil { return nil } - out := new(ConfigSensitiveParams) + out := new(ConfigOwnerInfoStatus) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ConfigValueFrom) DeepCopyInto(out *ConfigValueFrom) { +func (in *ConfigScheduleOptions) DeepCopyInto(out *ConfigScheduleOptions) { *out = *in - if in.SecretKeyRef != nil { - in, out := &in.SecretKeyRef, &out.SecretKeyRef - *out = new(k8sv1alpha1.SecretKeyRef) + if in.DisableAutoScheduling != nil { + in, out := &in.DisableAutoScheduling, &out.DisableAutoScheduling + *out = new(bool) + **out = **in + } + if in.EndTime != nil { + in, out := &in.EndTime, &out.EndTime + *out = new(string) + **out = **in + } + if in.StartTime != nil { + in, out := &in.StartTime, &out.StartTime + *out = new(string) **out = **in } return } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConfigValueFrom. -func (in *ConfigValueFrom) DeepCopy() *ConfigValueFrom { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConfigScheduleOptions. +func (in *ConfigScheduleOptions) DeepCopy() *ConfigScheduleOptions { if in == nil { return nil } - out := new(ConfigValueFrom) + out := new(ConfigScheduleOptions) in.DeepCopyInto(out) return out } From d681d5de4ee5e812e266260ee99203e1ff6dca42 Mon Sep 17 00:00:00 2001 From: Jingyi Hu Date: Tue, 17 Sep 2024 05:32:14 +0000 Subject: [PATCH 07/12] feat: add serviceAccountName field in BigQueryDataTransferConfig Surface the "serviceAccountName" in BigQueryDataTransferConfig API --- .../bigquerydatatransferconfig_types.go | 6 ++++ .../v1alpha1/zz_generated.deepcopy.go | 5 +++ ...erydatatransfer.cnrm.cloud.google.com.yaml | 32 +++++++++++++++++++ .../bigquerydatatransferconfig_controller.go | 16 +++++++++- 4 files changed, 58 insertions(+), 1 deletion(-) diff --git a/apis/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig_types.go b/apis/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig_types.go index 01053b7f58..bdafbcbe8b 100644 --- a/apis/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig_types.go +++ b/apis/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig_types.go @@ -108,6 +108,12 @@ type BigQueryDataTransferConfigSpec struct { // Deprecated. Unique ID of the user on whose behalf transfer is done. UserID *int64 `json:"userID,omitempty"` + + // Service account email. If this field is set, the transfer config will be created with this service account's credentials. + // It requires that the requesting user calling this API has permissions to act as this service account. + // Note that not all data sources support service account credentials when creating a transfer config. + // For the latest list of data sources, please refer to https://cloud.google.com/bigquery/docs/use-service-accounts. + ServiceAccountRef *refv1beta1.IAMServiceAccountRef `json:"serviceAccountRef,omitempty"` } type Parent struct { diff --git a/apis/bigquerydatatransfer/v1alpha1/zz_generated.deepcopy.go b/apis/bigquerydatatransfer/v1alpha1/zz_generated.deepcopy.go index 53d726a785..2fab60ac4b 100644 --- a/apis/bigquerydatatransfer/v1alpha1/zz_generated.deepcopy.go +++ b/apis/bigquerydatatransfer/v1alpha1/zz_generated.deepcopy.go @@ -185,6 +185,11 @@ func (in *BigQueryDataTransferConfigSpec) DeepCopyInto(out *BigQueryDataTransfer *out = new(int64) **out = **in } + if in.ServiceAccountRef != nil { + in, out := &in.ServiceAccountRef, &out.ServiceAccountRef + *out = new(v1beta1.IAMServiceAccountRef) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BigQueryDataTransferConfigSpec. diff --git a/config/crds/resources/apiextensions.k8s.io_v1_customresourcedefinition_bigquerydatatransferconfigs.bigquerydatatransfer.cnrm.cloud.google.com.yaml b/config/crds/resources/apiextensions.k8s.io_v1_customresourcedefinition_bigquerydatatransferconfigs.bigquerydatatransfer.cnrm.cloud.google.com.yaml index 521b989cbd..b684f5e98a 100644 --- a/config/crds/resources/apiextensions.k8s.io_v1_customresourcedefinition_bigquerydatatransferconfigs.bigquerydatatransfer.cnrm.cloud.google.com.yaml +++ b/config/crds/resources/apiextensions.k8s.io_v1_customresourcedefinition_bigquerydatatransferconfigs.bigquerydatatransfer.cnrm.cloud.google.com.yaml @@ -278,6 +278,38 @@ spec: transfer can be triggered manually is not limited by this option. type: string type: object + serviceAccountRef: + description: Service account email. If this field is set, the transfer + config will be created with this service account's credentials. + It requires that the requesting user calling this API has permissions + to act as this service account. Note that not all data sources support + service account credentials when creating a transfer config. For + the latest list of data sources, please refer to https://cloud.google.com/bigquery/docs/use-service-accounts. + oneOf: + - not: + required: + - external + required: + - name + - not: + anyOf: + - required: + - name + - required: + - namespace + required: + - external + properties: + external: + description: The `email` field of an `IAMServiceAccount` resource. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + type: object userID: description: Deprecated. Unique ID of the user on whose behalf transfer is done. diff --git a/pkg/controller/direct/bigquerydatatransfer/bigquerydatatransferconfig_controller.go b/pkg/controller/direct/bigquerydatatransfer/bigquerydatatransferconfig_controller.go index f488d316eb..f913de9492 100644 --- a/pkg/controller/direct/bigquerydatatransfer/bigquerydatatransferconfig_controller.go +++ b/pkg/controller/direct/bigquerydatatransfer/bigquerydatatransferconfig_controller.go @@ -123,6 +123,14 @@ func (m *model) AdapterForObject(ctx context.Context, reader client.Reader, u *u obj.Spec.EncryptionConfiguration.KmsKeyRef = kmsCryptoKeyRef } + // Resolve ServiceAccount Ref + if obj.Spec.ServiceAccountRef != nil { + err := obj.Spec.ServiceAccountRef.Resolve(ctx, reader, obj) + if err != nil { + return nil, err + } + } + // Resolve Project Ref projectRef, err := refv1beta1.ResolveProject(ctx, reader, obj, obj.Spec.ProjectRef) if err != nil { @@ -226,6 +234,9 @@ func (a *Adapter) Create(ctx context.Context, createOp *directbase.CreateOperati Parent: a.id.Parent(), TransferConfig: resource, } + if desired.Spec.ServiceAccountRef != nil { // special handling for service account field which is not present in GCP proto + req.ServiceAccountName = desired.Spec.ServiceAccountRef.External + } created, err := a.gcpClient.CreateTransferConfig(ctx, req) if err != nil { return fmt.Errorf("creating BigQueryDataTransferConfig %s: %w", a.id.FullyQualifiedName(), err) @@ -315,11 +326,14 @@ func (a *Adapter) Update(ctx context.Context, updateOp *directbase.UpdateOperati return nil } - resource.Name = a.id.FullyQualifiedName() // need to pass service generated ID to GCP API + resource.Name = a.id.FullyQualifiedName() // need to pass service generated ID to GCP API to identify the GCP resource req := &bigquerydatatransferpb.UpdateTransferConfigRequest{ TransferConfig: resource, UpdateMask: updateMask, } + if a.desired.Spec.ServiceAccountRef != nil { // special handling for service account field which is not present in GCP proto + req.ServiceAccountName = a.desired.Spec.ServiceAccountRef.External + } updated, err := a.gcpClient.UpdateTransferConfig(ctx, req) if err != nil { return fmt.Errorf("updating BigQueryDataTransferConfig %s: %w", a.id.FullyQualifiedName(), err) From bc4dd551bf9b42e24ce04bd746fae21a5091105e Mon Sep 17 00:00:00 2001 From: Jingyi Hu Date: Tue, 17 Sep 2024 05:36:34 +0000 Subject: [PATCH 08/12] test: add a test of scheuled query for BigQueryDataTransferConfig --- ...atransferconfig-scheduledquery.golden.yaml | 44 ++ .../_http.log | 558 ++++++++++++++++++ .../create.yaml | 33 ++ .../dependencies.yaml | 27 + .../update.yaml | 33 ++ tests/e2e/normalize.go | 1 + 6 files changed, 696 insertions(+) create mode 100644 pkg/test/resourcefixture/testdata/basic/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig/bigquerydatatransferconfig-scheduledquery/_generated_object_bigquerydatatransferconfig-scheduledquery.golden.yaml create mode 100644 pkg/test/resourcefixture/testdata/basic/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig/bigquerydatatransferconfig-scheduledquery/_http.log create mode 100644 pkg/test/resourcefixture/testdata/basic/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig/bigquerydatatransferconfig-scheduledquery/create.yaml create mode 100644 pkg/test/resourcefixture/testdata/basic/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig/bigquerydatatransferconfig-scheduledquery/dependencies.yaml create mode 100644 pkg/test/resourcefixture/testdata/basic/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig/bigquerydatatransferconfig-scheduledquery/update.yaml diff --git a/pkg/test/resourcefixture/testdata/basic/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig/bigquerydatatransferconfig-scheduledquery/_generated_object_bigquerydatatransferconfig-scheduledquery.golden.yaml b/pkg/test/resourcefixture/testdata/basic/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig/bigquerydatatransferconfig-scheduledquery/_generated_object_bigquerydatatransferconfig-scheduledquery.golden.yaml new file mode 100644 index 0000000000..776cbf2549 --- /dev/null +++ b/pkg/test/resourcefixture/testdata/basic/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig/bigquerydatatransferconfig-scheduledquery/_generated_object_bigquerydatatransferconfig-scheduledquery.golden.yaml @@ -0,0 +1,44 @@ +apiVersion: bigquerydatatransfer.cnrm.cloud.google.com/v1alpha1 +kind: BigQueryDataTransferConfig +metadata: + annotations: + cnrm.cloud.google.com/management-conflict-prevention-policy: none + finalizers: + - cnrm.cloud.google.com/finalizer + - cnrm.cloud.google.com/deletion-defender + generation: 2 + labels: + cnrm-test: "true" + name: bigquerydatatransferconfig-${uniqueId} + namespace: ${uniqueId} +spec: + dataSourceID: scheduled_query + datasetRef: + name: bigquerydataset${uniqueId} + displayName: an updated example of scheduled query + location: us-central1 + params: + destination_table_name_template: my_table2 + query: SELECT COUNT(*) as total_rows FROM my_table + write_disposition: WRITE_TRUNCATE + projectRef: + external: ${projectId} + schedule: every 24 hours + serviceAccountRef: + name: gsa-${uniqueId} +status: + conditions: + - lastTransitionTime: "1970-01-01T00:00:00Z" + message: The resource is up to date + reason: UpToDate + status: "True" + type: Ready + externalRef: //bigquerydatatransfer.googleapis.com/projects/${projectId}/locations/us-central1/transferConfigs/${transferConfigID} + observedGeneration: 2 + observedState: + datasetRegion: us-central1 + name: projects/${projectNumber}/locations/us-central1/transferConfigs/${transferConfigID} + nextRunTime: "1970-01-01T00:00:00Z" + ownerInfo: + email: user@google.com + updateTime: "1970-01-01T00:00:00Z" diff --git a/pkg/test/resourcefixture/testdata/basic/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig/bigquerydatatransferconfig-scheduledquery/_http.log b/pkg/test/resourcefixture/testdata/basic/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig/bigquerydatatransferconfig-scheduledquery/_http.log new file mode 100644 index 0000000000..d1818e2a22 --- /dev/null +++ b/pkg/test/resourcefixture/testdata/basic/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig/bigquerydatatransferconfig-scheduledquery/_http.log @@ -0,0 +1,558 @@ +GET https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets/${datasetID}?alt=json +Content-Type: application/json +User-Agent: Terraform/ (+https://www.terraform.io) Terraform-Plugin-SDK/2.10.1 terraform-provider-google-beta/kcc/controller-manager + +404 Not Found +Cache-Control: private +Content-Type: application/json; charset=UTF-8 +Server: ESF +Vary: Origin +Vary: X-Origin +Vary: Referer +X-Content-Type-Options: nosniff +X-Frame-Options: SAMEORIGIN +X-Xss-Protection: 0 + +{ + "error": { + "code": 404, + "errors": [ + { + "domain": "global", + "message": "Not found: Dataset ${projectId}:bigquerydataset${uniqueId}", + "reason": "notFound" + } + ], + "message": "Not found: Dataset ${projectId}:bigquerydataset${uniqueId}", + "status": "NOT_FOUND" + } +} + +--- + +POST https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets?alt=json +Content-Type: application/json +User-Agent: Terraform/ (+https://www.terraform.io) Terraform-Plugin-SDK/2.10.1 terraform-provider-google-beta/kcc/controller-manager + +{ + "datasetReference": { + "datasetId": "bigquerydataset${uniqueId}" + }, + "labels": { + "cnrm-test": "true", + "managed-by-cnrm": "true" + }, + "location": "us-central1" +} + +200 OK +Cache-Control: private +Content-Type: application/json; charset=UTF-8 +Server: ESF +Vary: Origin +Vary: X-Origin +Vary: Referer +X-Content-Type-Options: nosniff +X-Frame-Options: SAMEORIGIN +X-Xss-Protection: 0 + +{ + "access": [ + { + "role": "WRITER", + "specialGroup": "projectWriters" + }, + { + "role": "OWNER", + "specialGroup": "projectOwners" + }, + { + "role": "OWNER", + "userByEmail": "user@google.com" + }, + { + "role": "READER", + "specialGroup": "projectReaders" + } + ], + "creationTime": "123456789", + "datasetReference": { + "datasetId": "bigquerydataset${uniqueId}", + "projectId": "${projectId}" + }, + "etag": "abcdef0123A=", + "id": "000000000000000000000", + "kind": "bigquery#dataset", + "labels": { + "cnrm-test": "true", + "managed-by-cnrm": "true" + }, + "lastModifiedTime": "123456789", + "location": "us-central1", + "selfLink": "https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets/bigquerydataset${uniqueId}", + "type": "DEFAULT" +} + +--- + +GET https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets/${datasetID}?alt=json +Content-Type: application/json +User-Agent: Terraform/ (+https://www.terraform.io) Terraform-Plugin-SDK/2.10.1 terraform-provider-google-beta/kcc/controller-manager + +200 OK +Cache-Control: private +Content-Type: application/json; charset=UTF-8 +Server: ESF +Vary: Origin +Vary: X-Origin +Vary: Referer +X-Content-Type-Options: nosniff +X-Frame-Options: SAMEORIGIN +X-Xss-Protection: 0 + +{ + "access": [ + { + "role": "WRITER", + "specialGroup": "projectWriters" + }, + { + "role": "OWNER", + "specialGroup": "projectOwners" + }, + { + "role": "OWNER", + "userByEmail": "user@google.com" + }, + { + "role": "READER", + "specialGroup": "projectReaders" + } + ], + "creationTime": "123456789", + "datasetReference": { + "datasetId": "bigquerydataset${uniqueId}", + "projectId": "${projectId}" + }, + "etag": "abcdef0123A=", + "id": "000000000000000000000", + "kind": "bigquery#dataset", + "labels": { + "cnrm-test": "true", + "managed-by-cnrm": "true" + }, + "lastModifiedTime": "123456789", + "location": "us-central1", + "selfLink": "https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets/bigquerydataset${uniqueId}", + "type": "DEFAULT" +} + +--- + +GET https://iam.googleapis.com/v1/projects/${projectId}/serviceAccounts/gsa-${uniqueId}@${projectId}.iam.gserviceaccount.com?alt=json&prettyPrint=false +User-Agent: google-api-go-client/0.5 Terraform/ (+https://www.terraform.io) Terraform-Plugin-SDK/2.10.1 terraform-provider-google-beta/kcc/controller-manager + +404 Not Found +Cache-Control: private +Content-Type: application/json; charset=UTF-8 +Server: ESF +Vary: Origin +Vary: X-Origin +Vary: Referer +X-Content-Type-Options: nosniff +X-Frame-Options: SAMEORIGIN +X-Xss-Protection: 0 + +{ + "error": { + "code": 404, + "errors": [ + { + "domain": "global", + "message": "Unknown service account", + "reason": "notFound" + } + ], + "message": "Unknown service account", + "status": "NOT_FOUND" + } +} + +--- + +POST https://iam.googleapis.com/v1/projects/${projectId}/serviceAccounts?alt=json&prettyPrint=false +Content-Type: application/json +User-Agent: google-api-go-client/0.5 Terraform/ (+https://www.terraform.io) Terraform-Plugin-SDK/2.10.1 terraform-provider-google-beta/kcc/controller-manager + +{ + "accountId": "gsa-${uniqueId}", + "serviceAccount": { + "displayName": "Test GSA for big query data transfer config" + } +} + +200 OK +Cache-Control: private +Content-Type: application/json; charset=UTF-8 +Server: ESF +Vary: Origin +Vary: X-Origin +Vary: Referer +X-Content-Type-Options: nosniff +X-Frame-Options: SAMEORIGIN +X-Xss-Protection: 0 + +{ + "displayName": "Test GSA for big query data transfer config", + "email": "gsa-${uniqueId}@${projectId}.iam.gserviceaccount.com", + "etag": "abcdef0123A=", + "name": "projects/${projectId}/serviceAccounts/gsa-${uniqueId}@${projectId}.iam.gserviceaccount.com", + "oauth2ClientId": "888888888888888888888", + "projectId": "${projectId}", + "uniqueId": "111111111111111111111" +} + +--- + +GET https://iam.googleapis.com/v1/projects/${projectId}/serviceAccounts/gsa-${uniqueId}@${projectId}.iam.gserviceaccount.com?alt=json&prettyPrint=false +User-Agent: google-api-go-client/0.5 Terraform/ (+https://www.terraform.io) Terraform-Plugin-SDK/2.10.1 terraform-provider-google-beta/kcc/controller-manager + +200 OK +Cache-Control: private +Content-Type: application/json; charset=UTF-8 +Server: ESF +Vary: Origin +Vary: X-Origin +Vary: Referer +X-Content-Type-Options: nosniff +X-Frame-Options: SAMEORIGIN +X-Xss-Protection: 0 + +{ + "displayName": "Test GSA for big query data transfer config", + "email": "gsa-${uniqueId}@${projectId}.iam.gserviceaccount.com", + "etag": "abcdef0123A=", + "name": "projects/${projectId}/serviceAccounts/gsa-${uniqueId}@${projectId}.iam.gserviceaccount.com", + "oauth2ClientId": "888888888888888888888", + "projectId": "${projectId}", + "uniqueId": "111111111111111111111" +} + +--- + +POST https://bigquerydatatransfer.googleapis.com/v1/projects/${projectId}/locations/us-central1/transferConfigs?%24alt=json%3Benum-encoding%3Dint&serviceAccountName=gsa-${uniqueId}%40${projectId}.iam.gserviceaccount.com +Content-Type: application/json +x-goog-request-params: parent=projects%2F${projectId}%2Flocations%2Fus-central1 + +{ + "dataSourceId": "scheduled_query", + "destinationDatasetId": "bigquerydataset${uniqueId}", + "displayName": "example of scheduled query", + "params": { + "destination_table_name_template": "my_table", + "query": "SELECT name FROM tabl WHERE x = 'y'", + "write_disposition": "WRITE_APPEND" + }, + "schedule": "first sunday of quarter 00:00" +} + +200 OK +Cache-Control: private +Content-Type: application/json; charset=UTF-8 +Server: ESF +Vary: Origin +Vary: X-Origin +Vary: Referer +X-Content-Type-Options: nosniff +X-Frame-Options: SAMEORIGIN +X-Xss-Protection: 0 + +{ + "dataSourceId": "scheduled_query", + "datasetRegion": "us-central1", + "destinationDatasetId": "bigquerydataset${uniqueId}", + "displayName": "example of scheduled query", + "encryptionConfiguration": {}, + "name": "projects/${projectNumber}/locations/us-central1/transferConfigs/${transferConfigID}", + "nextRunTime": "2024-04-01T12:34:56.123456Z", + "ownerInfo": { + "email": "user@google.com" + }, + "params": { + "destination_table_name_template": "my_table", + "query": "SELECT name FROM tabl WHERE x = 'y'", + "write_disposition": "WRITE_APPEND" + }, + "schedule": "first sunday of quarter 00:00", + "updateTime": "2024-04-01T12:34:56.123456Z", + "userId": "0000000000000000000" +} + +--- + +GET https://bigquerydatatransfer.googleapis.com/v1/projects/${projectId}/locations/us-central1/transferConfigs/${transferConfigID}?%24alt=json%3Benum-encoding%3Dint +Content-Type: application/json +x-goog-request-params: name=projects%2F${projectId}%2Flocations%2Fus-central1%2FtransferConfigs%2F${transferConfigID} + +200 OK +Cache-Control: private +Content-Type: application/json; charset=UTF-8 +Server: ESF +Vary: Origin +Vary: X-Origin +Vary: Referer +X-Content-Type-Options: nosniff +X-Frame-Options: SAMEORIGIN +X-Xss-Protection: 0 + +{ + "dataSourceId": "scheduled_query", + "datasetRegion": "us-central1", + "destinationDatasetId": "bigquerydataset${uniqueId}", + "displayName": "example of scheduled query", + "emailPreferences": {}, + "name": "projects/${projectNumber}/locations/us-central1/transferConfigs/${transferConfigID}", + "nextRunTime": "2024-04-01T12:34:56.123456Z", + "ownerInfo": { + "email": "user@google.com" + }, + "params": { + "destination_table_name_template": "my_table", + "query": "SELECT name FROM tabl WHERE x = 'y'", + "write_disposition": "WRITE_APPEND" + }, + "schedule": "first sunday of quarter 00:00", + "updateTime": "2024-04-01T12:34:56.123456Z", + "userId": "0000000000000000000" +} + +--- + +PATCH https://bigquerydatatransfer.googleapis.com/v1/projects/${projectId}/locations/us-central1/transferConfigs/${transferConfigID}?%24alt=json%3Benum-encoding%3Dint&serviceAccountName=gsa-${uniqueId}%40${projectId}.iam.gserviceaccount.com&updateMask=displayName%2Cparams%2Cschedule +Content-Type: application/json +x-goog-request-params: transfer_config.name=projects%2F${projectId}%2Flocations%2Fus-central1%2FtransferConfigs%2F${transferConfigID} + +{ + "dataSourceId": "scheduled_query", + "datasetRegion": "us-central1", + "destinationDatasetId": "bigquerydataset${uniqueId}", + "displayName": "an updated example of scheduled query", + "emailPreferences": {}, + "name": "projects/${projectId}/locations/us-central1/transferConfigs/${transferConfigID}", + "nextRunTime": "2024-04-01T12:34:56.123456Z", + "ownerInfo": { + "email": "user@google.com" + }, + "params": { + "destination_table_name_template": "my_table2", + "query": "SELECT COUNT(*) as total_rows FROM my_table", + "write_disposition": "WRITE_TRUNCATE" + }, + "schedule": "every 24 hours", + "updateTime": "2024-04-01T12:34:56.123456Z", + "userId": "0000000000000000000" +} + +200 OK +Cache-Control: private +Content-Type: application/json; charset=UTF-8 +Server: ESF +Vary: Origin +Vary: X-Origin +Vary: Referer +X-Content-Type-Options: nosniff +X-Frame-Options: SAMEORIGIN +X-Xss-Protection: 0 + +{ + "dataSourceId": "scheduled_query", + "datasetRegion": "us-central1", + "destinationDatasetId": "bigquerydataset${uniqueId}", + "displayName": "an updated example of scheduled query", + "emailPreferences": {}, + "name": "projects/${projectNumber}/locations/us-central1/transferConfigs/${transferConfigID}", + "nextRunTime": "2024-04-01T12:34:56.123456Z", + "ownerInfo": { + "email": "user@google.com" + }, + "params": { + "destination_table_name_template": "my_table2", + "query": "SELECT COUNT(*) as total_rows FROM my_table", + "write_disposition": "WRITE_TRUNCATE" + }, + "schedule": "every 24 hours", + "updateTime": "2024-04-01T12:34:56.123456Z", + "userId": "0000000000000000000" +} + +--- + +GET https://bigquerydatatransfer.googleapis.com/v1/projects/${projectId}/locations/us-central1/transferConfigs/${transferConfigID}?%24alt=json%3Benum-encoding%3Dint +Content-Type: application/json +x-goog-request-params: name=projects%2F${projectId}%2Flocations%2Fus-central1%2FtransferConfigs%2F${transferConfigID} + +200 OK +Cache-Control: private +Content-Type: application/json; charset=UTF-8 +Server: ESF +Vary: Origin +Vary: X-Origin +Vary: Referer +X-Content-Type-Options: nosniff +X-Frame-Options: SAMEORIGIN +X-Xss-Protection: 0 + +{ + "dataSourceId": "scheduled_query", + "datasetRegion": "us-central1", + "destinationDatasetId": "bigquerydataset${uniqueId}", + "displayName": "an updated example of scheduled query", + "emailPreferences": {}, + "name": "projects/${projectNumber}/locations/us-central1/transferConfigs/${transferConfigID}", + "nextRunTime": "2024-04-01T12:34:56.123456Z", + "ownerInfo": { + "email": "user@google.com" + }, + "params": { + "destination_table_name_template": "my_table2", + "query": "SELECT COUNT(*) as total_rows FROM my_table", + "write_disposition": "WRITE_TRUNCATE" + }, + "schedule": "every 24 hours", + "scheduleOptions": {}, + "updateTime": "2024-04-01T12:34:56.123456Z", + "userId": "0000000000000000000" +} + +--- + +DELETE https://bigquerydatatransfer.googleapis.com/v1/projects/${projectId}/locations/us-central1/transferConfigs/${transferConfigID}?%24alt=json%3Benum-encoding%3Dint +Content-Type: application/json +x-goog-request-params: name=projects%2F${projectId}%2Flocations%2Fus-central1%2FtransferConfigs%2F${transferConfigID} + +200 OK +Cache-Control: private +Content-Type: application/json; charset=UTF-8 +Server: ESF +Vary: Origin +Vary: X-Origin +Vary: Referer +X-Content-Type-Options: nosniff +X-Frame-Options: SAMEORIGIN +X-Xss-Protection: 0 + +{} + +--- + +GET https://iam.googleapis.com/v1/projects/${projectId}/serviceAccounts/gsa-${uniqueId}@${projectId}.iam.gserviceaccount.com?alt=json&prettyPrint=false +User-Agent: google-api-go-client/0.5 Terraform/ (+https://www.terraform.io) Terraform-Plugin-SDK/2.10.1 terraform-provider-google-beta/kcc/controller-manager + +200 OK +Cache-Control: private +Content-Type: application/json; charset=UTF-8 +Server: ESF +Vary: Origin +Vary: X-Origin +Vary: Referer +X-Content-Type-Options: nosniff +X-Frame-Options: SAMEORIGIN +X-Xss-Protection: 0 + +{ + "displayName": "Test GSA for big query data transfer config", + "email": "gsa-${uniqueId}@${projectId}.iam.gserviceaccount.com", + "etag": "abcdef0123A=", + "name": "projects/${projectId}/serviceAccounts/gsa-${uniqueId}@${projectId}.iam.gserviceaccount.com", + "oauth2ClientId": "888888888888888888888", + "projectId": "${projectId}", + "uniqueId": "111111111111111111111" +} + +--- + +DELETE https://iam.googleapis.com/v1/projects/${projectId}/serviceAccounts/gsa-${uniqueId}@${projectId}.iam.gserviceaccount.com?alt=json&prettyPrint=false +User-Agent: google-api-go-client/0.5 Terraform/ (+https://www.terraform.io) Terraform-Plugin-SDK/2.10.1 terraform-provider-google-beta/kcc/controller-manager + +200 OK +Cache-Control: private +Content-Type: application/json; charset=UTF-8 +Server: ESF +Vary: Origin +Vary: X-Origin +Vary: Referer +X-Content-Type-Options: nosniff +X-Frame-Options: SAMEORIGIN +X-Xss-Protection: 0 + +{} + +--- + +GET https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets/${datasetID}?alt=json +Content-Type: application/json +User-Agent: Terraform/ (+https://www.terraform.io) Terraform-Plugin-SDK/2.10.1 terraform-provider-google-beta/kcc/controller-manager + +200 OK +Cache-Control: private +Content-Type: application/json; charset=UTF-8 +Server: ESF +Vary: Origin +Vary: X-Origin +Vary: Referer +X-Content-Type-Options: nosniff +X-Frame-Options: SAMEORIGIN +X-Xss-Protection: 0 + +{ + "access": [ + { + "role": "WRITER", + "specialGroup": "projectWriters" + }, + { + "role": "OWNER", + "specialGroup": "projectOwners" + }, + { + "role": "OWNER", + "userByEmail": "user@google.com" + }, + { + "role": "READER", + "specialGroup": "projectReaders" + } + ], + "creationTime": "123456789", + "datasetReference": { + "datasetId": "bigquerydataset${uniqueId}", + "projectId": "${projectId}" + }, + "etag": "abcdef0123A=", + "id": "000000000000000000000", + "kind": "bigquery#dataset", + "labels": { + "cnrm-test": "true", + "managed-by-cnrm": "true" + }, + "lastModifiedTime": "123456789", + "location": "us-central1", + "selfLink": "https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets/bigquerydataset${uniqueId}", + "type": "DEFAULT" +} + +--- + +DELETE https://bigquery.googleapis.com/bigquery/v2/projects/${projectId}/datasets/${datasetID}?alt=json&deleteContents=false +Content-Type: application/json +User-Agent: Terraform/ (+https://www.terraform.io) Terraform-Plugin-SDK/2.10.1 terraform-provider-google-beta/kcc/controller-manager + +204 No Content +Content-Type: application/json; charset=UTF-8 +Server: ESF +Vary: Origin +Vary: X-Origin +Vary: Referer +X-Content-Type-Options: nosniff +X-Frame-Options: SAMEORIGIN +X-Xss-Protection: 0 \ No newline at end of file diff --git a/pkg/test/resourcefixture/testdata/basic/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig/bigquerydatatransferconfig-scheduledquery/create.yaml b/pkg/test/resourcefixture/testdata/basic/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig/bigquerydatatransferconfig-scheduledquery/create.yaml new file mode 100644 index 0000000000..56b21dcf00 --- /dev/null +++ b/pkg/test/resourcefixture/testdata/basic/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig/bigquerydatatransferconfig-scheduledquery/create.yaml @@ -0,0 +1,33 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: bigquerydatatransfer.cnrm.cloud.google.com/v1alpha1 +kind: BigQueryDataTransferConfig +metadata: + name: bigquerydatatransferconfig-${uniqueId} +spec: + projectRef: + external: ${projectId} + location: us-central1 + displayName: "example of scheduled query" + dataSourceID: "scheduled_query" + datasetRef: + name: bigquerydataset${uniqueId} + params: + destination_table_name_template: "my_table" + write_disposition: "WRITE_APPEND" + query: "SELECT name FROM tabl WHERE x = 'y'" + schedule: "first sunday of quarter 00:00" + serviceAccountRef: + name: gsa-${uniqueId} diff --git a/pkg/test/resourcefixture/testdata/basic/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig/bigquerydatatransferconfig-scheduledquery/dependencies.yaml b/pkg/test/resourcefixture/testdata/basic/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig/bigquerydatatransferconfig-scheduledquery/dependencies.yaml new file mode 100644 index 0000000000..8d886a5807 --- /dev/null +++ b/pkg/test/resourcefixture/testdata/basic/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig/bigquerydatatransferconfig-scheduledquery/dependencies.yaml @@ -0,0 +1,27 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: bigquery.cnrm.cloud.google.com/v1beta1 +kind: BigQueryDataset +metadata: + name: bigquerydataset${uniqueId} # "-" is not supported in dataset ID +spec: + location: us-central1 +--- +apiVersion: iam.cnrm.cloud.google.com/v1beta1 +kind: IAMServiceAccount +metadata: + name: gsa-${uniqueId} +spec: + displayName: "Test GSA for big query data transfer config" diff --git a/pkg/test/resourcefixture/testdata/basic/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig/bigquerydatatransferconfig-scheduledquery/update.yaml b/pkg/test/resourcefixture/testdata/basic/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig/bigquerydatatransferconfig-scheduledquery/update.yaml new file mode 100644 index 0000000000..7cb911a4c0 --- /dev/null +++ b/pkg/test/resourcefixture/testdata/basic/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig/bigquerydatatransferconfig-scheduledquery/update.yaml @@ -0,0 +1,33 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: bigquerydatatransfer.cnrm.cloud.google.com/v1alpha1 +kind: BigQueryDataTransferConfig +metadata: + name: bigquerydatatransferconfig-${uniqueId} +spec: + projectRef: + external: ${projectId} + location: us-central1 + displayName: "an updated example of scheduled query" + dataSourceID: "scheduled_query" + datasetRef: + name: bigquerydataset${uniqueId} + params: + destination_table_name_template: "my_table2" + write_disposition: "WRITE_TRUNCATE" + query: "SELECT COUNT(*) as total_rows FROM my_table" + schedule: "every 24 hours" + serviceAccountRef: + name: gsa-${uniqueId} diff --git a/tests/e2e/normalize.go b/tests/e2e/normalize.go index eade0c0f74..0c8a09fc96 100644 --- a/tests/e2e/normalize.go +++ b/tests/e2e/normalize.go @@ -177,6 +177,7 @@ func normalizeKRMObject(t *testing.T, u *unstructured.Unstructured, project test // Specific to BigQueryDataTransferConfig visitor.replacePaths[".status.observedState.nextRunTime"] = "1970-01-01T00:00:00Z" visitor.replacePaths[".status.observedState.ownerInfo.email"] = "user@google.com" + visitor.removePaths.Insert(".status.observedState.state") // data transfer run state, which depends on timing // TODO: This should not be needed, we want to avoid churning the kube objects visitor.sortSlices.Insert(".spec.access") From 2be80be5bc724d2c2040ca93852d696c625f4a29 Mon Sep 17 00:00:00 2001 From: Jingyi Hu Date: Tue, 17 Sep 2024 06:42:37 +0000 Subject: [PATCH 09/12] fix: userID is an output field in BigQueryDataTransferConfig --- .../v1alpha1/bigquerydatatransferconfig_types.go | 6 +++--- .../v1alpha1/zz_generated.deepcopy.go | 10 +++++----- ...gs.bigquerydatatransfer.cnrm.cloud.google.com.yaml | 10 +++++----- .../v1alpha1/bigquerydatatransferconfig_types.go | 8 ++++++-- .../v1alpha1/zz_generated.deepcopy.go | 11 ++++++++--- .../bigquerydatatransfer_mappings.go | 4 ++-- 6 files changed, 29 insertions(+), 20 deletions(-) diff --git a/apis/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig_types.go b/apis/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig_types.go index bdafbcbe8b..8243b236d5 100644 --- a/apis/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig_types.go +++ b/apis/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig_types.go @@ -106,9 +106,6 @@ type BigQueryDataTransferConfigSpec struct { // Options customizing the data transfer schedule. ScheduleOptions *ScheduleOptions `json:"scheduleOptions,omitempty"` - // Deprecated. Unique ID of the user on whose behalf transfer is done. - UserID *int64 `json:"userID,omitempty"` - // Service account email. If this field is set, the transfer config will be created with this service account's credentials. // It requires that the requesting user calling this API has permissions to act as this service account. // Note that not all data sources support service account credentials when creating a transfer config. @@ -170,6 +167,9 @@ type BigQueryDataTransferConfigObservedState struct { // Output only. Data transfer modification time. Ignored by server on input. UpdateTime *string `json:"updateTime,omitempty"` + + // Deprecated. Unique ID of the user on whose behalf transfer is done. + UserID *int64 `json:"userID,omitempty"` } // +genclient diff --git a/apis/bigquerydatatransfer/v1alpha1/zz_generated.deepcopy.go b/apis/bigquerydatatransfer/v1alpha1/zz_generated.deepcopy.go index 2fab60ac4b..ce03cc4e41 100644 --- a/apis/bigquerydatatransfer/v1alpha1/zz_generated.deepcopy.go +++ b/apis/bigquerydatatransfer/v1alpha1/zz_generated.deepcopy.go @@ -102,6 +102,11 @@ func (in *BigQueryDataTransferConfigObservedState) DeepCopyInto(out *BigQueryDat *out = new(string) **out = **in } + if in.UserID != nil { + in, out := &in.UserID, &out.UserID + *out = new(int64) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BigQueryDataTransferConfigObservedState. @@ -180,11 +185,6 @@ func (in *BigQueryDataTransferConfigSpec) DeepCopyInto(out *BigQueryDataTransfer *out = new(ScheduleOptions) (*in).DeepCopyInto(*out) } - if in.UserID != nil { - in, out := &in.UserID, &out.UserID - *out = new(int64) - **out = **in - } if in.ServiceAccountRef != nil { in, out := &in.ServiceAccountRef, &out.ServiceAccountRef *out = new(v1beta1.IAMServiceAccountRef) diff --git a/config/crds/resources/apiextensions.k8s.io_v1_customresourcedefinition_bigquerydatatransferconfigs.bigquerydatatransfer.cnrm.cloud.google.com.yaml b/config/crds/resources/apiextensions.k8s.io_v1_customresourcedefinition_bigquerydatatransferconfigs.bigquerydatatransfer.cnrm.cloud.google.com.yaml index b684f5e98a..29567d7a7f 100644 --- a/config/crds/resources/apiextensions.k8s.io_v1_customresourcedefinition_bigquerydatatransferconfigs.bigquerydatatransfer.cnrm.cloud.google.com.yaml +++ b/config/crds/resources/apiextensions.k8s.io_v1_customresourcedefinition_bigquerydatatransferconfigs.bigquerydatatransfer.cnrm.cloud.google.com.yaml @@ -310,11 +310,6 @@ spec: description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' type: string type: object - userID: - description: Deprecated. Unique ID of the user on whose behalf transfer - is done. - format: int64 - type: integer required: - dataSourceID - datasetRef @@ -400,6 +395,11 @@ spec: description: Output only. Data transfer modification time. Ignored by server on input. type: string + userID: + description: Deprecated. Unique ID of the user on whose behalf + transfer is done. + format: int64 + type: integer type: object type: object required: diff --git a/pkg/clients/generated/apis/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig_types.go b/pkg/clients/generated/apis/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig_types.go index e122f8ed97..7303dc0550 100644 --- a/pkg/clients/generated/apis/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig_types.go +++ b/pkg/clients/generated/apis/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig_types.go @@ -125,9 +125,9 @@ type BigQueryDataTransferConfigSpec struct { // +optional ScheduleOptions *ConfigScheduleOptions `json:"scheduleOptions,omitempty"` - /* Deprecated. Unique ID of the user on whose behalf transfer is done. */ + /* Service account email. If this field is set, the transfer config will be created with this service account's credentials. It requires that the requesting user calling this API has permissions to act as this service account. Note that not all data sources support service account credentials when creating a transfer config. For the latest list of data sources, please refer to https://cloud.google.com/bigquery/docs/use-service-accounts. */ // +optional - UserID *int64 `json:"userID,omitempty"` + ServiceAccountRef *v1alpha1.ResourceRef `json:"serviceAccountRef,omitempty"` } type ConfigObservedStateStatus struct { @@ -154,6 +154,10 @@ type ConfigObservedStateStatus struct { /* Output only. Data transfer modification time. Ignored by server on input. */ // +optional UpdateTime *string `json:"updateTime,omitempty"` + + /* Deprecated. Unique ID of the user on whose behalf transfer is done. */ + // +optional + UserID *int64 `json:"userID,omitempty"` } type ConfigOwnerInfoStatus struct { diff --git a/pkg/clients/generated/apis/bigquerydatatransfer/v1alpha1/zz_generated.deepcopy.go b/pkg/clients/generated/apis/bigquerydatatransfer/v1alpha1/zz_generated.deepcopy.go index 836a32c67e..4b64999896 100644 --- a/pkg/clients/generated/apis/bigquerydatatransfer/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/clients/generated/apis/bigquerydatatransfer/v1alpha1/zz_generated.deepcopy.go @@ -147,9 +147,9 @@ func (in *BigQueryDataTransferConfigSpec) DeepCopyInto(out *BigQueryDataTransfer *out = new(ConfigScheduleOptions) (*in).DeepCopyInto(*out) } - if in.UserID != nil { - in, out := &in.UserID, &out.UserID - *out = new(int64) + if in.ServiceAccountRef != nil { + in, out := &in.ServiceAccountRef, &out.ServiceAccountRef + *out = new(k8sv1alpha1.ResourceRef) **out = **in } return @@ -276,6 +276,11 @@ func (in *ConfigObservedStateStatus) DeepCopyInto(out *ConfigObservedStateStatus *out = new(string) **out = **in } + if in.UserID != nil { + in, out := &in.UserID, &out.UserID + *out = new(int64) + **out = **in + } return } diff --git a/pkg/controller/direct/bigquerydatatransfer/bigquerydatatransfer_mappings.go b/pkg/controller/direct/bigquerydatatransfer/bigquerydatatransfer_mappings.go index 08e554fc33..0d98df3bc9 100644 --- a/pkg/controller/direct/bigquerydatatransfer/bigquerydatatransfer_mappings.go +++ b/pkg/controller/direct/bigquerydatatransfer/bigquerydatatransfer_mappings.go @@ -34,6 +34,7 @@ func BigQueryDataTransferConfigObservedState_FromProto(mapCtx *direct.MapContext out.State = direct.Enum_FromProto(mapCtx, in.GetState()) out.DatasetRegion = direct.LazyPtr(in.GetDatasetRegion()) out.OwnerInfo = UserInfo_FromProto(mapCtx, in.GetOwnerInfo()) + out.UserID = direct.LazyPtr(in.GetUserId()) return out } func BigQueryDataTransferConfigObservedState_ToProto(mapCtx *direct.MapContext, in *krm.BigQueryDataTransferConfigObservedState) *pb.TransferConfig { @@ -47,6 +48,7 @@ func BigQueryDataTransferConfigObservedState_ToProto(mapCtx *direct.MapContext, out.State = direct.Enum_ToProto[pb.TransferState](mapCtx, in.State) out.DatasetRegion = direct.ValueOf(in.DatasetRegion) out.OwnerInfo = UserInfo_ToProto(mapCtx, in.OwnerInfo) + out.UserId = direct.ValueOf(in.UserID) return out } func BigQueryDataTransferConfigSpec_FromProto(mapCtx *direct.MapContext, in *pb.TransferConfig) *krm.BigQueryDataTransferConfigSpec { @@ -64,7 +66,6 @@ func BigQueryDataTransferConfigSpec_FromProto(mapCtx *direct.MapContext, in *pb. out.ScheduleOptions = ScheduleOptions_FromProto(mapCtx, in.GetScheduleOptions()) out.DataRefreshWindowDays = direct.LazyPtr(in.GetDataRefreshWindowDays()) out.Disabled = direct.LazyPtr(in.GetDisabled()) - out.UserID = direct.LazyPtr(in.GetUserId()) if in.GetNotificationPubsubTopic() != "" { out.PubSubTopicRef = &refv1beta1.PubSubTopicRef{External: in.GetNotificationPubsubTopic()} } @@ -87,7 +88,6 @@ func BigQueryDataTransferConfigSpec_ToProto(mapCtx *direct.MapContext, in *krm.B out.ScheduleOptions = ScheduleOptions_ToProto(mapCtx, in.ScheduleOptions) out.DataRefreshWindowDays = direct.ValueOf(in.DataRefreshWindowDays) out.Disabled = direct.ValueOf(in.Disabled) - out.UserId = direct.ValueOf(in.UserID) if in.PubSubTopicRef != nil { out.NotificationPubsubTopic = in.PubSubTopicRef.External } From 0f14fe0d0513d28fe559b919e9365fa5113411e9 Mon Sep 17 00:00:00 2001 From: Jingyi Hu Date: Tue, 17 Sep 2024 07:00:50 +0000 Subject: [PATCH 10/12] chore: update test outputs --- ...ated_object_bigquerydatatransferconfig-salesforce.golden.yaml | 1 + ..._object_bigquerydatatransferconfig-scheduledquery.golden.yaml | 1 + tests/e2e/normalize.go | 1 + 3 files changed, 3 insertions(+) diff --git a/pkg/test/resourcefixture/testdata/basic/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig/bigquerydatatransferconfig-salesforce/_generated_object_bigquerydatatransferconfig-salesforce.golden.yaml b/pkg/test/resourcefixture/testdata/basic/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig/bigquerydatatransferconfig-salesforce/_generated_object_bigquerydatatransferconfig-salesforce.golden.yaml index 6385e523c3..538b93c3ce 100644 --- a/pkg/test/resourcefixture/testdata/basic/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig/bigquerydatatransferconfig-salesforce/_generated_object_bigquerydatatransferconfig-salesforce.golden.yaml +++ b/pkg/test/resourcefixture/testdata/basic/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig/bigquerydatatransferconfig-salesforce/_generated_object_bigquerydatatransferconfig-salesforce.golden.yaml @@ -40,3 +40,4 @@ status: ownerInfo: email: user@google.com updateTime: "1970-01-01T00:00:00Z" + userID: "0000000000000000000" diff --git a/pkg/test/resourcefixture/testdata/basic/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig/bigquerydatatransferconfig-scheduledquery/_generated_object_bigquerydatatransferconfig-scheduledquery.golden.yaml b/pkg/test/resourcefixture/testdata/basic/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig/bigquerydatatransferconfig-scheduledquery/_generated_object_bigquerydatatransferconfig-scheduledquery.golden.yaml index 776cbf2549..6ef1f2141b 100644 --- a/pkg/test/resourcefixture/testdata/basic/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig/bigquerydatatransferconfig-scheduledquery/_generated_object_bigquerydatatransferconfig-scheduledquery.golden.yaml +++ b/pkg/test/resourcefixture/testdata/basic/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig/bigquerydatatransferconfig-scheduledquery/_generated_object_bigquerydatatransferconfig-scheduledquery.golden.yaml @@ -42,3 +42,4 @@ status: ownerInfo: email: user@google.com updateTime: "1970-01-01T00:00:00Z" + userID: "0000000000000000000" diff --git a/tests/e2e/normalize.go b/tests/e2e/normalize.go index 0c8a09fc96..f27bccff9c 100644 --- a/tests/e2e/normalize.go +++ b/tests/e2e/normalize.go @@ -177,6 +177,7 @@ func normalizeKRMObject(t *testing.T, u *unstructured.Unstructured, project test // Specific to BigQueryDataTransferConfig visitor.replacePaths[".status.observedState.nextRunTime"] = "1970-01-01T00:00:00Z" visitor.replacePaths[".status.observedState.ownerInfo.email"] = "user@google.com" + visitor.replacePaths[".status.observedState.userID"] = "0000000000000000000" visitor.removePaths.Insert(".status.observedState.state") // data transfer run state, which depends on timing // TODO: This should not be needed, we want to avoid churning the kube objects From ad942bb424c62a2506235ced8c77bd2fc834d8af Mon Sep 17 00:00:00 2001 From: Jingyi Hu Date: Wed, 18 Sep 2024 01:38:06 +0000 Subject: [PATCH 11/12] test: restrict custom normalizations to BigQueryDataTransferConfig --- tests/e2e/normalize.go | 10 ++++++---- tests/e2e/unified_test.go | 7 ++++++- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/tests/e2e/normalize.go b/tests/e2e/normalize.go index f27bccff9c..61f55106ec 100644 --- a/tests/e2e/normalize.go +++ b/tests/e2e/normalize.go @@ -175,10 +175,12 @@ func normalizeKRMObject(t *testing.T, u *unstructured.Unstructured, project test visitor.replacePaths[".status.observedState.cloudResource.serviceAccountID"] = "bqcx-${projectNumber}-abcd@gcp-sa-bigquery-condel.iam.gserviceaccount.com" // Specific to BigQueryDataTransferConfig - visitor.replacePaths[".status.observedState.nextRunTime"] = "1970-01-01T00:00:00Z" - visitor.replacePaths[".status.observedState.ownerInfo.email"] = "user@google.com" - visitor.replacePaths[".status.observedState.userID"] = "0000000000000000000" - visitor.removePaths.Insert(".status.observedState.state") // data transfer run state, which depends on timing + if u.GetKind() == "BigQueryDataTransferConfig" { + visitor.replacePaths[".status.observedState.nextRunTime"] = "1970-01-01T00:00:00Z" + visitor.replacePaths[".status.observedState.ownerInfo.email"] = "user@google.com" + visitor.replacePaths[".status.observedState.userID"] = "0000000000000000000" + visitor.removePaths.Insert(".status.observedState.state") // data transfer run state, which depends on timing + } // TODO: This should not be needed, we want to avoid churning the kube objects visitor.sortSlices.Insert(".spec.access") diff --git a/tests/e2e/unified_test.go b/tests/e2e/unified_test.go index aa6924e0a1..f17972f2d5 100644 --- a/tests/e2e/unified_test.go +++ b/tests/e2e/unified_test.go @@ -768,7 +768,12 @@ func runScenario(ctx context.Context, t *testing.T, testPause bool, fixture reso addReplacement("nextRunTime", "2024-04-01T12:34:56.123456Z") addReplacement("ownerInfo.email", "user@google.com") addReplacement("userId", "0000000000000000000") - jsonMutators = append(jsonMutators, func(obj map[string]any) { // special handling because the field includes dot + jsonMutators = append(jsonMutators, func(obj map[string]any) { + if _, found, err := unstructured.NestedString(obj, "destinationDatasetId"); err != nil || !found { + // This is a hack to only run this mutator for BigQueryDataTransferConfig objects. + return + } + // special handling because the field includes dot if _, found, _ := unstructured.NestedString(obj, "params", "connector.authentication.oauth.clientId"); found { if err := unstructured.SetNestedField(obj, "client-id", "params", "connector.authentication.oauth.clientId"); err != nil { t.Fatal(err) From f8daaa526b3cd01bacf2d8afd14be2cee021d8f5 Mon Sep 17 00:00:00 2001 From: Jingyi Hu Date: Wed, 18 Sep 2024 20:30:47 +0000 Subject: [PATCH 12/12] chore: update bigquerydatatransfer controller to use latest template --- .../bigquerydatatransferconfig_controller.go | 12 +++--------- .../bigquerydatatransferconfig-salesforce/_http.log | 5 +++++ .../_http.log | 5 +++++ 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/pkg/controller/direct/bigquerydatatransfer/bigquerydatatransferconfig_controller.go b/pkg/controller/direct/bigquerydatatransfer/bigquerydatatransferconfig_controller.go index f913de9492..4aab39ebc0 100644 --- a/pkg/controller/direct/bigquerydatatransfer/bigquerydatatransferconfig_controller.go +++ b/pkg/controller/direct/bigquerydatatransfer/bigquerydatatransferconfig_controller.go @@ -58,16 +58,10 @@ type model struct { func (m *model) client(ctx context.Context) (*gcp.Client, error) { var opts []option.ClientOption - if m.config.UserAgent != "" { - opts = append(opts, option.WithUserAgent(m.config.UserAgent)) - } - if m.config.HTTPClient != nil { - opts = append(opts, option.WithHTTPClient(m.config.HTTPClient)) - } - if m.config.UserProjectOverride && m.config.BillingProject != "" { - opts = append(opts, option.WithQuotaProject(m.config.BillingProject)) + opts, err := m.config.RESTClientOptions() + if err != nil { + return nil, err } - gcpClient, err := gcp.NewRESTClient(ctx, opts...) if err != nil { return nil, fmt.Errorf("building bigquerydatatransfer client: %w", err) diff --git a/pkg/test/resourcefixture/testdata/basic/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig/bigquerydatatransferconfig-salesforce/_http.log b/pkg/test/resourcefixture/testdata/basic/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig/bigquerydatatransferconfig-salesforce/_http.log index b16761ca38..679c1d4fb3 100644 --- a/pkg/test/resourcefixture/testdata/basic/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig/bigquerydatatransferconfig-salesforce/_http.log +++ b/pkg/test/resourcefixture/testdata/basic/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig/bigquerydatatransferconfig-salesforce/_http.log @@ -151,6 +151,7 @@ X-Xss-Protection: 0 POST https://bigquerydatatransfer.googleapis.com/v1/projects/${projectId}/locations/us-central1/transferConfigs?%24alt=json%3Benum-encoding%3Dint Content-Type: application/json +User-Agent: kcc/controller-manager x-goog-request-params: parent=projects%2F${projectId}%2Flocations%2Fus-central1 { @@ -201,6 +202,7 @@ X-Xss-Protection: 0 GET https://bigquerydatatransfer.googleapis.com/v1/projects/${projectId}/locations/us-central1/transferConfigs/${transferConfigID}?%24alt=json%3Benum-encoding%3Dint Content-Type: application/json +User-Agent: kcc/controller-manager x-goog-request-params: name=projects%2F${projectId}%2Flocations%2Fus-central1%2FtransferConfigs%2F${transferConfigID} 200 OK @@ -239,6 +241,7 @@ X-Xss-Protection: 0 PATCH https://bigquerydatatransfer.googleapis.com/v1/projects/${projectId}/locations/us-central1/transferConfigs/${transferConfigID}?%24alt=json%3Benum-encoding%3Dint&updateMask=disabled%2CdisplayName%2Cparams Content-Type: application/json +User-Agent: kcc/controller-manager x-goog-request-params: transfer_config.name=projects%2F${projectId}%2Flocations%2Fus-central1%2FtransferConfigs%2F${transferConfigID} { @@ -299,6 +302,7 @@ X-Xss-Protection: 0 GET https://bigquerydatatransfer.googleapis.com/v1/projects/${projectId}/locations/us-central1/transferConfigs/${transferConfigID}?%24alt=json%3Benum-encoding%3Dint Content-Type: application/json +User-Agent: kcc/controller-manager x-goog-request-params: name=projects%2F${projectId}%2Flocations%2Fus-central1%2FtransferConfigs%2F${transferConfigID} 200 OK @@ -337,6 +341,7 @@ X-Xss-Protection: 0 DELETE https://bigquerydatatransfer.googleapis.com/v1/projects/${projectId}/locations/us-central1/transferConfigs/${transferConfigID}?%24alt=json%3Benum-encoding%3Dint Content-Type: application/json +User-Agent: kcc/controller-manager x-goog-request-params: name=projects%2F${projectId}%2Flocations%2Fus-central1%2FtransferConfigs%2F${transferConfigID} 200 OK diff --git a/pkg/test/resourcefixture/testdata/basic/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig/bigquerydatatransferconfig-scheduledquery/_http.log b/pkg/test/resourcefixture/testdata/basic/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig/bigquerydatatransferconfig-scheduledquery/_http.log index d1818e2a22..bf9fcf2aee 100644 --- a/pkg/test/resourcefixture/testdata/basic/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig/bigquerydatatransferconfig-scheduledquery/_http.log +++ b/pkg/test/resourcefixture/testdata/basic/bigquerydatatransfer/v1alpha1/bigquerydatatransferconfig/bigquerydatatransferconfig-scheduledquery/_http.log @@ -242,6 +242,7 @@ X-Xss-Protection: 0 POST https://bigquerydatatransfer.googleapis.com/v1/projects/${projectId}/locations/us-central1/transferConfigs?%24alt=json%3Benum-encoding%3Dint&serviceAccountName=gsa-${uniqueId}%40${projectId}.iam.gserviceaccount.com Content-Type: application/json +User-Agent: kcc/controller-manager x-goog-request-params: parent=projects%2F${projectId}%2Flocations%2Fus-central1 { @@ -292,6 +293,7 @@ X-Xss-Protection: 0 GET https://bigquerydatatransfer.googleapis.com/v1/projects/${projectId}/locations/us-central1/transferConfigs/${transferConfigID}?%24alt=json%3Benum-encoding%3Dint Content-Type: application/json +User-Agent: kcc/controller-manager x-goog-request-params: name=projects%2F${projectId}%2Flocations%2Fus-central1%2FtransferConfigs%2F${transferConfigID} 200 OK @@ -330,6 +332,7 @@ X-Xss-Protection: 0 PATCH https://bigquerydatatransfer.googleapis.com/v1/projects/${projectId}/locations/us-central1/transferConfigs/${transferConfigID}?%24alt=json%3Benum-encoding%3Dint&serviceAccountName=gsa-${uniqueId}%40${projectId}.iam.gserviceaccount.com&updateMask=displayName%2Cparams%2Cschedule Content-Type: application/json +User-Agent: kcc/controller-manager x-goog-request-params: transfer_config.name=projects%2F${projectId}%2Flocations%2Fus-central1%2FtransferConfigs%2F${transferConfigID} { @@ -389,6 +392,7 @@ X-Xss-Protection: 0 GET https://bigquerydatatransfer.googleapis.com/v1/projects/${projectId}/locations/us-central1/transferConfigs/${transferConfigID}?%24alt=json%3Benum-encoding%3Dint Content-Type: application/json +User-Agent: kcc/controller-manager x-goog-request-params: name=projects%2F${projectId}%2Flocations%2Fus-central1%2FtransferConfigs%2F${transferConfigID} 200 OK @@ -428,6 +432,7 @@ X-Xss-Protection: 0 DELETE https://bigquerydatatransfer.googleapis.com/v1/projects/${projectId}/locations/us-central1/transferConfigs/${transferConfigID}?%24alt=json%3Benum-encoding%3Dint Content-Type: application/json +User-Agent: kcc/controller-manager x-goog-request-params: name=projects%2F${projectId}%2Flocations%2Fus-central1%2FtransferConfigs%2F${transferConfigID} 200 OK