chore: add presubmit check for generated types for direct resources

Makefile

@@ -391,4 +391,14 @@ BILLING_ACCOUNT ?= 010E8D-490B6B-088E1C
 operator-e2e-tests:
 	export TEST_ORG_ID=${ORG_ID}
 	export TEST_BILLING_ACCOUNT_ID=${BILLING_ACCOUNT}
-	cd operator/tests/e2e/ && go test --project-id=${PROJECT_ID}
+	cd operator/tests/e2e/ && go test --project-id=${PROJECT_ID}
+
+# Generate Go types for direct resources specified in the config files located under `dev/tools/controllerbuilder/config`.
+.PHONY: generate-types
+generate-types:
+	cd dev/tools/controllerbuilder && \
+	./generate-proto.sh && \
+	for config in config/*.yaml; do \
+		go run . generate-types --config $$config; \
+	done
+	dev/tasks/fix-gofmt 

dev/tools/controllerbuilder/config/bigquerydatatransfer.yaml

@@ -0,0 +1,21 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+service: google.cloud.bigquery.datatransfer.v1
+apiVersion: bigquerydatatransfer.cnrm.cloud.google.com/v1beta1
+generateMapper: true
+resources:
+  - kind: BigQueryDataTransferConfig
+    protoName: TransferConfig
+    skipScaffoldFiles: true # files were scaffolded using a previous template, making them incompatible with the new scaffolding template.

dev/tools/controllerbuilder/generate-proto.sh

@@ -44,6 +44,7 @@ fi
 
 
 protoc --include_imports --include_source_info \
+    --experimental_allow_proto3_optional \
     -I ${THIRD_PARTY}/googleapis/ \
     -I ${REPO_ROOT}/mockgcp/apis \
     ${REPO_ROOT}/mockgcp/apis/mockgcp/cloud/networkconnectivity/*/*.proto \

dev/tools/controllerbuilder/pkg/codegen/config.go

@@ -0,0 +1,53 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package codegen
+
+import (
+	"fmt"
+	"os"
+
+	"gopkg.in/yaml.v3"
+)
+
+type ServiceConfig struct {
+	Service        string           `yaml:"service"`
+	APIVersion     string           `yaml:"apiVersion"`
+	GenerateMapper bool             `yaml:"generateMapper"`
+	Resources      []ResourceConfig `yaml:"resources"`
+}
+
+type ResourceConfig struct {
+	Kind              string `yaml:"kind"`
+	ProtoName         string `yaml:"protoName"`
+	SkipScaffoldFiles bool   `yaml:"skipScaffoldFiles"`
+}
+
+func LoadConfig(configPath string) (*ServiceConfig, error) {
+	if configPath == "" {
+		return nil, nil
+	}
+
+	data, err := os.ReadFile(configPath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read config file: %v", err)
+	}
+
+	var config ServiceConfig
+	if err := yaml.Unmarshal(data, &config); err != nil {
+		return nil, fmt.Errorf("failed to parse config file: %v", err)
+	}
+
+	return &config, nil
+}

dev/tools/controllerbuilder/pkg/commands/generatemapper/generatemappercommand.go

@@ -68,22 +68,10 @@ func BuildCommand(baseOptions *options.GenerateOptions) *cobra.Command {
 		Use:   "generate-mapper",
 		Short: "generate mapper functions for a proto service",
 		PreRunE: func(cmd *cobra.Command, args []string) error {
-			if opt.ServiceName == "" {
-				return fmt.Errorf("ServiceName is required")
-			}
-			if opt.GenerateOptions.ProtoSourcePath == "" {
-				return fmt.Errorf("ProtoSourcePath is required")
-			}
-			if opt.APIGoPackagePath == "" {
-				return fmt.Errorf("GoPackagePath is required")
-			}
-			if opt.OutputMapperDirectory == "" {
-				return fmt.Errorf("OutputMapperDirectory is required")
-			}
-			if opt.APIVersion == "" {
-				return fmt.Errorf("APIVersion is required")
+			if err := opt.loadAndApplyConfig(); err != nil {
+				return err
 			}
-			return nil
+			return opt.validate()
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {
 			ctx := cmd.Context()
@@ -154,3 +142,43 @@ func RunGenerateMapper(ctx context.Context, o *GenerateMapperOptions) error {
 	return nil
 
 }
+
+func (o *GenerateMapperOptions) loadAndApplyConfig() error {
+	if o.ConfigFilePath == "" {
+		return nil
+	}
+	config, err := codegen.LoadConfig(o.ConfigFilePath)
+	if err != nil {
+		return fmt.Errorf("loading service config: %w", err)
+	}
+	if config == nil {
+		return nil
+	}
+
+	if !config.GenerateMapper {
+		return fmt.Errorf("mapper generation is disabled for this service in config file %s", o.ConfigFilePath)
+	}
+
+	o.ServiceName = config.Service
+	o.APIVersion = config.APIVersion
+	return nil
+}
+
+func (o *GenerateMapperOptions) validate() error {
+	if o.ServiceName == "" {
+		return fmt.Errorf("ServiceName is required")
+	}
+	if o.GenerateOptions.ProtoSourcePath == "" {
+		return fmt.Errorf("ProtoSourcePath is required")
+	}
+	if o.APIGoPackagePath == "" {
+		return fmt.Errorf("GoPackagePath is required")
+	}
+	if o.OutputMapperDirectory == "" {
+		return fmt.Errorf("OutputMapperDirectory is required")
+	}
+	if o.APIVersion == "" {
+		return fmt.Errorf("APIVersion is required")
+	}
+	return nil
+}

dev/tools/controllerbuilder/pkg/commands/generatetypes/generatetypescommand.go

@@ -19,7 +19,6 @@ import (
 	"fmt"
 	"os"
 	"strings"
-	"unicode"
 
 	"github.com/GoogleCloudPlatform/k8s-config-connector/dev/tools/controllerbuilder/pkg/codegen"
 	"github.com/GoogleCloudPlatform/k8s-config-connector/dev/tools/controllerbuilder/pkg/options"
@@ -42,8 +41,9 @@ type GenerateCRDOptions struct {
 }
 
 type Resource struct {
-	Kind      string
-	ProtoName string
+	Kind              string
+	ProtoName         string
+	SkipScaffoldFiles bool
 }
 
 type ResourceList []Resource
@@ -84,7 +84,7 @@ func (o *GenerateCRDOptions) InitDefaults() error {
 func (o *GenerateCRDOptions) BindFlags(cmd *cobra.Command) {
 	cmd.Flags().StringVar(&o.OutputAPIDirectory, "output-api", o.OutputAPIDirectory, "base directory for writing APIs")
 	cmd.Flags().Var(&o.Resources, "resource", "the KRM Kind and the equivalent proto resource separated with a colon.  e.g. for resource google.storage.v1.Bucket, the flag should be `StorageBucket:Bucket`.  Can be specified multiple times.")
-	cmd.Flags().BoolVar(&o.SkipScaffoldFiles, "skip-scaffold-files", false, "skip generating scaffold files (types, refs, and identity)")
+	cmd.Flags().BoolVar(&o.SkipScaffoldFiles, "skip-scaffold-files", false, "skip generating scaffold files (types, refs, and identity) for all resources generated by this command")
 }
 
 func BuildCommand(baseOptions *options.GenerateOptions) *cobra.Command {
@@ -101,16 +101,10 @@ func BuildCommand(baseOptions *options.GenerateOptions) *cobra.Command {
 		Use:   "generate-types",
 		Short: "generate KRM types for a proto service",
 		PreRunE: func(cmd *cobra.Command, args []string) error {
-			if opt.ServiceName == "" {
-				return fmt.Errorf("`service` is required")
-			}
-			if opt.GenerateOptions.ProtoSourcePath == "" {
-				return fmt.Errorf("`proto-source-path` is required")
-			}
-			if len(opt.Resources) == 0 {
-				return fmt.Errorf("`--resource` is required")
+			if err := opt.loadAndApplyConfig(); err != nil {
+				return err
 			}
-			return nil
+			return opt.validate()
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {
 			ctx := cmd.Context()
@@ -129,8 +123,6 @@ func BuildCommand(baseOptions *options.GenerateOptions) *cobra.Command {
 func RunGenerateCRD(ctx context.Context, o *GenerateCRDOptions) error {
 	log := klog.FromContext(ctx)
 
-	// o.ResourceProtoName = capitalizeFirstRune(o.ResourceProtoName)
-
 	gv, err := schema.ParseGroupVersion(o.APIVersion)
 	if err != nil {
 		return fmt.Errorf("APIVersion %q is not valid: %w", o.APIVersion, err)
@@ -173,7 +165,8 @@ func RunGenerateCRD(ctx context.Context, o *GenerateCRDOptions) error {
 			return err
 		}
 
-		if o.SkipScaffoldFiles {
+		skipScaffold := o.SkipScaffoldFiles || resource.SkipScaffoldFiles
+		if skipScaffold {
 			log.Info("skipping scaffolding type, refs and identity files", "resource", resource.ProtoName)
 		} else {
 			kind := resource.Kind
@@ -216,11 +209,39 @@ func RunGenerateCRD(ctx context.Context, o *GenerateCRDOptions) error {
 	return nil
 }
 
-func capitalizeFirstRune(s string) string {
-	if s == "" {
-		return s
+func (o *GenerateCRDOptions) loadAndApplyConfig() error {
+	if o.ConfigFilePath == "" {
+		return nil
 	}
-	runes := []rune(s)
-	runes[0] = unicode.ToUpper(runes[0])
-	return string(runes)
+	config, err := codegen.LoadConfig(o.ConfigFilePath)
+	if err != nil {
+		return fmt.Errorf("loading service config: %w", err)
+	}
+	if config == nil {
+		return nil
+	}
+
+	o.ServiceName = config.Service
+	o.APIVersion = config.APIVersion
+	for _, res := range config.Resources {
+		o.Resources = append(o.Resources, Resource{
+			Kind:              res.Kind,
+			ProtoName:         res.ProtoName,
+			SkipScaffoldFiles: res.SkipScaffoldFiles,
+		})
+	}
+	return nil
+}
+
+func (o *GenerateCRDOptions) validate() error {
+	if o.ServiceName == "" {
+		return fmt.Errorf("`service` is required")
+	}
+	if o.GenerateOptions.ProtoSourcePath == "" {
+		return fmt.Errorf("`proto-source-path` is required")
+	}
+	if len(o.Resources) == 0 {
+		return fmt.Errorf("`--resource` is required")
+	}
+	return nil
 }

dev/tools/controllerbuilder/pkg/options/generateoptions.go

@@ -25,6 +25,7 @@ type GenerateOptions struct {
 	ProtoSourcePath string
 	ServiceName     string
 	APIVersion      string
+	ConfigFilePath  string
 }
 
 func (o *GenerateOptions) InitDefaults() error {
@@ -40,6 +41,7 @@ func (o *GenerateOptions) BindPersistentFlags(cmd *cobra.Command) {
 	cmd.PersistentFlags().StringVar(&o.ProtoSourcePath, "proto-source-path", o.ProtoSourcePath, "path to (compiled) proto for APIs")
 	cmd.PersistentFlags().StringVarP(&o.APIVersion, "api-version", "v", o.APIVersion, "the KRM API version. used to import the KRM API")
 	cmd.PersistentFlags().StringVarP(&o.ServiceName, "service", "s", o.ServiceName, "the GCP service name")
+	cmd.PersistentFlags().StringVar(&o.ConfigFilePath, "config", "", "path to service config file, the config file will override other flags")
 }
 
 func RepoRoot() (string, error) {