go rewrite - copy script and adjustments for compute (#10929)

mmv1/api/compiler.go

@@ -27,5 +27,5 @@ func Compile(yamlPath string, obj interface{}) {
 	}
 
 	yamlValidator := google.YamlValidator{}
-	yamlValidator.Parse(objYaml, obj)
+	yamlValidator.Parse(objYaml, obj, yamlPath)
 }

mmv1/api/resource.rb

@@ -393,6 +393,12 @@ def all_nested_properties(props)
       nested
     end
 
+    def convert_go_file(file)
+      dir, base = File.split(file)
+      base.slice! '.erb'
+      "#{dir}/go/#{base}.tmpl"
+    end
+
     # All settable properties in the resource.
     # Fingerprints aren't *really" settable properties, but they behave like one.
     # At Create, they have no value but they can just be read in anyways, and after a Read

mmv1/api/type.rb

@@ -583,7 +583,9 @@ def nested_properties
 
       def item_type_class
         return @item_type \
-          if @item_type.instance_of?(Class)
+          if @item_type.instance_of?(Class) \
+            || @item_type.is_a?(Api::Type::ResourceRef) \
+            || @item_type.is_a?(Api::Type::Enum)
 
         Object.const_get(@item_type)
       end

mmv1/compiler.rb

@@ -36,6 +36,7 @@
 products_to_generate = nil
 all_products = false
 yaml_dump = false
+go_yaml = false
 generate_code = true
 generate_docs = true
 output_path = nil
@@ -94,6 +95,9 @@
   opt.on('--openapi-generate', 'Generate MMv1 YAML from openapi directory (Experimental)') do
     openapi_generate = true
   end
+  opt.on('--go-yaml', 'Generate MMv1 Go YAML from Ruby YAML') do
+    go_yaml = true
+  end
 end.parse!
 # rubocop:enable Metrics/BlockLength
 
@@ -270,7 +274,8 @@
     product_name,
     yaml_dump,
     generate_code,
-    generate_docs
+    generate_docs,
+    go_yaml
   )
 
   # we need to preserve a single provider instance to use outside of this loop.

mmv1/description-copy.go

@@ -0,0 +1,147 @@
+package main
+
+import (
+	"bufio"
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+)
+
+// Used to copy/paste text from Ruby -> Go YAML files
+func CopyText(identifier string) {
+	var allProductFiles []string = make([]string, 0)
+	files, err := filepath.Glob("products/**/go_product.yaml")
+	if err != nil {
+		return
+	}
+	for _, filePath := range files {
+		dir := filepath.Dir(filePath)
+		allProductFiles = append(allProductFiles, fmt.Sprintf("products/%s", filepath.Base(dir)))
+	}
+
+	for _, productPath := range allProductFiles {
+		// Gather go and ruby file pairs
+		yamlMap := make(map[string][]string)
+		yamlPaths, err := filepath.Glob(fmt.Sprintf("%s/*", productPath))
+		if err != nil {
+			log.Fatalf("Cannot get yaml files: %v", err)
+		}
+		for _, yamlPath := range yamlPaths {
+			if strings.HasSuffix(yamlPath, "_new") {
+				continue
+			}
+			fileName := filepath.Base(yamlPath)
+			baseName, found := strings.CutPrefix(fileName, "go_")
+			if yamlMap[baseName] == nil {
+				yamlMap[baseName] = make([]string, 2)
+			}
+			if found {
+				yamlMap[baseName][1] = yamlPath
+			} else {
+				yamlMap[baseName][0] = yamlPath
+			}
+		}
+
+		for _, files := range yamlMap {
+			rubyPath := files[0]
+			goPath := files[1]
+			var text []string
+			currText := ""
+			recording := false
+
+			if strings.Contains(rubyPath, "product.yaml") {
+				// log.Printf("skipping %s", rubyPath)
+				continue
+			}
+
+			// Ready Ruby yaml
+			file, _ := os.Open(rubyPath)
+			defer file.Close()
+			scanner := bufio.NewScanner(file)
+			for scanner.Scan() {
+				line := scanner.Text()
+				if strings.Contains(line, identifier) && !strings.HasPrefix(strings.TrimSpace(line), "#") {
+					currText = strings.SplitAfter(line, identifier)[1]
+					recording = true
+				} else if recording {
+					if terminateText(line) {
+						text = append(text, currText)
+						currText = ""
+						recording = false
+					} else {
+						currText = fmt.Sprintf("%s\n%s", currText, line)
+					}
+				}
+			}
+			if recording {
+				text = append(text, currText)
+			}
+
+			// Read Go yaml while writing to a temp file
+			index := 0
+			firstLine := true
+			newFilePath := fmt.Sprintf("%s_new", goPath)
+			fo, _ := os.Create(newFilePath)
+			w := bufio.NewWriter(fo)
+			file, _ = os.Open(goPath)
+			defer file.Close()
+			scanner = bufio.NewScanner(file)
+			for scanner.Scan() {
+				line := scanner.Text()
+				if firstLine {
+					if line != "NOT CONVERTED - RUN YAML MODE" {
+						// log.Printf("skipping %s", goPath)
+						break
+					} else {
+						firstLine = false
+						continue
+					}
+				}
+				if strings.Contains(line, identifier) {
+					if index >= len(text) {
+						log.Printf("did not replace %s correctly! Is the file named correctly?", goPath)
+						w.Flush()
+						break
+					}
+					line = fmt.Sprintf("%s%s", line, text[index])
+					index += 1
+				}
+				w.WriteString(fmt.Sprintf("%s\n", line))
+			}
+
+			if !firstLine {
+				if index != len(text) {
+					log.Printf("potential issue with %s, only completed %d index out of %d replacements", goPath, index, len(text))
+				}
+				if err = w.Flush(); err != nil {
+					panic(err)
+				}
+
+				// Overwrite original file with temp
+				os.Rename(newFilePath, goPath)
+			} else {
+				os.Remove(newFilePath)
+			}
+		}
+
+	}
+
+}
+
+// quick and dirty logic to determine if a description/note is terminated
+func terminateText(line string) bool {
+	terminalStrings := []string{
+		"!ruby/",
+	}
+
+	for _, t := range terminalStrings {
+		if strings.Contains(line, t) {
+			return true
+		}
+	}
+
+	return regexp.MustCompile(`^\s*[a-z_]+:[\s$]*`).MatchString(line)
+}

mmv1/google/yaml_validator.go

@@ -22,12 +22,12 @@ import (
 // A helper class to validate contents coming from YAML files.
 type YamlValidator struct{}
 
-func (v *YamlValidator) Parse(content []byte, obj interface{}) {
+func (v *YamlValidator) Parse(content []byte, obj interface{}, yamlPath string) {
 	// TODO(nelsonjr): Allow specifying which symbols to restrict it further.
 	// But it requires inspecting all configuration files for symbol sources,
 	// such as Enum values. Leaving it as a nice-to-have for the future.
 	if err := yaml.Unmarshal(content, obj); err != nil {
-		log.Fatalf("Cannot unmarshal data: %v", err)
+		log.Fatalf("Cannot unmarshal data from file %s: %v", yamlPath, err)
 	}
 }
 

mmv1/main.go

@@ -27,8 +27,18 @@ var version = flag.String("version", "", "optional version name. If specified, t
 
 var product = flag.String("product", "", "optional product name. If specified, the resources under the specific product will be generated. Otherwise, resources under all products will be generated.")
 
+// Example usage: --yaml
+var yamlMode = flag.Bool("yaml", false, "strictly copy text over from ruby yaml to go yaml")
+
 func main() {
 	flag.Parse()
+
+	if *yamlMode {
+		CopyText("description:")
+		CopyText("note:")
+		return
+	}
+
 	var generateCode = true
 	var generateDocs = true
 

mmv1/products/compute/ForwardingRule.yaml

@@ -220,6 +220,12 @@ custom_code: !ruby/object:Provider::Terraform::CustomCode
 custom_diff: [
   'forwardingRuleCustomizeDiff',
 ]
+virtual_fields:
+  - !ruby/object:Api::Type::Boolean
+    name: recreate_closed_psc
+    description:
+      This is used in PSC consumer ForwardingRule to make terraform recreate the ForwardingRule when the status is closed
+    default_value: false
 parameters:
   - !ruby/object:Api::Type::ResourceRef
     name: 'region'
@@ -656,9 +662,3 @@ properties:
       - :IPV6
     immutable: true
     default_from_api: true
-virtual_fields:
-  - !ruby/object:Api::Type::Boolean
-    name: recreate_closed_psc
-    description:
-      This is used in PSC consumer ForwardingRule to make terraform recreate the ForwardingRule when the status is closed
-    default_value: false