From bd51b4ece47bd72a8129abea2ddb90b8afb4e6c2 Mon Sep 17 00:00:00 2001 From: Cody Oss Date: Thu, 25 Apr 2024 17:08:39 -0500 Subject: [PATCH] fix: split large compute file up This is a followup to #2544. That change reduced the size of the compute client from 9.4M to 6.7. This change now splits the compute client into three files ranging from 2.7-1.8M each. This should allow editors and tooling to better be able to process the client. --- google-api-go-generator/gen.go | 111 ++++++++++++++++++++++++++++++--- 1 file changed, 103 insertions(+), 8 deletions(-) diff --git a/google-api-go-generator/gen.go b/google-api-go-generator/gen.go index b2cffe6051c..0aefd0e8097 100644 --- a/google-api-go-generator/gen.go +++ b/google-api-go-generator/gen.go @@ -32,6 +32,8 @@ const ( googleDiscoveryURL = "https://www.googleapis.com/discovery/v1/apis" googleDefaultUniverse = "googleapis.com" universeDomainPlaceholder = "UNIVERSE_DOMAIN" + + splitFileSeperator = `// =*=*"` ) var ( @@ -75,6 +77,10 @@ var skipAPIGeneration = map[string]bool{ "datalineage:v1": true, } +var apisToSplit = map[string]bool{ + "compute": true, +} + // API represents an API to generate, as well as its state while it's // generating. type API struct { @@ -226,7 +232,7 @@ func getAPIs() []*API { } else { bytes = slurpURL(*apisURL) if *publicOnly { - if err := writeFile(apiListFile, bytes); err != nil { + if err := writeFile(apiListFile, "", bytes); err != nil { log.Fatal(err) } } @@ -300,7 +306,7 @@ func apiFromFile(file string) (*API, error) { func checkAndUpdateSpecFile(file string, contents []byte) error { // check if file exists if _, err := os.Stat(file); os.IsNotExist(err) { - return writeFile(file, contents) + return writeFile(file, "", contents) } existing, err := os.ReadFile(file) if err != nil { @@ -309,7 +315,7 @@ func checkAndUpdateSpecFile(file string, contents []byte) error { if err := isNewerRevision(existing, contents); err != nil { return err } - return writeFile(file, contents) + return writeFile(file, "", contents) } // isNewerRevision returns nil if the contents of new has a newer revision than @@ -331,7 +337,7 @@ func isNewerRevision(old []byte, new []byte) error { return nil } -func writeFile(file string, contents []byte) error { +func writeFile(file, pkg string, contents []byte) error { // Don't write it if the contents are identical. existing, err := os.ReadFile(file) if err == nil && (bytes.Equal(existing, contents) || basicallyEqual(existing, contents)) { @@ -341,7 +347,42 @@ func writeFile(file string, contents []byte) error { if err = os.MkdirAll(outdir, 0755); err != nil { return fmt.Errorf("failed to Mkdir %s: %v", outdir, err) } - return os.WriteFile(file, contents, 0644) + // Don't try to split spec files, json or non-allowlisted packages + if pkg == "" || !apisToSplit[pkg] { + return os.WriteFile(file, contents, 0644) + } + + // Split generated file out into multiple + bs := bytes.Split(contents, []byte(splitFileSeperator)) + for i, b := range bs { + var name string + var newB []byte + if i == 0 { + // For the base case, use the provided inputs as is + name = file + newB = b + } else { + // determine the new file name + base := filepath.Dir(file) + fileNum := i + 1 + name = base + string(filepath.Separator) + pkg + fmt.Sprint(fileNum) + "-gen.go" + + // prepend file header, package, and imports + var buf bytes.Buffer + // Size the buffer so there is room for the header and contents + buf.Grow(len(b) + 500) + splitFileHeading(&buf, pkg) + _, err := buf.Write(b) + if err != nil { + return err + } + newB = buf.Bytes() + } + if err := os.WriteFile(name, newB, 0644); err != nil { + return err + } + } + return nil } var ignoreLines = regexp.MustCompile(`(?m)^\s+"(?:etag|revision)": ".+\n`) @@ -589,7 +630,7 @@ func (a *API) WriteGeneratedCode() error { } code, err := a.GenerateCode() - errw := writeFile(genfilename, code) + errw := writeFile(genfilename, a.Package(), code) if err == nil { err = errw } @@ -862,8 +903,12 @@ func (a *API) GenerateCode() ([]byte, error) { for _, meth := range a.APIMethods() { meth.generateCode() } - - for _, res := range a.doc.Resources { + a.insertSplitFileComment() + rCnt := len(a.doc.Resources) / 2 + for i, res := range a.doc.Resources { + if i == rCnt { + a.insertSplitFileComment() + } a.generateResourceMethods(res) } @@ -874,6 +919,56 @@ func (a *API) GenerateCode() ([]byte, error) { return clean, nil } +func (a *API) insertSplitFileComment() { + if apisToSplit[a.Package()] { + a.pn("") + a.pn(splitFileSeperator) + a.pn("") + } +} + +// splitFileHeading writes the file preamble used when generating a split file +// client like compute. +func splitFileHeading(w io.Writer, pkg string) { + pn := func(format string, args ...interface{}) { + _, err := fmt.Fprintf(w, format+"\n", args...) + if err != nil { + panic(err) + } + } + + pn(`// Copyright %s Google LLC. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Code generated file. DO NOT EDIT. +`, *copyrightYear) + pn("") + pn("package %s", pkg) + pn("") + pn("import (") + for _, imp := range []string{ + "context", + "fmt", + "io", + "net/http", + } { + pn(" %q", imp) + } + pn("") + for _, imp := range []struct { + pkg string + lname string + }{ + {*gensupportPkg, "gensupport"}, + {*googleapiPkg, "googleapi"}, + } { + pn(" %s %q", imp.lname, imp.pkg) + } + pn(")") + pn("") +} + func (a *API) generateScopeConstants() { scopes := a.doc.Auth.OAuth2Scopes if len(scopes) == 0 {