Add code and scheme generation (#20)

Problem:
- Ensure the Attributes method of Exprotable interface are generated
automatically for the telemetry data types.
- Ensure the avro scheme (.avdl) is generated automatically for the
telemetry data types.

Solution:
- Add generator tool in cmd/generator that generates code (Attributes
method) and the scheme.
- Generator can be used in //go:generate annotations in go source files.
- Generator has "generator" build tag so that it is not included into
the telemetry library by default.
- Generator uses golang.org/x/tools/go/packages to parse source files.

Testing:
- Unit tests of parsing.
- Unit tests of code and scheme generation - ensuring non-zero output.
- Unit tests of generated code in cmd/generator/tests package
- Manual validation of the generated scheme
  (cmd/generator/tests/data.avdl) using Apache Avro IDL Scheme Support
  IntelliJ plugin.

CLOSES - #18

Co-authored-by: Saylor Berman <[email protected]>
Co-authored-by: Ciara Stacke <[email protected]>

.gitignore

@@ -6,8 +6,8 @@
 *.dylib
 
 # Output of the go coverage tool
-cmd-cover.out
-cmd-cover.html
+*cover.out
+*cover.html
 
 # Vim
 *.swp

.golangci.yml

@@ -66,3 +66,5 @@ issues:
   max-same-issues: 0
 run:
   timeout: 3m
+  build-tags:
+    - generator

Makefile

@@ -9,6 +9,8 @@ help: Makefile ## Display this help
 unit-test:
 	go test ./... -race -coverprofile cmd-cover.out
 	go tool cover -html=cmd-cover.out -o cmd-cover.html
+	go test -tags generator ./cmd/generator/... -race -coverprofile generator-cmd-cover.out
+	go tool cover -html=generator-cmd-cover.out -o generator-cmd-cover.html
 
 .PHONY: clean-go-cache
 clean-go-cache: ## Clean go cache
@@ -36,3 +38,9 @@ dev-all: deps fmt vet lint unit-test ## Run all the development checks
 .PHONY: generate
 generate: ## Run go generate
 	go generate ./...
+	go generate -tags generator ./cmd/generator/...
+
+.PHONY: generator-tests
+generator-tests: ## Regenerate the generator generated files and run generator unit tests
+	go generate -tags generator ./cmd/generator/... # ensure the generated files generated by the generator are up to date
+	go test -tags generator ./cmd/generator/... -race -coverprofile cmd-cover.out

cmd/generator/code.go

@@ -0,0 +1,152 @@
+//go:build generator
+
+package main
+
+import (
+	"fmt"
+	"go/types"
+	"io"
+	"reflect"
+	"strings"
+	"text/template"
+
+	"github.com/nginxinc/telemetry-exporter/pkg/telemetry"
+)
+
+var telemetryPackagePath = reflect.TypeOf((*telemetry.Exportable)(nil)).Elem().PkgPath()
+
+const codeTemplate = `{{ if .BuildTags }}//go:build {{ .BuildTags }}{{ end }}
+package {{ .PackageName }}
+/*
+This is a generated file. DO NOT EDIT.
+*/
+
+import (
+	"go.opentelemetry.io/otel/attribute"
+
+	{{ if .TelemetryPackagePath }}
+	{{ if .TelemetryPackageAlias }}{{ .TelemetryPackageAlias }} {{ end -}}
+	"{{ .TelemetryPackagePath }}"
+	{{ end }}
+)
+
+func (d *{{ .StructName }}) Attributes() []attribute.KeyValue {
+	var attrs []attribute.KeyValue
+
+	{{ range .Fields -}}
+	attrs = append(attrs, {{ .AttributesSource }})
+	{{ end }}
+
+	return attrs
+}
+
+var _ {{ .ExportablePackagePrefix }}Exportable = (*{{ .StructName }})(nil)
+`
+
+type codeGen struct {
+	PackageName             string
+	TelemetryPackagePath    string
+	TelemetryPackageAlias   string
+	ExportablePackagePrefix string
+	StructName              string
+	BuildTags               string
+	Fields                  []codeField
+}
+
+type codeField struct {
+	AttributesSource string
+}
+
+func getAttributeType(kind types.BasicKind) string {
+	switch kind {
+	case types.Int64:
+		return "Int64"
+	case types.Float64:
+		return "Float64"
+	case types.String:
+		return "String"
+	case types.Bool:
+		return "Bool"
+	default:
+		panic(fmt.Sprintf("unexpected kind %v", kind))
+	}
+}
+
+type codeGenConfig struct {
+	packagePath string
+	typeName    string
+	buildTags   string
+	fields      []field
+}
+
+func generateCode(writer io.Writer, cfg codeGenConfig) error {
+	codeFields := make([]codeField, 0, len(cfg.fields))
+
+	for _, f := range cfg.fields {
+		var cf codeField
+
+		if f.embeddedStruct {
+			cf = codeField{
+				AttributesSource: fmt.Sprintf(`d.%s.Attributes()...`, f.name),
+			}
+		} else if f.slice {
+			cf = codeField{
+				AttributesSource: fmt.Sprintf(`attribute.%sSlice("%s", d.%s)`, getAttributeType(f.fieldType), f.name, f.name),
+			}
+		} else {
+			cf = codeField{
+				AttributesSource: fmt.Sprintf(`attribute.%s("%s", d.%s)`, getAttributeType(f.fieldType), f.name, f.name),
+			}
+		}
+
+		codeFields = append(codeFields, cf)
+	}
+
+	const alias = "ngxTelemetry"
+
+	var (
+		telemetryPkg        string
+		exportablePkgPrefix string
+		telemetryPkgAlias   string
+	)
+
+	// check if we generate code for the type in the telemetry package or any other package
+	if cfg.packagePath != telemetryPackagePath {
+		telemetryPkg = telemetryPackagePath
+
+		// if the name of the package is the same as the telemetry package, we need to use an alias
+		if getPackageName(cfg.packagePath) == getPackageName(telemetryPackagePath) {
+			exportablePkgPrefix = alias + "."
+			telemetryPkgAlias = alias
+		} else {
+			exportablePkgPrefix = getPackageName(telemetryPackagePath) + "."
+		}
+	}
+
+	cg := codeGen{
+		PackageName:             getPackageName(cfg.packagePath),
+		ExportablePackagePrefix: exportablePkgPrefix,
+		TelemetryPackageAlias:   telemetryPkgAlias,
+		TelemetryPackagePath:    telemetryPkg,
+		StructName:              cfg.typeName,
+		Fields:                  codeFields,
+		BuildTags:               cfg.buildTags,
+	}
+
+	funcMap := template.FuncMap{
+		"getAttributeType": getAttributeType,
+	}
+
+	tmpl := template.Must(template.New("scheme").Funcs(funcMap).Parse(codeTemplate))
+
+	if err := tmpl.Execute(writer, cg); err != nil {
+		return fmt.Errorf("failed to execute template: %w", err)
+	}
+
+	return nil
+}
+
+func getPackageName(packagePath string) string {
+	packageParts := strings.Split(packagePath, "/")
+	return packageParts[len(packageParts)-1]
+}

cmd/generator/code_test.go

@@ -0,0 +1,41 @@
+//go:build generator
+
+package main
+
+import (
+	"bytes"
+	"testing"
+
+	. "github.com/onsi/gomega"
+
+	"github.com/nginxinc/telemetry-exporter/cmd/generator/tests"
+)
+
+func TestGenerateCode(t *testing.T) {
+	g := NewGomegaWithT(t)
+
+	cfg := parsingConfig{
+		pkgName:     "tests",
+		typeName:    "Data",
+		loadPattern: "github.com/nginxinc/telemetry-exporter/cmd/generator/tests",
+		buildFlags:  []string{"-tags=generator"},
+	}
+
+	_ = tests.Data{} // depends on the type being defined
+
+	parsingResult, err := parse(cfg)
+
+	g.Expect(err).ToNot(HaveOccurred())
+
+	var buf bytes.Buffer
+
+	codeCfg := codeGenConfig{
+		packagePath: parsingResult.packagePath,
+		typeName:    "Data",
+		fields:      parsingResult.fields,
+	}
+
+	g.Expect(generateCode(&buf, codeCfg)).To(Succeed())
+
+	g.Expect(buf.Bytes()).ToNot(BeEmpty())
+}

cmd/generator/main.go

@@ -0,0 +1,129 @@
+//go:build generator
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"strings"
+)
+
+var (
+	code                     = flag.Bool("code", true, "Generate code")
+	buildTags                = flag.String("build-tags", "", "Comma separated list of build tags expected in the source files that will be added to the generated code") //nolint:lll
+	scheme                   = flag.Bool("scheme", false, "Generate Avro scheme")
+	schemeNamespace          = flag.String("scheme-namespace", "gateway.nginx.org", "Scheme namespace; required when -scheme is set") //nolint:lll
+	schemeProtocol           = flag.String("scheme-protocol", "", "Scheme protocol; required when -scheme is set")
+	schemeDataFabricDataType = flag.String("scheme-df-datatype", "", "Scheme data fabric data type; required when -scheme is set") //nolint:lll
+	typeName                 = flag.String("type", "", "Type to generate; required")
+)
+
+func exitWithError(err error) {
+	fmt.Fprintln(os.Stderr, "error: "+err.Error())
+	os.Exit(1)
+}
+
+func exitWithUsage() {
+	flag.Usage()
+	os.Exit(1)
+}
+
+func validateFlags() {
+	if *typeName == "" {
+		exitWithUsage()
+	}
+
+	if *scheme {
+		if *schemeNamespace == "" {
+			exitWithUsage()
+		}
+		if *schemeProtocol == "" {
+			exitWithUsage()
+		}
+		if *schemeDataFabricDataType == "" {
+			exitWithUsage()
+		}
+	}
+}
+
+func main() {
+	flag.Parse()
+
+	validateFlags()
+
+	pkgName := os.Getenv("GOPACKAGE")
+	if pkgName == "" {
+		exitWithError(fmt.Errorf("GOPACKAGE is not set"))
+	}
+
+	var buildFlags []string
+	if *buildTags != "" {
+		buildFlags = []string{"-tags=" + *buildTags}
+	}
+
+	cfg := parsingConfig{
+		pkgName:    pkgName,
+		typeName:   *typeName,
+		buildFlags: buildFlags,
+	}
+
+	result, err := parse(cfg)
+	if err != nil {
+		exitWithError(fmt.Errorf("failed to parse struct: %w", err))
+	}
+
+	fmt.Printf("Successfully parsed struct %s\n", *typeName)
+
+	if *code {
+		fmt.Println("Generating code")
+
+		fileName := fmt.Sprintf("%s_attributes_generated.go", strings.ToLower(*typeName))
+
+		file, err := os.Create(fileName)
+		if err != nil {
+			exitWithError(fmt.Errorf("failed to create file: %w", err))
+		}
+		defer file.Close()
+
+		var codeGenBuildTags string
+		if *buildTags != "" {
+			codeGenBuildTags = strings.ReplaceAll(*buildTags, ",", " && ")
+		}
+
+		codeCfg := codeGenConfig{
+			packagePath: result.packagePath,
+			typeName:    *typeName,
+			fields:      result.fields,
+			buildTags:   codeGenBuildTags,
+		}
+
+		if err := generateCode(file, codeCfg); err != nil {
+			exitWithError(fmt.Errorf("failed to generate code: %w", err))
+		}
+	}
+
+	if *scheme {
+		fmt.Println("Generating scheme")
+
+		fileName := fmt.Sprintf("%s.avdl", strings.ToLower(*typeName))
+
+		file, err := os.Create(fileName)
+		if err != nil {
+			exitWithError(fmt.Errorf("failed to create file: %w", err))
+		}
+		defer file.Close()
+
+		schemeCfg := schemeGenConfig{
+			namespace:          *schemeNamespace,
+			protocol:           *schemeProtocol,
+			dataFabricDataType: *schemeDataFabricDataType,
+			record:             *typeName,
+			fields:             result.fields,
+		}
+
+		if err := generateScheme(file, schemeCfg); err != nil {
+			exitWithError(fmt.Errorf("failed to generate scheme: %w", err))
+		}
+	}
+}