Merge pull request #215 from turkenf/cleanup-examples

Add the cleanupexamples tool

cmd/cleanupexamples/main.go

@@ -0,0 +1,170 @@
+// Copyright 2024 Upbound Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// main package for the cleanupexamples tooling, the tool to remove
+// uptest-specific code from published examples on the marketplace
+package main
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/alecthomas/kingpin/v2"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	kyaml "k8s.io/apimachinery/pkg/util/yaml"
+	"sigs.k8s.io/yaml"
+)
+
+// filterAnnotations removes specific annotations from a Kubernetes
+// unstructured object. It looks for annotations with prefixes
+// "upjet.upbound.io/" and "uptest.upbound.io/" and removes
+// them if they are present.
+func filterAnnotations(u *unstructured.Unstructured) {
+	annotations := u.GetAnnotations()
+	annotationsToRemove := []string{
+		"upjet.upbound.io/",
+		"uptest.upbound.io/",
+	}
+
+	for key := range annotations {
+		for _, prefix := range annotationsToRemove {
+			if strings.HasPrefix(key, prefix) {
+				delete(annotations, key)
+				break
+			}
+		}
+	}
+
+	if len(annotations) == 0 {
+		u.SetAnnotations(nil)
+	} else {
+		u.SetAnnotations(annotations)
+	}
+}
+
+// processYAML processes a YAML file by replacing specific placeholders
+// and removing certain annotations from the Kubernetes objects within it.
+// It returns the modified YAML content as a byte slice with YAML document
+// separators ("---") between each object.
+func processYAML(yamlData []byte) ([]byte, error) {
+	// TODO(turkenf): Handle replacing UPTEST_DATASOURCE placeholders like ${data.aws_account_id}
+	yamlData = bytes.ReplaceAll(yamlData, []byte("${Rand.RFC1123Subdomain}"), []byte("random"))
+
+	// Create a YAML or JSON decoder to read and decode the input YAML data
+	decoder := kyaml.NewYAMLOrJSONDecoder(bytes.NewReader(yamlData), 1024)
+	var modifiedYAMLs []byte
+	separator := []byte("---\n")
+	first := true
+
+	for {
+		u := &unstructured.Unstructured{}
+		if err := decoder.Decode(&u); err != nil {
+			if errors.Is(err, io.EOF) {
+				break
+			}
+			return nil, fmt.Errorf("cannot decode the YAML file: %w", err)
+		}
+
+		// Remove specific annotations from the decoded Kubernetes object.
+		filterAnnotations(u)
+
+		modifiedYAML, err := yaml.Marshal(u.Object)
+		if err != nil {
+			return nil, fmt.Errorf("cannot marshal the YAML file: %w", err)
+		}
+
+		if !first {
+			modifiedYAMLs = append(modifiedYAMLs, separator...)
+		}
+		modifiedYAMLs = append(modifiedYAMLs, modifiedYAML...)
+		first = false
+	}
+
+	return modifiedYAMLs, nil
+}
+
+// processDirectory walks through a directory structure, processes all YAML
+// files  within it by calling `processYAML`, and saves the modified files
+// to a specified  output directory while preserving the original
+// directory structure.
+func processDirectory(inputDir string, outputDir string) error { //nolint:gocyclo // sequential flow easier to follow
+	// Walk through the input directory
+	err := filepath.Walk(inputDir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return fmt.Errorf("error accessing path %s: %w", path, err)
+		}
+
+		// Check if it's a directory, skip processing but ensure the directory exists in outputDir
+		if info.IsDir() {
+			relativePath, err := filepath.Rel(inputDir, path)
+			if err != nil {
+				return fmt.Errorf("error finding relative path: %w", err)
+			}
+			newDir := filepath.Join(outputDir, relativePath)
+			if _, err := os.Stat(newDir); os.IsNotExist(err) {
+				err = os.MkdirAll(newDir, 0750)
+				if err != nil {
+					return fmt.Errorf("cannot create directory %s: %w", newDir, err)
+				}
+			}
+			return nil
+		}
+
+		// Only process YAML files
+		if filepath.Ext(path) == ".yaml" || filepath.Ext(path) == ".yml" {
+			yamlFile, err := os.ReadFile(filepath.Clean(path))
+			if err != nil {
+				return fmt.Errorf("cannot read the YAML file %s: %w", path, err)
+			}
+
+			modifiedYAMLs, err := processYAML(yamlFile)
+			if err != nil {
+				return fmt.Errorf("error processing YAML file %s: %w", path, err)
+			}
+
+			// Create the output path, preserving directory structure
+			relativePath, err := filepath.Rel(inputDir, path)
+			if err != nil {
+				return fmt.Errorf("error finding relative path: %w", err)
+			}
+			outputPath := filepath.Join(outputDir, relativePath)
+			err = os.WriteFile(outputPath, modifiedYAMLs, 0600)
+			if err != nil {
+				return fmt.Errorf("cannot write the YAML file %s: %w", outputPath, err)
+			}
+		}
+		return nil
+	})
+	return err
+}
+
+func main() {
+	inputDir := kingpin.Arg("inputDir", "Directory containing the input YAML files.").Required().String()
+	outputDir := kingpin.Arg("outputDir", "Directory to save the processed YAML files.").Required().String()
+
+	kingpin.Parse()
+
+	err := processDirectory(*inputDir, *outputDir)
+	if err != nil {
+		fmt.Printf("error processing directory: %v\n", err)
+		return
+	}
+
+	fmt.Printf("All YAML files processed and saved to: %s\n", *outputDir)
+}

cmd/cleanupexamples/main_test.go

@@ -0,0 +1,202 @@
+package main
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+)
+
+func Test_processYAML(t *testing.T) {
+	type args struct {
+		yamlData []byte
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    []byte
+		wantErr bool
+	}{
+		{
+			name: "simple YAML without annotations",
+			args: args{
+				yamlData: []byte(`apiVersion: apigateway.aws.upbound.io/v1beta1
+kind: BasePathMapping
+metadata:
+  name: example
+spec:
+  forProvider:
+    region: us-west-1`),
+			},
+			want: []byte(`apiVersion: apigateway.aws.upbound.io/v1beta1
+kind: BasePathMapping
+metadata:
+  name: example
+spec:
+  forProvider:
+    region: us-west-1
+`),
+			wantErr: false,
+		},
+		{
+			name: "single YAML with multiple annotations",
+			args: args{
+				yamlData: []byte(`apiVersion: apigateway.aws.upbound.io/v1beta1
+kind: BasePathMapping
+metadata:
+  annotations:
+    upjet.upbound.io/manual-intervention: "The BasePathMapping resource needs a DomainName and DomainName resource needs valid certificates."
+    uptest.upbound.io/timeout: "3600" # one hour timeout
+    uptest.upbound.io/disable-import: "true"
+    uptest.upbound.io/pre-delete-hook: testhooks/delete
+  labels:
+    testing.upbound.io/example-name: domainname
+  name: example-${Rand.RFC1123Subdomain}
+spec:
+  forProvider:
+    region: us-west-1`),
+			},
+			want: []byte(`apiVersion: apigateway.aws.upbound.io/v1beta1
+kind: BasePathMapping
+metadata:
+  labels:
+    testing.upbound.io/example-name: domainname
+  name: example-random
+spec:
+  forProvider:
+    region: us-west-1
+`),
+			wantErr: false,
+		},
+		{
+			name: "multiple YAML with annotations",
+			args: args{
+				yamlData: []byte(`apiVersion: appconfig.aws.upbound.io/v1beta1
+kind: Deployment
+metadata:
+  annotations:
+    upjet.upbound.io/manual-intervention: "Requires configuration version to replaced manually."
+    meta.upbound.io/example-id: appconfig/v1beta1/deployment
+    crossplane.io/external-name: example
+  labels:
+    testing.upbound.io/example-name: example
+  name: example
+spec:
+  forProvider:
+    region: us-east-1
+    tags:
+      Type: AppConfig Deployment
+---
+apiVersion: appconfig.aws.upbound.io/v1beta1
+kind: HostedConfigurationVersion
+metadata:
+  annotations:
+    upjet.upbound.io/manual-intervention: "Requires configuration version to replaced manually."
+    meta.upbound.io/example-id: appconfig/v1beta1/deployment
+  labels:
+    testing.upbound.io/example-name: example
+  name: example
+spec:
+  forProvider:
+    region: us-east-1
+---
+apiVersion: appconfig.aws.upbound.io/v1beta1
+kind: Application
+metadata:
+  annotations:
+    upjet.upbound.io/manual-intervention: "Requires configuration version to replaced manually."
+    uptest.upbound.io/timeout: "5400"
+    meta.upbound.io/example-id: appconfig/v1beta1/deployment
+  labels:
+    testing.upbound.io/example-name: example
+  name: example
+spec:
+  forProvider:
+    region: us-east-1`),
+			},
+			want: []byte(`apiVersion: appconfig.aws.upbound.io/v1beta1
+kind: Deployment
+metadata:
+  annotations:
+    crossplane.io/external-name: example
+    meta.upbound.io/example-id: appconfig/v1beta1/deployment
+  labels:
+    testing.upbound.io/example-name: example
+  name: example
+spec:
+  forProvider:
+    region: us-east-1
+    tags:
+      Type: AppConfig Deployment
+---
+apiVersion: appconfig.aws.upbound.io/v1beta1
+kind: HostedConfigurationVersion
+metadata:
+  annotations:
+    meta.upbound.io/example-id: appconfig/v1beta1/deployment
+  labels:
+    testing.upbound.io/example-name: example
+  name: example
+spec:
+  forProvider:
+    region: us-east-1
+---
+apiVersion: appconfig.aws.upbound.io/v1beta1
+kind: Application
+metadata:
+  annotations:
+    meta.upbound.io/example-id: appconfig/v1beta1/deployment
+  labels:
+    testing.upbound.io/example-name: example
+  name: example
+spec:
+  forProvider:
+    region: us-east-1
+`),
+			wantErr: false,
+		},
+		{
+			name: "single YAML with randomized string without annotation",
+			args: args{
+				yamlData: []byte(`apiVersion: appconfig.aws.upbound.io/v1beta1
+kind: Environment
+metadata:
+  name: example
+spec:
+  forProvider:
+    name: example-${Rand.RFC1123Subdomain}
+    region: us-east-1`),
+			},
+			want: []byte(`apiVersion: appconfig.aws.upbound.io/v1beta1
+kind: Environment
+metadata:
+  name: example
+spec:
+  forProvider:
+    name: example-random
+    region: us-east-1
+`),
+			wantErr: false,
+		},
+		{
+			name: "invalid YAML",
+			args: args{
+				yamlData: []byte(`invalid_yaml: [this is not valid`),
+			},
+			want:    nil,
+			wantErr: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := processYAML(tt.args.yamlData)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("processYAML() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if diff := cmp.Diff(got, tt.want); diff != "" {
+				t.Errorf("processYAML() there is a diff: %s", diff)
+			}
+		})
+	}
+}

go.mod

@@ -5,6 +5,7 @@ go 1.21
 require (
 	cloud.google.com/go/storage v1.27.0
 	github.com/adrg/frontmatter v0.2.0
+	github.com/alecthomas/kingpin/v2 v2.4.0
 	github.com/alecthomas/kong v0.7.1
 	github.com/crossplane/crossplane v1.10.0
 	github.com/crossplane/crossplane-runtime v0.20.0-rc.0.0.20230406155702-4e1673b7141f
@@ -87,6 +88,7 @@ require (
 	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
 	github.com/prometheus/client_model v0.3.0 // indirect
 	github.com/prometheus/procfs v0.8.0 // indirect
+	github.com/xhit/go-str2duration/v2 v2.1.0 // indirect
 	github.com/xlab/treeprint v1.1.0 // indirect
 	github.com/yuin/goldmark v1.5.3 // indirect
 	go.opencensus.io v0.24.0 // indirect