diff --git a/cmd/cleanupexamples/main.go b/cmd/cleanupexamples/main.go new file mode 100644 index 0000000..2f8538d --- /dev/null +++ b/cmd/cleanupexamples/main.go @@ -0,0 +1,170 @@ +// Copyright 2024 Upbound Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// main package for the cleanupexamples tooling, the tool to remove +// uptest-specific code from published examples on the marketplace +package main + +import ( + "bytes" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/alecthomas/kingpin/v2" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + kyaml "k8s.io/apimachinery/pkg/util/yaml" + "sigs.k8s.io/yaml" +) + +// filterAnnotations removes specific annotations from a Kubernetes +// unstructured object. It looks for annotations with prefixes +// "upjet.upbound.io/" and "uptest.upbound.io/" and removes +// them if they are present. +func filterAnnotations(u *unstructured.Unstructured) { + annotations := u.GetAnnotations() + annotationsToRemove := []string{ + "upjet.upbound.io/", + "uptest.upbound.io/", + } + + for key := range annotations { + for _, prefix := range annotationsToRemove { + if strings.HasPrefix(key, prefix) { + delete(annotations, key) + break + } + } + } + + if len(annotations) == 0 { + u.SetAnnotations(nil) + } else { + u.SetAnnotations(annotations) + } +} + +// processYAML processes a YAML file by replacing specific placeholders +// and removing certain annotations from the Kubernetes objects within it. +// It returns the modified YAML content as a byte slice with YAML document +// separators ("---") between each object. +func processYAML(yamlData []byte) ([]byte, error) { + // TODO(turkenf): Handle replacing UPTEST_DATASOURCE placeholders like ${data.aws_account_id} + yamlData = bytes.ReplaceAll(yamlData, []byte("${Rand.RFC1123Subdomain}"), []byte("random")) + + // Create a YAML or JSON decoder to read and decode the input YAML data + decoder := kyaml.NewYAMLOrJSONDecoder(bytes.NewReader(yamlData), 1024) + var modifiedYAMLs []byte + separator := []byte("---\n") + first := true + + for { + u := &unstructured.Unstructured{} + if err := decoder.Decode(&u); err != nil { + if errors.Is(err, io.EOF) { + break + } + return nil, fmt.Errorf("cannot decode the YAML file: %w", err) + } + + // Remove specific annotations from the decoded Kubernetes object. + filterAnnotations(u) + + modifiedYAML, err := yaml.Marshal(u.Object) + if err != nil { + return nil, fmt.Errorf("cannot marshal the YAML file: %w", err) + } + + if !first { + modifiedYAMLs = append(modifiedYAMLs, separator...) + } + modifiedYAMLs = append(modifiedYAMLs, modifiedYAML...) + first = false + } + + return modifiedYAMLs, nil +} + +// processDirectory walks through a directory structure, processes all YAML +// files within it by calling `processYAML`, and saves the modified files +// to a specified output directory while preserving the original +// directory structure. +func processDirectory(inputDir string, outputDir string) error { //nolint:gocyclo // sequential flow easier to follow + // Walk through the input directory + err := filepath.Walk(inputDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return fmt.Errorf("error accessing path %s: %w", path, err) + } + + // Check if it's a directory, skip processing but ensure the directory exists in outputDir + if info.IsDir() { + relativePath, err := filepath.Rel(inputDir, path) + if err != nil { + return fmt.Errorf("error finding relative path: %w", err) + } + newDir := filepath.Join(outputDir, relativePath) + if _, err := os.Stat(newDir); os.IsNotExist(err) { + err = os.MkdirAll(newDir, 0750) + if err != nil { + return fmt.Errorf("cannot create directory %s: %w", newDir, err) + } + } + return nil + } + + // Only process YAML files + if filepath.Ext(path) == ".yaml" || filepath.Ext(path) == ".yml" { + yamlFile, err := os.ReadFile(filepath.Clean(path)) + if err != nil { + return fmt.Errorf("cannot read the YAML file %s: %w", path, err) + } + + modifiedYAMLs, err := processYAML(yamlFile) + if err != nil { + return fmt.Errorf("error processing YAML file %s: %w", path, err) + } + + // Create the output path, preserving directory structure + relativePath, err := filepath.Rel(inputDir, path) + if err != nil { + return fmt.Errorf("error finding relative path: %w", err) + } + outputPath := filepath.Join(outputDir, relativePath) + err = os.WriteFile(outputPath, modifiedYAMLs, 0600) + if err != nil { + return fmt.Errorf("cannot write the YAML file %s: %w", outputPath, err) + } + } + return nil + }) + return err +} + +func main() { + inputDir := kingpin.Arg("inputDir", "Directory containing the input YAML files.").Required().String() + outputDir := kingpin.Arg("outputDir", "Directory to save the processed YAML files.").Required().String() + + kingpin.Parse() + + err := processDirectory(*inputDir, *outputDir) + if err != nil { + fmt.Printf("error processing directory: %v\n", err) + return + } + + fmt.Printf("All YAML files processed and saved to: %s\n", *outputDir) +} diff --git a/cmd/cleanupexamples/main_test.go b/cmd/cleanupexamples/main_test.go new file mode 100644 index 0000000..fbb4b66 --- /dev/null +++ b/cmd/cleanupexamples/main_test.go @@ -0,0 +1,202 @@ +package main + +import ( + "testing" + + "github.com/google/go-cmp/cmp" +) + +func Test_processYAML(t *testing.T) { + type args struct { + yamlData []byte + } + tests := []struct { + name string + args args + want []byte + wantErr bool + }{ + { + name: "simple YAML without annotations", + args: args{ + yamlData: []byte(`apiVersion: apigateway.aws.upbound.io/v1beta1 +kind: BasePathMapping +metadata: + name: example +spec: + forProvider: + region: us-west-1`), + }, + want: []byte(`apiVersion: apigateway.aws.upbound.io/v1beta1 +kind: BasePathMapping +metadata: + name: example +spec: + forProvider: + region: us-west-1 +`), + wantErr: false, + }, + { + name: "single YAML with multiple annotations", + args: args{ + yamlData: []byte(`apiVersion: apigateway.aws.upbound.io/v1beta1 +kind: BasePathMapping +metadata: + annotations: + upjet.upbound.io/manual-intervention: "The BasePathMapping resource needs a DomainName and DomainName resource needs valid certificates." + uptest.upbound.io/timeout: "3600" # one hour timeout + uptest.upbound.io/disable-import: "true" + uptest.upbound.io/pre-delete-hook: testhooks/delete + labels: + testing.upbound.io/example-name: domainname + name: example-${Rand.RFC1123Subdomain} +spec: + forProvider: + region: us-west-1`), + }, + want: []byte(`apiVersion: apigateway.aws.upbound.io/v1beta1 +kind: BasePathMapping +metadata: + labels: + testing.upbound.io/example-name: domainname + name: example-random +spec: + forProvider: + region: us-west-1 +`), + wantErr: false, + }, + { + name: "multiple YAML with annotations", + args: args{ + yamlData: []byte(`apiVersion: appconfig.aws.upbound.io/v1beta1 +kind: Deployment +metadata: + annotations: + upjet.upbound.io/manual-intervention: "Requires configuration version to replaced manually." + meta.upbound.io/example-id: appconfig/v1beta1/deployment + crossplane.io/external-name: example + labels: + testing.upbound.io/example-name: example + name: example +spec: + forProvider: + region: us-east-1 + tags: + Type: AppConfig Deployment +--- +apiVersion: appconfig.aws.upbound.io/v1beta1 +kind: HostedConfigurationVersion +metadata: + annotations: + upjet.upbound.io/manual-intervention: "Requires configuration version to replaced manually." + meta.upbound.io/example-id: appconfig/v1beta1/deployment + labels: + testing.upbound.io/example-name: example + name: example +spec: + forProvider: + region: us-east-1 +--- +apiVersion: appconfig.aws.upbound.io/v1beta1 +kind: Application +metadata: + annotations: + upjet.upbound.io/manual-intervention: "Requires configuration version to replaced manually." + uptest.upbound.io/timeout: "5400" + meta.upbound.io/example-id: appconfig/v1beta1/deployment + labels: + testing.upbound.io/example-name: example + name: example +spec: + forProvider: + region: us-east-1`), + }, + want: []byte(`apiVersion: appconfig.aws.upbound.io/v1beta1 +kind: Deployment +metadata: + annotations: + crossplane.io/external-name: example + meta.upbound.io/example-id: appconfig/v1beta1/deployment + labels: + testing.upbound.io/example-name: example + name: example +spec: + forProvider: + region: us-east-1 + tags: + Type: AppConfig Deployment +--- +apiVersion: appconfig.aws.upbound.io/v1beta1 +kind: HostedConfigurationVersion +metadata: + annotations: + meta.upbound.io/example-id: appconfig/v1beta1/deployment + labels: + testing.upbound.io/example-name: example + name: example +spec: + forProvider: + region: us-east-1 +--- +apiVersion: appconfig.aws.upbound.io/v1beta1 +kind: Application +metadata: + annotations: + meta.upbound.io/example-id: appconfig/v1beta1/deployment + labels: + testing.upbound.io/example-name: example + name: example +spec: + forProvider: + region: us-east-1 +`), + wantErr: false, + }, + { + name: "single YAML with randomized string without annotation", + args: args{ + yamlData: []byte(`apiVersion: appconfig.aws.upbound.io/v1beta1 +kind: Environment +metadata: + name: example +spec: + forProvider: + name: example-${Rand.RFC1123Subdomain} + region: us-east-1`), + }, + want: []byte(`apiVersion: appconfig.aws.upbound.io/v1beta1 +kind: Environment +metadata: + name: example +spec: + forProvider: + name: example-random + region: us-east-1 +`), + wantErr: false, + }, + { + name: "invalid YAML", + args: args{ + yamlData: []byte(`invalid_yaml: [this is not valid`), + }, + want: nil, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := processYAML(tt.args.yamlData) + if (err != nil) != tt.wantErr { + t.Errorf("processYAML() error = %v, wantErr %v", err, tt.wantErr) + return + } + if diff := cmp.Diff(got, tt.want); diff != "" { + t.Errorf("processYAML() there is a diff: %s", diff) + } + }) + } +} diff --git a/go.mod b/go.mod index 913dc63..c57e248 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.21 require ( cloud.google.com/go/storage v1.27.0 github.com/adrg/frontmatter v0.2.0 + github.com/alecthomas/kingpin/v2 v2.4.0 github.com/alecthomas/kong v0.7.1 github.com/crossplane/crossplane v1.10.0 github.com/crossplane/crossplane-runtime v0.20.0-rc.0.0.20230406155702-4e1673b7141f @@ -87,6 +88,7 @@ require ( github.com/peterbourgon/diskv v2.0.1+incompatible // indirect github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/procfs v0.8.0 // indirect + github.com/xhit/go-str2duration/v2 v2.1.0 // indirect github.com/xlab/treeprint v1.1.0 // indirect github.com/yuin/goldmark v1.5.3 // indirect go.opencensus.io v0.24.0 // indirect diff --git a/go.sum b/go.sum index 893fddf..701866f 100644 --- a/go.sum +++ b/go.sum @@ -56,6 +56,8 @@ github.com/adrg/frontmatter v0.2.0 h1:/DgnNe82o03riBd1S+ZDjd43wAmC6W35q67NHeLkPd github.com/adrg/frontmatter v0.2.0/go.mod h1:93rQCj3z3ZlwyxxpQioRKC1wDLto4aXHrbqIsnH9wmE= github.com/alecthomas/assert/v2 v2.1.0 h1:tbredtNcQnoSd3QBhQWI7QZ3XHOVkw1Moklp2ojoH/0= github.com/alecthomas/assert/v2 v2.1.0/go.mod h1:b/+1DI2Q6NckYi+3mXyH3wFb8qG37K/DuK80n7WefXA= +github.com/alecthomas/kingpin/v2 v2.4.0 h1:f48lwail6p8zpO1bC4TxtqACaGqHYA22qkHjHpqDjYY= +github.com/alecthomas/kingpin/v2 v2.4.0/go.mod h1:0gyi0zQnjuFk8xrkNKamJoyUo382HRL7ATRpFZCw6tE= github.com/alecthomas/kong v0.7.1 h1:azoTh0IOfwlAX3qN9sHWTxACE2oV8Bg2gAwBsMwDQY4= github.com/alecthomas/kong v0.7.1/go.mod h1:n1iCIO2xS46oE8ZfYCNDqdR0b0wZNrXAIAqro/2132U= github.com/alecthomas/repr v0.1.0 h1:ENn2e1+J3k09gyj2shc0dHr/yjaWSHRlrJ4DPMevDqE= @@ -390,8 +392,9 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/tufin/oasdiff v1.2.6 h1:npdeFfs1lvTTcsT5HB/zDJZntl6tG2oMMK0dfFSF+yM= github.com/tufin/oasdiff v1.2.6/go.mod h1:7Ukdw7vE8EFNl8mf7e9rNX/aXptYn0RGtXoPCjjjlBU= github.com/vbatts/tar-split v0.11.2 h1:Via6XqJr0hceW4wff3QRzD5gAk/tatMw/4ZA7cTlIME= @@ -399,6 +402,8 @@ github.com/vbatts/tar-split v0.11.2/go.mod h1:vV3ZuO2yWSVsz+pfFzDG/upWH1JhjOiEaW github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= +github.com/xhit/go-str2duration/v2 v2.1.0 h1:lxklc02Drh6ynqX+DdPyp5pCKLUQpRT8bp8Ydu2Bstc= +github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU= github.com/xlab/treeprint v1.1.0 h1:G/1DjNkPpfZCFt9CSh6b5/nY4VimlbHF3Rh4obvtzDk= github.com/xlab/treeprint v1.1.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=