Fixed errors while running AWS cloudformation query to list the stacks …

…Closes #1954 (#1959)

aws/table_aws_cloudformation_stack.go

@@ -2,12 +2,18 @@ package aws
 
 import (
 	"context"
+	"encoding/json"
+	"net/url"
+	"regexp"
+	"strings"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/service/cloudformation"
 	"github.com/aws/aws-sdk-go-v2/service/cloudformation/types"
+	go_kit "github.com/turbot/go-kit/types"
 
 	cloudformationv1 "github.com/aws/aws-sdk-go/service/cloudformation"
+	"github.com/goccy/go-yaml"
 
 	"github.com/turbot/steampipe-plugin-sdk/v5/grpc/proto"
 	"github.com/turbot/steampipe-plugin-sdk/v5/plugin"
@@ -146,14 +152,14 @@ func tableAwsCloudFormationStack(_ context.Context) *plugin.Table {
 				Description: "Structure containing the template body.",
 				Type:        proto.ColumnType_STRING,
 				Hydrate:     getStackTemplate,
-				Transform:   transform.FromField("TemplateBody"),
+				Transform:   transform.FromField("TemplateBody").Transform(transform.ToString),
 			},
 			{
 				Name:        "template_body_json",
 				Description: "Structure containing the template body. Parsed into json object for better readability.",
 				Type:        proto.ColumnType_JSON,
 				Hydrate:     getStackTemplate,
-				Transform:   transform.FromField("TemplateBody").Transform(transform.UnmarshalYAML),
+				Transform:   transform.FromField("TemplateBody").Transform(formatJsonBody),
 			},
 			{
 				Name:        "resources",
@@ -343,3 +349,71 @@ func cfnStackTagsToTurbotTags(_ context.Context, d *transform.TransformData) (in
 	}
 	return turbotTagsMap, nil
 }
+
+/*
+* Encountering errors when parsing YAML to JSON with `transform.UnmarshalYAML` in the Steampipe SDK, primarily due to `url.QueryUnescape()` usage.
+* The API may return the template body in various formats like JSON or YAML, based on configuration made by user. Consequently, converting YAML to JSON isn't always required.
+* To address this, a dedicated function has been implemented. It ensures the template body is correctly formatted to JSON as per the API's output, adapting the transformation process accordingly.
+* This approach is specifically designed for this table, providing a tailored and accurate response handling as per the API specifications.
+*/
+
+// Functionality Overview
+// Identifies and decodes URLs within the template body. (URLs are decoded selectively to avoid issues with '%' characters not part of a valid escaped sequence.)
+// Integrates the decoded URLs back into the original template body.
+// Determines if the template body is in JSON or YAML format.
+// Directly unmarshal the content if it's a JSON string.
+// Converts YAML format to JSON if the template body is in YAML.
+func formatJsonBody(ctx context.Context, d *transform.TransformData) (interface{}, error) {
+	if d.Value == nil {
+		return nil, nil
+	}
+
+	inputStr := go_kit.SafeString(d.Value)
+	var result interface{}
+	if inputStr != "" {
+
+		// Unescape only URLs instead of checking if any '%' character is not followed by two hexadecimal digits in the template body.
+		// QueryUnescape does the inverse transformation of QueryEscape, converting each 3-byte encoded substring of the form "%AB" into the hex-decoded byte 0xAB. It returns an error if any % is not followed by two hexadecimal digits.
+		// Regex to match URLs
+		regex := regexp.MustCompile(`(https?://[^\s]+)`)
+
+		// Find all URLs in the input string
+		matches := regex.FindAllString(inputStr, -1)
+
+		// Iterate the URLs present in the template body.
+		for _, match := range matches {
+			// The `QueryUnescape()` function returns an error if any '%' character is not followed by two hexadecimal digits while unescaping the URL.
+			// The template body may contain instances where '%' is not followed by two hexadecimal digits.
+			// We should Unescape only the URLs not all the template body.
+			decoded, err := url.QueryUnescape(match)
+			if err != nil {
+				return nil, err
+			}
+			inputStr = strings.ReplaceAll(inputStr, match, decoded)
+		}
+
+		// Check if the template body return by API is in JSON/YAML format
+		if isJSON(inputStr) {
+			err := json.Unmarshal([]byte(inputStr), &result)
+			if err != nil {
+				return nil, err
+			}
+		} else {
+			// Occasionally, the API response includes the template body with carriage return characters (`\r`).
+			// This leads to an "Unmarshal error [2:4] unexpected key name" when attempting to parse the YAML string to JSON.
+			// Therefore, it's necessary to substitute the carriage return character `\r` with `\n` to resolve this issue.
+			inputStr = strings.ReplaceAll(inputStr, "\r", "\n")
+			err := yaml.Unmarshal([]byte(inputStr), &result)
+			if err != nil {
+				return nil, err
+			}
+		}
+
+	}
+	return result, nil
+}
+
+func isJSON(str string) bool {
+	var js json.RawMessage
+	return json.Unmarshal([]byte(str), &js) == nil
+}

go.mod

@@ -125,6 +125,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/workspaces v1.28.0
 	github.com/aws/smithy-go v1.19.0
 	github.com/gocarina/gocsv v0.0.0-20201208093247-67c824bc04d4
+	github.com/goccy/go-yaml v1.11.3
 	github.com/golang/protobuf v1.5.3
 	github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529
 	github.com/turbot/go-kit v0.9.0
@@ -133,6 +134,8 @@ require (
 	golang.org/x/text v0.14.0
 )
 
+require golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
+
 require (
 	cloud.google.com/go v0.111.0 // indirect
 	cloud.google.com/go/compute v1.23.3 // indirect

go.sum

@@ -580,11 +580,19 @@ github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ=
 github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
 github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q=
+github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
+github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no=
+github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
+github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE=
+github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/go-test/deep v1.0.3 h1:ZrJSEWsXzPOxaZnFteGEfooLba+ju3FYIbOrS+rQd68=
 github.com/go-test/deep v1.0.3/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA=
 github.com/gocarina/gocsv v0.0.0-20201208093247-67c824bc04d4 h1:Q7s2AN3DhFJKOnzO0uTKLhJTfXTEcXcvw5ylf2BHJw4=
 github.com/gocarina/gocsv v0.0.0-20201208093247-67c824bc04d4/go.mod h1:5YoVOkjYAQumqlV356Hj3xeYh4BdZuLE0/nRkf2NKkI=
+github.com/goccy/go-yaml v1.11.3 h1:B3W9IdWbvrUu2OYQGwvU1nZtvMQJPBKgBUuweJjLj6I=
+github.com/goccy/go-yaml v1.11.3/go.mod h1:wKnAMd44+9JAAnGQpWVEgBzGt3YuTaQ4uXoHvE4m7WU=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/glog v1.1.2 h1:DVjP2PbBOzHyzA+dn3WhHIq4NdVu3Q+pvivFICf/7fo=
@@ -744,6 +752,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3vkb4ax0b5D2DHbNAUsen0Gx5wZoq3lV4=
 github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
+github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
+github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
 github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=