Change api, Add support for s3 bucket resource and better cft loader

go.mod

@@ -39,8 +39,8 @@ require (
 	github.com/stretchr/testify v1.7.0
 	github.com/zclconf/go-cty v1.8.2
 	go.uber.org/zap v1.16.0
-	golang.org/x/sys v0.0.0-20210601080250-7ecdf8ef093b
-	golang.org/x/tools v0.1.2 // indirect
+	golang.org/x/sys v0.0.0-20210611083646-a4fc73990273
+	golang.org/x/tools v0.1.3 // indirect
 	gopkg.in/src-d/go-git.v4 v4.13.1
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
 	helm.sh/helm/v3 v3.4.0

go.sum

@@ -1526,6 +1526,10 @@ golang.org/x/sys v0.0.0-20210525143221-35b2ab0089ea h1:+WiDlPBBaO+h9vPNZi8uJ3k4B
 golang.org/x/sys v0.0.0-20210525143221-35b2ab0089ea/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210601080250-7ecdf8ef093b h1:qh4f65QIVFjq9eBURLEYWqaEXmOyqdUyiBSgaXWccWk=
 golang.org/x/sys v0.0.0-20210601080250-7ecdf8ef093b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210608053332-aa57babbf139 h1:C+AwYEtBp/VQwoLntUmQ/yx3MS9vmZaKNdw5eOpoQe8=
+golang.org/x/sys v0.0.0-20210608053332-aa57babbf139/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210611083646-a4fc73990273 h1:faDu4veV+8pcThn4fewv6TVlNCezafGoC1gM/mxQLbQ=
+golang.org/x/sys v0.0.0-20210611083646-a4fc73990273/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -1603,6 +1607,8 @@ golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4f
 golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
 golang.org/x/tools v0.1.2 h1:kRBLX7v7Af8W7Gdbbc908OJcdgtK8bOz9Uaj8/F1ACA=
 golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/tools v0.1.3 h1:L69ShwSZEyCsLKoAxDKeMvLDZkumEe8gXUZAjab0tX8=
+golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

pkg/iac-providers/cft/v1/load-dir_test.go

@@ -17,16 +17,20 @@
 package cftv1
 
 import (
+	"encoding/json"
 	"fmt"
-	"go.uber.org/zap"
-	"gopkg.in/src-d/go-git.v4"
-	gitConfig "gopkg.in/src-d/go-git.v4/config"
-	"gopkg.in/src-d/go-git.v4/plumbing"
+	"io/ioutil"
+	"log"
 	"os"
 	"path/filepath"
 	"syscall"
 	"testing"
 
+	"go.uber.org/zap"
+	"gopkg.in/src-d/go-git.v4"
+	gitConfig "gopkg.in/src-d/go-git.v4/config"
+	"gopkg.in/src-d/go-git.v4/plumbing"
+
 	"github.com/accurics/terrascan/pkg/iac-providers/output"
 	"github.com/accurics/terrascan/pkg/utils"
 	"github.com/hashicorp/go-multierror"
@@ -55,11 +59,11 @@ func TestLoadIacDir(t *testing.T) {
 	}
 
 	table := []struct {
+		wantErr error
+		want    output.AllResourceConfigs
+		cftv1   CFTV1
 		name    string
 		dirPath string
-		cftv1   CFTV1
-		want    output.AllResourceConfigs
-		wantErr error
 	}{
 		{
 			name:    "empty config",
@@ -69,8 +73,9 @@ func TestLoadIacDir(t *testing.T) {
 		},
 		{
 			name:    "load config dir with sub directories",
-			dirPath: testDataDir,
+			dirPath: filepath.Join(testDataDir, "templates"),
 			cftv1:   CFTV1{},
+			want:    map[string][]output.ResourceConfig{},
 			wantErr: nil,
 		},
 		{
@@ -81,8 +86,9 @@ func TestLoadIacDir(t *testing.T) {
 		},
 		{
 			name:    "load valid dir",
-			dirPath: filepath.Join(testDataDir, "s3"),
+			dirPath: filepath.Join(testDataDir, "templates", "s3"),
 			cftv1:   CFTV1{},
+			want:    map[string][]output.ResourceConfig{},
 			wantErr: nil,
 		},
 	}
@@ -107,23 +113,48 @@ func TestLoadIacDir(t *testing.T) {
 
 func TestCFTMapper(t *testing.T) {
 	root := filepath.Join(basePath, provider)
-	dirList := make([]string, 0)
-	err := filepath.Walk(root, func(filePath string, fileInfo os.FileInfo, err error) error {
-		if fileInfo != nil && fileInfo.IsDir() {
-			dirList = append(dirList, filePath)
-		}
-		return err
-	})
-
+	dirList, err := ioutil.ReadDir(root)
 	if err != nil {
-		t.Error(err)
+		log.Fatal(err)
 	}
 
 	cftv1 := CFTV1{}
-	for i := 1; i < len(dirList); i++ {
-		dir := dirList[i]
-		t.Run(dir, func(t *testing.T) {
-			_, gotErr := cftv1.LoadIacDir(dir, false)
+	for _, dir := range dirList {
+		resourceDir := filepath.Join(root, dir.Name())
+		t.Run(resourceDir, func(t *testing.T) {
+			allResourceConfigs, gotErr := cftv1.LoadIacDir(resourceDir, false)
+
+			// load expected output.json from test artifacts
+			var testArc output.AllResourceConfigs
+			outputData, err := ioutil.ReadFile(filepath.Join(resourceDir, "output.json"))
+			if err != nil {
+				t.Errorf("error reading output.json ResourceConfig, %T", err)
+			}
+
+			err = json.Unmarshal(outputData, &testArc)
+			if err != nil {
+				t.Errorf("error loading output.json ResourceConfig, %T", err)
+			}
+
+			// check if resourcetype and resources are present
+			for name, resources := range testArc {
+				if allResourceConfigs[name] == nil {
+					t.Errorf("resource Type %s from test data %s not found", name, resourceDir)
+				}
+				for _, testResource := range resources {
+					var found bool
+					for _, resource := range allResourceConfigs[name] {
+						if resource.ID == testResource.ID {
+							found = true
+							break
+						}
+					}
+					if !found {
+						t.Errorf("resource %s of type %s from test data not found", testResource.ID, name)
+					}
+				}
+			}
+
 			_, ok := gotErr.(*multierror.Error)
 			if !ok {
 				t.Errorf("expected multierror.Error, got %T", gotErr)

pkg/iac-providers/cft/v1/load-file.go

@@ -25,79 +25,70 @@ import (
 
 	"github.com/accurics/terrascan/pkg/iac-providers/output"
 	"github.com/accurics/terrascan/pkg/mapper"
-	"github.com/accurics/terrascan/pkg/utils"
+	"github.com/awslabs/goformation/v4"
+	"github.com/awslabs/goformation/v4/cloudformation"
 	"go.uber.org/zap"
 )
 
 // LoadIacFile loads the specified CFT template file.
 // Note that a single CFT template json file may contain multiple resource definitions.
 func (a *CFTV1) LoadIacFile(absFilePath string) (allResourcesConfig output.AllResourceConfigs, err error) {
-	var iacDocuments []*utils.IacDocument
-	fileExt := a.getFileType(absFilePath)
+	fileData, err := ioutil.ReadFile(absFilePath)
+	if err != nil {
+		zap.S().Debug("unable to read file", zap.Error(err), zap.String("file", absFilePath))
+		return allResourcesConfig, fmt.Errorf("unable to read file %s", absFilePath)
+	}
+
+	// parse the file as cloudformation.Template
+	fileExt := a.getFileType(absFilePath, &fileData)
+	var template *cloudformation.Template
 	switch fileExt {
-	case YAMLExtension:
-		fallthrough
-	case YAMLExtension2:
-		iacDocuments, err = utils.LoadYAML(absFilePath)
+	case YAMLExtension, YAMLExtension2:
+		template, err = goformation.ParseYAML(fileData)
+		if err != nil {
+			zap.S().Debug("failed to parse file", zap.String("file", absFilePath))
+			return allResourcesConfig, err
+		}
 	case JSONExtension:
-		iacDocuments, err = utils.LoadJSON(absFilePath)
+		template, err = goformation.ParseJSON(fileData)
+		if err != nil {
+			zap.S().Debug("failed to parse file", zap.String("file", absFilePath))
+			return allResourcesConfig, err
+		}
 	default:
 		zap.S().Debug("unknown extension found", zap.String("extension", fileExt))
-		return allResourcesConfig, fmt.Errorf("unknown file extension for file %s", absFilePath)
+		return allResourcesConfig, fmt.Errorf("unsupported extension for file %s", absFilePath)
 	}
+
+	// map resource to a terrascan type
+	m := mapper.NewMapper("cft")
+	configs, err := m.Map(template)
 	if err != nil {
-		zap.S().Debug("failed to load file", zap.String("file", absFilePath))
+		zap.S().Debug("unable to normalize data", zap.Error(err), zap.String("file", absFilePath))
 		return allResourcesConfig, err
 	}
-	allResourcesConfig = make(map[string][]output.ResourceConfig)
-	for _, doc := range iacDocuments {
-
-		// replacing yaml data as the default yaml.v3 removes
-		// intrinsic tags for cloudformation templates
-		// (!Ref, !Fn::<> etc are removed and resolved to a string
-		// which disables parameter resolution by goformation)
-		if fileExt != JSONExtension {
-			templateData, err := ioutil.ReadFile(absFilePath)
-			if err != nil {
-				zap.S().Debug("unable to read template data", zap.Error(err), zap.String("file", absFilePath))
-				return allResourcesConfig, err
-			}
-			doc.Data = templateData
-		}
 
-		var config *output.ResourceConfig
-		m := mapper.NewMapper("cft")
-		arc, err := m.Map(doc)
-		if err != nil {
-			zap.S().Debug("unable to normalize data", zap.Error(err), zap.String("file", absFilePath))
-			return allResourcesConfig, err
-		}
-		for t, resources := range arc {
-			for _, resource := range resources {
-				config = &resource
-				config.Type = t
-				config.Source = a.getSourceRelativePath(absFilePath)
-				allResourcesConfig[config.Type] = append(allResourcesConfig[config.Type], *config)
-			}
-		}
+	// fill AllResourceConfigs
+	allResourcesConfig = make(map[string][]output.ResourceConfig)
+	var config *output.ResourceConfig
+	for _, resource := range configs {
+		config = &resource
+		config.Line = 1
+		config.Source = a.getSourceRelativePath(absFilePath)
+		allResourcesConfig[config.Type] = append(allResourcesConfig[config.Type], *config)
 	}
 	return allResourcesConfig, nil
 }
 
-func (*CFTV1) getFileType(file string) string {
+func (*CFTV1) getFileType(file string, data *[]byte) string {
 	if strings.HasSuffix(file, YAMLExtension) {
 		return YAMLExtension
 	} else if strings.HasSuffix(file, YAMLExtension2) {
 		return YAMLExtension2
 	} else if strings.HasSuffix(file, JSONExtension) {
 		return JSONExtension
 	} else if strings.HasSuffix(file, TXTExtension) || strings.HasSuffix(file, TemplateExtension) {
-		f, err := ioutil.ReadFile(file)
-		if err != nil {
-			zap.S().Debug("unable to read file", zap.Error(err), zap.String("file", file))
-			return UnknownExtension
-		}
-		if isJSON(string(f)) {
+		if isJSON(string(*data)) {
 			return JSONExtension
 		}
 		return YAMLExtension

pkg/iac-providers/cft/v1/load-file_test.go

@@ -17,27 +17,69 @@
 package cftv1
 
 import (
+	"fmt"
+	"path"
+	"path/filepath"
 	"reflect"
 	"testing"
 
 	"github.com/accurics/terrascan/pkg/iac-providers/output"
 )
 
 func TestLoadIacFile(t *testing.T) {
+	testDataDir := "testdata"
+	testFile, _ := filepath.Abs(path.Join(testDataDir, "testfile"))
+	invalidFile, _ := filepath.Abs(path.Join(testDataDir, "deploy.yaml"))
+	validFile, _ := filepath.Abs(path.Join(testDataDir, "templates", "s3", "deploy.template"))
+
+	testErrString1 := fmt.Sprintf("unsupported extension for file %s", testFile)
+	testErrString2 := "unable to read file nonexistent.txt"
+	testErrString3 := "invalid YAML template: yaml: line 27: did not find expected alphabetic or numeric character"
+
 	table := []struct {
+		wantErr  error
+		want     output.AllResourceConfigs
+		cftv1    CFTV1
 		name     string
 		filePath string
-		cftv1    CFTV1
 		typeOnly bool
-		want     output.AllResourceConfigs
-		wantErr  error
-	}{}
+	}{
+		{
+			wantErr:  fmt.Errorf(testErrString1),
+			want:     output.AllResourceConfigs{},
+			cftv1:    CFTV1{},
+			name:     "invalid extension",
+			filePath: testFile,
+			typeOnly: false,
+		}, {
+			wantErr:  fmt.Errorf(testErrString2),
+			want:     output.AllResourceConfigs{},
+			cftv1:    CFTV1{},
+			name:     "nonexistent file",
+			filePath: "nonexistent.txt",
+			typeOnly: false,
+		}, {
+			wantErr:  fmt.Errorf(testErrString3),
+			want:     output.AllResourceConfigs{},
+			cftv1:    CFTV1{},
+			name:     "invalid file",
+			filePath: invalidFile,
+			typeOnly: false,
+		}, {
+			wantErr:  nil,
+			want:     output.AllResourceConfigs{},
+			cftv1:    CFTV1{},
+			name:     "invalid file",
+			filePath: validFile,
+			typeOnly: false,
+		},
+	}
 
 	for _, tt := range table {
 		t.Run(tt.name, func(t *testing.T) {
 			_, gotErr := tt.cftv1.LoadIacFile(tt.filePath)
 			if !reflect.DeepEqual(gotErr, tt.wantErr) {
-				t.Errorf("unexpected error; gotErr: '%v', wantErr: '%v'", gotErr, tt.wantErr)
+				t.Errorf("unexpected error; gotErr: '%+v', wantErr: '%+v'", gotErr, tt.wantErr)
 			} else if tt.typeOnly && (reflect.TypeOf(gotErr)) != reflect.TypeOf(tt.wantErr) {
 				t.Errorf("unexpected error; gotErr: '%v', wantErr: '%v'", reflect.TypeOf(gotErr), reflect.TypeOf(tt.wantErr))
 			}

pkg/iac-providers/cft/v1/testdata/deploy.yaml

@@ -0,0 +1,30 @@
+Resources:
+  S3Bucket:
+    Type: 'AWS::S3::Bucket'
+    Metadata:
+      runterrascan.io/skip: |
+        [{"rule": "AC_AWS_047", "comment": "reason to skip the rule"}]
+    Properties:
+      BucketName: DOC-EXAMPLE-BUCKET
+      AccessControl: Public
+      WebsiteConfiguration:
+        IndexDocument: index.html
+      PublicAccessBlockConfiguration:
+        BlockPublicAcls: true
+      BucketEncryption:
+        ServerSideEncryptionConfiguration:
+          - BucketKeyEnabled: true
+            ServerSideEncryptionByDefault:
+              KMSMasterKeyID: test
+              SSEAlgorithm: AES
+  SampleBucketPolicy:
+    Type: 'AWS::S3::BucketPolicy'
+    Properties:
+      Bucket: !Ref DOC-EXAMPLE-BUCKET
+      PolicyDocument:
+        Version: "2012-10-17"
+        Statement:
+          - Effect: Allow
+            Action: *
+            Principal: '*'
+