fix(analyzer): improve performance (#314)

Co-authored-by: knqyf263 <[email protected]>
aquasecurity · Dec 23, 2021 · 6726966 · 6726966
1 parent 1ba1250
commit 6726966
Show file tree

Hide file tree

Showing 65 changed files with 361 additions and 249 deletions.
diff --git a/analyzer/analyzer.go b/analyzer/analyzer.go
@@ -13,6 +13,7 @@ import (
 	aos "github.com/aquasecurity/fanal/analyzer/os"
 	"github.com/aquasecurity/fanal/log"
 	"github.com/aquasecurity/fanal/types"
+	dio "github.com/aquasecurity/go-dep-parser/pkg/io"
 )
 
 var (
@@ -30,7 +31,8 @@ var (
 type AnalysisTarget struct {
 	Dir      string
 	FilePath string
-	Content  []byte
+	Info     os.FileInfo
+	Content  dio.ReadSeekerAt
 }
 
 type analyzer interface {
@@ -55,7 +57,7 @@ func RegisterConfigAnalyzer(analyzer configAnalyzer) {
 	configAnalyzers[analyzer.Type()] = analyzer
 }
 
-type Opener func() ([]byte, error)
+type Opener func() (dio.ReadSeekCloserAt, error)
 
 type AnalysisResult struct {
 	m                    sync.Mutex
@@ -122,9 +124,7 @@ func (r *AnalysisResult) Merge(new *AnalysisResult) {
 		r.Applications = append(r.Applications, new.Applications...)
 	}
 
-	for _, m := range new.Configs {
-		r.Configs = append(r.Configs, m)
-	}
+	r.Configs = append(r.Configs, new.Configs...)
 
 	r.SystemInstalledFiles = append(r.SystemInstalledFiles, new.SystemInstalledFiles...)
 }
@@ -195,9 +195,9 @@ func (a Analyzer) AnalyzeFile(ctx context.Context, wg *sync.WaitGroup, limit *se
 		if !d.Required(strings.TrimLeft(filePath, "/"), info) {
 			continue
 		}
-		b, err := opener()
+		rc, err := opener()
 		if err != nil {
-			return xerrors.Errorf("unable to open a file (%s): %w", filePath, err)
+			return xerrors.Errorf("unable to open %s: %w", filePath, err)
 		}
 
 		if err = limit.Acquire(ctx, 1); err != nil {
@@ -208,15 +208,17 @@ func (a Analyzer) AnalyzeFile(ctx context.Context, wg *sync.WaitGroup, limit *se
 		go func(a analyzer, target AnalysisTarget) {
 			defer limit.Release(1)
 			defer wg.Done()
+			defer rc.Close()
 
 			ret, err := a.Analyze(ctx, target)
 			if err != nil && !xerrors.Is(err, aos.AnalyzeOSError) {
 				log.Logger.Debugf("Analysis error: %s", err)
 				return
 			}
 			result.Merge(ret)
-		}(d, AnalysisTarget{Dir: dir, FilePath: filePath, Content: b})
+		}(d, AnalysisTarget{Dir: dir, FilePath: filePath, Info: info, Content: rc})
 	}
+
 	return nil
 }
 

diff --git a/analyzer/analyzer_test.go b/analyzer/analyzer_test.go
@@ -18,6 +18,7 @@ import (
 	aos "github.com/aquasecurity/fanal/analyzer/os"
 	_ "github.com/aquasecurity/fanal/hook/all"
 	"github.com/aquasecurity/fanal/types"
+	dio "github.com/aquasecurity/go-dep-parser/pkg/io"
 )
 
 type mockConfigAnalyzer struct{}
@@ -349,7 +350,7 @@ func TestAnalyzeFile(t *testing.T) {
 				filePath:     "/lib/apk/db/installed",
 				testFilePath: "testdata/error",
 			},
-			wantErr: "unable to open a file (/lib/apk/db/installed)",
+			wantErr: "unable to open /lib/apk/db/installed",
 		},
 	}
 	for _, tt := range tests {
@@ -364,12 +365,14 @@ func TestAnalyzeFile(t *testing.T) {
 			require.NoError(t, err)
 
 			ctx := context.Background()
-			err = a.AnalyzeFile(ctx, &wg, limit, got, "", tt.args.filePath, info, func() ([]byte, error) {
-				if tt.args.testFilePath == "testdata/error" {
-					return nil, xerrors.New("error")
-				}
-				return os.ReadFile(tt.args.testFilePath)
-			})
+			err = a.AnalyzeFile(ctx, &wg, limit, got, "", tt.args.filePath, info,
+				func() (dio.ReadSeekCloserAt, error) {
+					if tt.args.testFilePath == "testdata/error" {
+						return nil, xerrors.New("error")
+					}
+					return os.Open(tt.args.testFilePath)
+				},
+			)
 
 			wg.Wait()
 			if tt.wantErr != "" {

diff --git a/analyzer/config/cloudformation/cloudformation.go b/analyzer/config/cloudformation/cloudformation.go
@@ -2,12 +2,14 @@ package cloudformation
 
 import (
 	"context"
+	"io"
 	"os"
 	"path/filepath"
 	"regexp"
 
 	"github.com/aquasecurity/fanal/analyzer"
 	"github.com/aquasecurity/fanal/types"
+	"golang.org/x/xerrors"
 )
 
 const version = 1
@@ -29,8 +31,12 @@ func NewConfigAnalyzer() ConfigAnalyzer {
 
 // Analyze returns a results of CloudFormation file
 func (a ConfigAnalyzer) Analyze(_ context.Context, target analyzer.AnalysisTarget) (*analyzer.AnalysisResult, error) {
+	content, err := io.ReadAll(target.Content)
+	if err != nil {
+		return nil, xerrors.Errorf("failed to read the CloudFormation file: %w", err)
+	}
 
-	if looksLikeCloudFormation(target.Content) {
+	if looksLikeCloudFormation(content) {
 		return &analyzer.AnalysisResult{
 			Configs: []types.Config{
 				{

diff --git a/analyzer/config/cloudformation/cloudformation_test.go b/analyzer/config/cloudformation/cloudformation_test.go
@@ -2,11 +2,14 @@ package cloudformation
 
 import (
 	"context"
+	"strings"
 	"testing"
 
-	"github.com/aquasecurity/fanal/analyzer"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+
+	"github.com/aquasecurity/fanal/analyzer"
+	dio "github.com/aquasecurity/go-dep-parser/pkg/io"
 )
 
 func TestConfigAnalyzer_Required(t *testing.T) {
@@ -48,25 +51,25 @@ func TestConfigAnalyzer_Required(t *testing.T) {
 func TestConfigAnalyzer_Analyzed(t *testing.T) {
 	tests := []struct {
 		name     string
-		content  string
+		content  dio.ReadSeekerAt
 		filePath string
 		want     int
 	}{
 		{
 			name: "CloudFormation yaml",
-			content: `---
+			content: strings.NewReader(`---
 Parameters:
   SomeParameter:
 Resources:
   SomeResource:
     Type: Something
-`,
+`),
 			filePath: "main.yaml",
 			want:     1,
 		},
 		{
 			name: "Cloudformation JSON",
-			content: `{
+			content: strings.NewReader(`{
   "Parameters": {
     "SomeParameter": null
   },
@@ -75,13 +78,13 @@ Resources:
       "Type": "Something"
     }
   }
-}`,
+}`),
 			filePath: "main.json",
 			want:     1,
 		},
 		{
 			name: "non CloudFormation yaml",
-			content: `---
+			content: strings.NewReader(`---
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -101,20 +104,20 @@ spec:
         image: nginx:1.14.2
         ports:
         - containerPort: 80
-`,
+`),
 			filePath: "k8s.yaml",
 			want:     0,
 		},
 		{
 			name: "non CloudFormation json",
-			content: `{
+			content: strings.NewReader(`{
   "foo": [ 
        "baaaaa", 
        "bar", 
        "baa"
     ]
 }
-`,
+`),
 			filePath: "random.yaml",
 			want:     0,
 		},
@@ -124,7 +127,7 @@ spec:
 			a := ConfigAnalyzer{}
 			got, err := a.Analyze(context.Background(), analyzer.AnalysisTarget{
 				FilePath: tt.filePath,
-				Content:  []byte(tt.content),
+				Content:  tt.content,
 			})
 			require.NoError(t, err)
 			assert.Len(t, got.Configs, tt.want)

diff --git a/analyzer/config/docker/docker_test.go b/analyzer/config/docker/docker_test.go
@@ -2,7 +2,7 @@ package docker_test
 
 import (
 	"context"
-	"io/ioutil"
+	"os"
 	"regexp"
 	"testing"
 
@@ -158,14 +158,15 @@ func Test_dockerConfigAnalyzer_Analyze(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			b, err := ioutil.ReadFile(tt.inputFile)
+			f, err := os.Open(tt.inputFile)
 			require.NoError(t, err)
+			defer f.Close()
 
 			a := docker.NewConfigAnalyzer(nil)
 			ctx := context.Background()
 			got, err := a.Analyze(ctx, analyzer.AnalysisTarget{
 				FilePath: tt.inputFile,
-				Content:  b,
+				Content:  f,
 			})
 
 			if tt.wantErr != "" {

diff --git a/analyzer/config/hcl/hcl.go b/analyzer/config/hcl/hcl.go
@@ -2,6 +2,7 @@ package hcl
 
 import (
 	"context"
+	"io"
 	"os"
 	"path/filepath"
 	"regexp"
@@ -52,13 +53,18 @@ func (a ConfigAnalyzer) analyze(target analyzer.AnalysisTarget) (interface{}, er
 	var errs error
 	var parsed interface{}
 
-	if err := hcl2.Unmarshal(target.Content, &parsed); err != nil {
+	content, err := io.ReadAll(target.Content)
+	if err != nil {
+		return nil, xerrors.Errorf("failed to read the HCL2 file: %w", err)
+	}
+
+	if err := hcl2.Unmarshal(content, &parsed); err != nil {
 		errs = multierror.Append(errs, xerrors.Errorf("unable to parse HCL2 (%s): %w", target.FilePath, err))
 	} else {
 		return parsed, nil
 	}
 
-	if err := hcl.Unmarshal(target.Content, &parsed); err != nil {
+	if err := hcl.Unmarshal(content, &parsed); err != nil {
 		errs = multierror.Append(errs, xerrors.Errorf("unable to parse HCL1 (%s): %w", target.FilePath, err))
 	} else {
 		return parsed, nil

diff --git a/analyzer/config/hcl/hcl_test.go b/analyzer/config/hcl/hcl_test.go
@@ -109,15 +109,16 @@ func TestConfigAnalyzer_Analyze(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			b, err := os.ReadFile(tt.inputFile)
+			f, err := os.Open(tt.inputFile)
 			require.NoError(t, err)
+			defer f.Close()
 
 			a := hcl.NewConfigAnalyzer(nil)
 			require.NoError(t, err)
 			ctx := context.Background()
 			got, err := a.Analyze(ctx, analyzer.AnalysisTarget{
 				FilePath: tt.inputFile,
-				Content:  b,
+				Content:  f,
 			})
 
 			if tt.wantErr != "" {

diff --git a/analyzer/config/json/json.go b/analyzer/config/json/json.go
@@ -32,8 +32,8 @@ func NewConfigAnalyzer(filePattern *regexp.Regexp) ConfigAnalyzer {
 
 func (a ConfigAnalyzer) Analyze(_ context.Context, target analyzer.AnalysisTarget) (*analyzer.AnalysisResult, error) {
 	var parsed interface{}
-	if err := json.Unmarshal(target.Content, &parsed); err != nil {
-		return nil, xerrors.Errorf("unable to parse JSON (%s): %w", target.FilePath, err)
+	if err := json.NewDecoder(target.Content).Decode(&parsed); err != nil {
+		return nil, xerrors.Errorf("unable to decode JSON (%s): %w", target.FilePath, err)
 	}
 
 	return &analyzer.AnalysisResult{

diff --git a/analyzer/config/json/json_test.go b/analyzer/config/json/json_test.go
@@ -2,7 +2,7 @@ package json_test
 
 import (
 	"context"
-	"io/ioutil"
+	"os"
 	"regexp"
 	"testing"
 
@@ -122,21 +122,22 @@ func Test_jsonConfigAnalyzer_Analyze(t *testing.T) {
 				policyPaths: []string{"../testdata/kubernetes.rego"},
 			},
 			inputFile: "testdata/broken.json",
-			wantErr:   "unable to parse JSON",
+			wantErr:   "unable to decode JSON",
 		},
 	}
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			b, err := ioutil.ReadFile(tt.inputFile)
+			f, err := os.Open(tt.inputFile)
 			require.NoError(t, err)
+			defer f.Close()
 
 			s := json.NewConfigAnalyzer(nil)
 
 			ctx := context.Background()
 			got, err := s.Analyze(ctx, analyzer.AnalysisTarget{
 				FilePath: tt.inputFile,
-				Content:  b,
+				Content:  f,
 			})
 
 			if tt.wantErr != "" {

diff --git a/analyzer/config/toml/toml.go b/analyzer/config/toml/toml.go
@@ -6,9 +6,9 @@ import (
 	"path/filepath"
 	"regexp"
 
-	"github.com/BurntSushi/toml"
 	"golang.org/x/xerrors"
 
+	"github.com/BurntSushi/toml"
 	"github.com/aquasecurity/fanal/analyzer"
 	"github.com/aquasecurity/fanal/types"
 )
@@ -29,8 +29,8 @@ func NewConfigAnalyzer(filePattern *regexp.Regexp) ConfigAnalyzer {
 
 func (a ConfigAnalyzer) Analyze(_ context.Context, target analyzer.AnalysisTarget) (*analyzer.AnalysisResult, error) {
 	var parsed interface{}
-	if err := toml.Unmarshal(target.Content, &parsed); err != nil {
-		return nil, xerrors.Errorf("unable to parse TOML (%s): %w", target.FilePath, err)
+	if _, err := toml.NewDecoder(target.Content).Decode(&parsed); err != nil {
+		return nil, xerrors.Errorf("unable to decode TOML (%s): %w", target.FilePath, err)
 	}
 
 	return &analyzer.AnalysisResult{