Feature/endpoint policy download (#1200)

* phase 1 * download commercial poilcies phase 2 * phase 3 * unescape characters in json metadata * add test data * add test case for conversion * fix variable name * status code check add * add response status code * add comments for exported functions * make empty docker folder to satisfy dir structure * change to use 'environment' keyword * env keyword, %w to wrap errors * add file hader * wrap errors in %w * rm tabsapce const * addressing review comments * use bytes.equal * add method for getType, constructor for newPolicy * changes: 1. save IO operations, avoid overwriting rego code file 2. wrap errors wherever required * invalid policy test cases * add table error cases structure * minor fix * docker error return nil check * no error if response code 404 Co-authored-by: Gaurav Gogia <[email protected]>
tenable · Mar 31, 2022 · d818b60 · d818b60
1 parent 830e079
commit d818b60
Show file tree

Hide file tree

Showing 15 changed files with 822 additions and 18 deletions.
diff --git a/pkg/config/global.go b/pkg/config/global.go
@@ -18,14 +18,16 @@ package config
 
 import (
 	"path/filepath"
+	"strings"
 
 	"github.com/accurics/terrascan/pkg/utils"
 	"go.uber.org/zap"
 )
 
 const (
-	defaultPolicyRepoURL = "https://github.com/accurics/terrascan.git"
-	defaultPolicyBranch  = "master"
+	defaultPolicyRepoURL     = "https://github.com/accurics/terrascan.git"
+	defaultPolicyBranch      = "master"
+	defaultPolicyEnvironment = "https://cloud.tenable.com"
 )
 
 // ConfigEnvvarName env variable
@@ -92,6 +94,12 @@ func LoadGlobalConfig(configFile string) error {
 	if len(configReader.getPolicyConfig().Branch) > 0 {
 		global.Policy.Branch = configReader.getPolicyConfig().Branch
 	}
+	if len(configReader.getPolicyConfig().Environment) > 0 {
+		global.Policy.Environment = configReader.getPolicyConfig().Environment
+	}
+	if len(configReader.getPolicyConfig().AccessToken) > 0 {
+		global.Policy.AccessToken = configReader.getPolicyConfig().AccessToken
+	}
 
 	if len(configReader.getRules().ScanRules) > 0 {
 		global.Rules.ScanRules = configReader.getRules().ScanRules
@@ -152,6 +160,22 @@ func GetPolicyBranch() string {
 	return global.Policy.Branch
 }
 
+// GetPolicyEnvironment returns the configured policy environment url
+func GetPolicyEnvironment() string {
+	if global == nil {
+		return defaultPolicyEnvironment
+	}
+	return strings.TrimRight(global.Policy.Environment, "/")
+}
+
+// GetPolicyAccessToken returns the configured policy access token
+func GetPolicyAccessToken() string {
+	if global == nil {
+		return ""
+	}
+	return global.Policy.AccessToken
+}
+
 // GetScanRules returns the configured scan rules
 func GetScanRules() []string {
 	if global == nil {

diff --git a/pkg/config/types.go b/pkg/config/types.go
@@ -49,6 +49,10 @@ type Policy struct {
 	// policy git url and branch
 	RepoURL string `toml:"repo_url,omitempty" yaml:"repo_url,omitempty"`
 	Branch  string `toml:"branch,omitempty" yaml:"branch,omitempty"`
+
+	// policy environment and access token
+	Environment string `toml:"environment,omitempty" yaml:"environment,omitempty"`
+	AccessToken string `toml:"access_token,omitempty" yaml:"access_token,omitempty"`
 }
 
 // Notifier represent a single notification in the terrascan config file

diff --git a/pkg/initialize/run.go b/pkg/initialize/run.go
@@ -17,9 +17,14 @@
 package initialize
 
 import (
+	"bytes"
+	"encoding/json"
 	"fmt"
+	"io/fs"
+	"io/ioutil"
 	"net/http"
 	"os"
+	"path/filepath"
 
 	"github.com/accurics/terrascan/pkg/config"
 	"go.uber.org/zap"
@@ -33,6 +38,7 @@ var (
 )
 
 const terrascanReadmeURL string = "https://raw.githubusercontent.com/accurics/terrascan/master/README.md"
+const filePermissionBits fs.FileMode = 0755
 
 // Run initializes terrascan if not done already
 func Run(isNonInitCmd bool) error {
@@ -45,10 +51,6 @@ func Run(isNonInitCmd bool) error {
 
 	zap.S().Debug("initializing terrascan")
 
-	if !connected(terrascanReadmeURL) {
-		return errNoConnection
-	}
-
 	// download policies
 	if err := DownloadPolicies(); err != nil {
 		return err
@@ -60,20 +62,169 @@ func Run(isNonInitCmd bool) error {
 
 // DownloadPolicies clones the policies to a local folder
 func DownloadPolicies() error {
-
+	accessToken := config.GetPolicyAccessToken()
 	policyBasePath := config.GetPolicyBasePath()
-	repoURL := config.GetPolicyRepoURL()
-	branch := config.GetPolicyBranch()
 
 	zap.S().Debug("downloading policies")
-
 	zap.S().Debugf("base directory path : %s", policyBasePath)
-	zap.S().Debugf("policy directory path : %s", config.GetPolicyRepoPath())
-	zap.S().Debugf("policy repo url : %s", repoURL)
-	zap.S().Debugf("policy repo git branch : %s", branch)
 
-	os.RemoveAll(policyBasePath)
+	err := os.RemoveAll(policyBasePath)
+	if err != nil {
+		return fmt.Errorf("unable to delete base folder. error: '%w'", err)
+	}
+
+	if accessToken == "" {
+		return downloadDefaultPolicies(policyBasePath)
+	}
+
+	return dowloadEnvironmentPolicies(policyBasePath, accessToken)
+}
+
+func dowloadEnvironmentPolicies(policyBasePath, accessToken string) error {
+	err := ensureDir(policyBasePath)
+	if err != nil {
+		return err
+	}
+
+	policyRepoPath := config.GetPolicyRepoPath()
+	err = os.MkdirAll(policyRepoPath, filePermissionBits)
+	if err != nil {
+		return fmt.Errorf("unable to prepare directories representing policyRepoPath. err: '%w', policyRepoPath: '%s'", err, policyRepoPath)
+	}
+
+	const apiPath = "/v1/api/app/rules?default=true"
+	environment := config.GetPolicyEnvironment()
 
+	zap.S().Debugf("policy environment : %s", environment)
+	zap.S().Debugf("downloading environment policies in %s", policyBasePath)
+
+	var client http.Client
+
+	req, err := http.NewRequest(http.MethodGet, environment+apiPath, nil)
+	if err != nil {
+		return fmt.Errorf("error constructing request object. error: '%w'", err)
+	}
+	req.Header.Add("Authorization", "Bearer "+accessToken)
+
+	res, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("error downloading environment policies. error: '%w'", err)
+	}
+	if res.StatusCode != 200 {
+		return fmt.Errorf("error downloading environment policies, response status code: '%d'", res.StatusCode)
+	}
+
+	policies, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		return fmt.Errorf("error reading api call response for environment policies. error: '%w'", err)
+	}
+
+	err = convertEnvironmentPolicies(policies, policyRepoPath)
+	if err != nil {
+		return err
+	}
+
+	// creating empty docker folder to satisfy folder structure dep
+	dockerPath := filepath.Join(policyRepoPath, "docker")
+	err = os.Mkdir(dockerPath, filePermissionBits)
+	if err != nil {
+		return fmt.Errorf("unable to create empty docker dir. error: '%w'", err)
+	}
+
+	return nil
+}
+
+func convertEnvironmentPolicies(policies []byte, policyRepoPath string) error {
+	var ruleMetadataList []environmentPolicyMetadata
+
+	err := json.Unmarshal(policies, &ruleMetadataList)
+	if err != nil {
+		return fmt.Errorf("failed to unmarshal policies into structure. error: '%w'", err)
+	}
+
+	for _, ruleMetadata := range ruleMetadataList {
+		policy, err := newPolicy(ruleMetadata)
+		if err != nil {
+			return err
+		}
+
+		err = saveEnvironmentPolicies(policy, policyRepoPath)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func saveEnvironmentPolicies(policy environmentPolicy, policyRepoPath string) error {
+	policy.policyMetadata.PolicyType = policy.getType()
+	cspDir := filepath.Join(policyRepoPath, policy.policyMetadata.PolicyType)
+	err := ensureDir(cspDir)
+	if err != nil {
+		return err
+	}
+
+	resourceDir := filepath.Join(cspDir, policy.resourceType)
+	err = ensureDir(resourceDir)
+	if err != nil {
+		return err
+	}
+
+	var buffer bytes.Buffer
+	encoder := json.NewEncoder(&buffer)
+	encoder.SetEscapeHTML(false)
+	encoder.SetIndent("", "    ")
+	err = encoder.Encode(policy.policyMetadata)
+	if err != nil {
+		return fmt.Errorf("could not marshal json object into byte array. error: '%w'", err)
+	}
+
+	metadata := buffer.Bytes()
+	metadata = bytes.TrimRight(metadata, "\n")
+	metaDataPath := filepath.Join(resourceDir, policy.metadataFileName)
+	err = ioutil.WriteFile(metaDataPath, metadata, os.ModePerm)
+	if err != nil {
+		return fmt.Errorf("could not write rule metadata file on disk. error: '%w'", err)
+	}
+
+	regoPath := filepath.Join(resourceDir, policy.policyMetadata.File)
+
+	if _, err := os.Stat(regoPath); os.IsExist(err) {
+		zap.S().Debug("rego code file %s exists, skipping", regoPath)
+		return nil
+	}
+
+	err = ioutil.WriteFile(regoPath, []byte(policy.regoTemplate), filePermissionBits)
+	if err != nil {
+		return fmt.Errorf("could not write rego code file on disk. error: '%w'", err)
+	}
+
+	return nil
+}
+
+func ensureDir(path string) error {
+	_, err := os.Stat(path)
+	if os.IsNotExist(err) {
+		err = os.Mkdir(path, filePermissionBits)
+		if err != nil {
+			return fmt.Errorf("unable to create requested directory error: '%w' for path: '%s'", err, path)
+		}
+	}
+	return nil
+}
+
+func downloadDefaultPolicies(policyBasePath string) error {
+	if !connected(terrascanReadmeURL) {
+		return errNoConnection
+	}
+
+	repoURL := config.GetPolicyRepoURL()
+	branch := config.GetPolicyBranch()
+
+	zap.S().Debugf("policy directory path : %s", repoURL)
+	zap.S().Debugf("policy repo url : %s", repoURL)
+	zap.S().Debugf("policy repo git branch : %s", branch)
 	zap.S().Debugf("cloning terrascan repo at %s", policyBasePath)
 
 	// clone the repo
@@ -82,21 +233,21 @@ func DownloadPolicies() error {
 	})
 
 	if err != nil {
-		return fmt.Errorf("failed to download policies. error: '%v'", err)
+		return fmt.Errorf("failed to download policies. error: '%w'", err)
 	}
 
 	// create working tree
 	w, err := r.Worktree()
 	if err != nil {
-		return fmt.Errorf("failed to create working tree. error: '%v'", err)
+		return fmt.Errorf("failed to create working tree. error: '%w'", err)
 	}
 
 	// fetch references
 	err = r.Fetch(&git.FetchOptions{
 		RefSpecs: []gitConfig.RefSpec{"refs/*:refs/*", "HEAD:refs/heads/HEAD"},
 	})
 	if err != nil {
-		return fmt.Errorf("failed to fetch references from git repo. error: '%v'", err)
+		return fmt.Errorf("failed to fetch references from git repo. error: '%w'", err)
 	}
 
 	// checkout policies branch
@@ -105,7 +256,7 @@ func DownloadPolicies() error {
 		Force:  true,
 	})
 	if err != nil {
-		return fmt.Errorf("failed to checkout git branch '%v'. error: '%v'", branch, err)
+		return fmt.Errorf("failed to checkout git branch '%s'. error: '%w'", branch, err)
 	}
 
 	return nil