⭐ add structure for explorer

Signed-off-by: Dominik Richter <[email protected]>

Makefile

@@ -58,7 +58,7 @@ prep/tools:
 
 #   🌙 MQL/MOTOR   #
 
-cnquery/generate: clean/proto motor/generate resources/generate llx/generate lr shared/generate
+cnquery/generate: clean/proto motor/generate resources/generate llx/generate lr shared/generate explorer/generate
 
 motor/generate:
 	go generate .
@@ -199,6 +199,9 @@ mqlc: | llx mqlc/test
 mqlc/test:
 	go test -timeout 5s $(shell go list ./mqlc/... | grep -v '/vendor/')
 
+explorer/generate:
+	go generate ./explorer
+
 #   🏗 Binary / Build   #
 
 .PHONY: cnquery/install

explorer/bundle.go

@@ -0,0 +1,259 @@
+package explorer
+
+import (
+	"context"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/pkg/errors"
+	"github.com/rs/zerolog/log"
+	"go.mondoo.com/cnquery/checksums"
+	llx "go.mondoo.com/cnquery/llx"
+	"sigs.k8s.io/yaml"
+)
+
+const (
+	MRN_RESOURCE_QUERY     = "queries"
+	MRN_RESOURCE_QUERYPACK = "querypack"
+	MRN_RESOURCE_ASSET     = "assets"
+)
+
+// BundleMap is a Bundle with easier access to its data
+type BundleMap struct {
+	OwnerMrn string                     `json:"owner_mrn,omitempty"`
+	Packs    map[string]*QueryPack      `json:"packs,omitempty"`
+	Queries  map[string]*Mquery         `json:"queries,omitempty"`
+	Code     map[string]*llx.CodeBundle `json:"code,omitempty"`
+	Library  Library                    `json:"library,omitempty"`
+}
+
+// NewBundleMap creates a new empty initialized map
+// dataLake (optional) connects an additional data layer which may provide queries/policies
+func NewBundleMap(ownerMrn string) *BundleMap {
+	return &BundleMap{
+		OwnerMrn: ownerMrn,
+		Packs:    make(map[string]*QueryPack),
+		Queries:  make(map[string]*Mquery),
+		Code:     make(map[string]*llx.CodeBundle),
+	}
+}
+
+// BundleFromPaths loads a single bundle file or a bundle that
+// was split into multiple files into a single Bundle struct
+func BundleFromPaths(paths ...string) (*Bundle, error) {
+	// load all the source files
+	resolvedFilenames, err := walkBundleFiles(paths)
+	if err != nil {
+		log.Error().Err(err).Msg("could not resolve bundle files")
+		return nil, err
+	}
+
+	// aggregate all files into a single bundle
+	aggregatedBundle, err := aggregateFilesToBundle(resolvedFilenames)
+	if err != nil {
+		log.Error().Err(err).Msg("could merge bundle files")
+		return nil, err
+	}
+	return aggregatedBundle, nil
+}
+
+// walkBundleFiles iterates over all provided filenames and
+// checks if the name is a file or a directory. If the filename
+// is a directory, it walks the directory recursively
+func walkBundleFiles(filenames []string) ([]string, error) {
+	// resolve file names
+	resolvedFilenames := []string{}
+	for i := range filenames {
+		filename := filenames[i]
+		fi, err := os.Stat(filename)
+		if err != nil {
+			return nil, errors.Wrap(err, "could not load bundle file: "+filename)
+		}
+
+		if fi.IsDir() {
+			filepath.WalkDir(filename, func(path string, d fs.DirEntry, err error) error {
+				if err != nil {
+					return err
+				}
+				// we ignore nested directories
+				if d.IsDir() {
+					return nil
+				}
+
+				// only consider .yaml|.yml files
+				if strings.HasSuffix(d.Name(), ".yaml") || strings.HasSuffix(d.Name(), ".yml") {
+					resolvedFilenames = append(resolvedFilenames, path)
+				}
+
+				return nil
+			})
+		} else {
+			resolvedFilenames = append(resolvedFilenames, filename)
+		}
+	}
+
+	return resolvedFilenames, nil
+}
+
+// aggregateFilesToBundle iterates over all provided files and loads its content.
+// It assumes that all provided files are checked upfront and are not a directory
+func aggregateFilesToBundle(paths []string) (*Bundle, error) {
+	// iterate over all files, load them and merge them
+	mergedBundle := &Bundle{}
+
+	for i := range paths {
+		path := paths[i]
+		bundle, err := bundleFromSingleFile(path)
+		if err != nil {
+			return nil, errors.Wrap(err, "could not load file: "+path)
+		}
+
+		mergedBundle.AddBundle(bundle)
+	}
+
+	return mergedBundle, nil
+}
+
+// bundleFromSingleFile loads a bundle from a single file
+func bundleFromSingleFile(path string) (*Bundle, error) {
+	bundleData, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+
+	return BundleFromYAML(bundleData)
+}
+
+// BundleFromYAML create a policy bundle from yaml contents
+func BundleFromYAML(data []byte) (*Bundle, error) {
+	var res Bundle
+	err := yaml.Unmarshal(data, &res)
+	return &res, err
+}
+
+// ToYAML returns the policy bundle as yaml
+func (p *Bundle) ToYAML() ([]byte, error) {
+	return yaml.Marshal(p)
+}
+
+func (p *Bundle) SourceHash() (string, error) {
+	raw, err := p.ToYAML()
+	if err != nil {
+		return "", err
+	}
+	c := checksums.New
+	c = c.Add(string(raw))
+	return c.String(), nil
+}
+
+// ToMap turns the PolicyBundle into a BundleMap
+// dataLake (optional) may be used to provide queries/policies not found in the bundle
+func (p *Bundle) ToMap() *BundleMap {
+	res := NewBundleMap(p.OwnerMrn)
+
+	for i := range p.Packs {
+		c := p.Packs[i]
+		res.Packs[c.Mrn] = c
+		for j := range c.Queries {
+			cq := c.Queries[j]
+			res.Queries[cq.Mrn] = cq
+		}
+	}
+
+	return res
+}
+
+// Add another policy bundle into this. No duplicate policies, queries, or
+// properties are allowed and will lead to an error. Both bundles must have
+// MRNs for everything. OwnerMRNs must be identical as well.
+func (p *Bundle) AddBundle(other *Bundle) error {
+	if p.OwnerMrn == "" {
+		p.OwnerMrn = other.OwnerMrn
+	} else if p.OwnerMrn != other.OwnerMrn {
+		return errors.New("when combining policy bundles the owner MRNs must be identical")
+	}
+
+	for i := range other.Packs {
+		c := other.Packs[i]
+		if c.Mrn == "" {
+			return errors.New("source bundle that is added has missing policy MRNs")
+		}
+
+		for j := range p.Packs {
+			if p.Packs[j].Mrn == c.Mrn {
+				return errors.New("cannot combine query packs, duplicate policy: " + c.Mrn)
+			}
+		}
+
+		p.Packs = append(p.Packs, c)
+	}
+
+	return nil
+}
+
+// Compile PolicyBundle into a BundleMap
+// Does 4 things:
+// 1. turns it into a map for easier access
+// 2. compile all queries. store code in the bundle map
+// 3. validation of all contents
+// 4. generate MRNs for all packs, queries, and updates referencing local fields
+func (p *Bundle) Compile(ctx context.Context) (*BundleMap, error) {
+	ownerMrn := p.OwnerMrn
+	if ownerMrn == "" {
+		return nil, errors.New("failed to compile bundle, the owner MRN is empty")
+	}
+
+	var warnings []error
+
+	code := map[string]*llx.CodeBundle{}
+
+	// Index policies + update MRNs and checksums, link properties via MRNs
+	for i := range p.Packs {
+		querypack := p.Packs[i]
+
+		// !this is very important to prevent user overrides! vv
+		querypack.InvalidateAllChecksums()
+
+		err := querypack.RefreshMRN(ownerMrn)
+		if err != nil {
+			return nil, errors.New("failed to refresh policy " + querypack.Mrn + ": " + err.Error())
+		}
+
+		for i := range querypack.Queries {
+			query := querypack.Queries[i]
+
+			// remove leading and trailing whitespace of docs, refs and tags
+			query.Sanitize()
+
+			// ensure the correct mrn is set
+			if err = query.RefreshMRN(ownerMrn); err != nil {
+				return nil, err
+			}
+
+			// recalculate the checksums
+			codeBundle, err := query.RefreshChecksumAndType(nil)
+			if err != nil {
+				log.Error().Err(err).Msg("could not compile the query")
+				warnings = append(warnings, errors.Wrap(err, "failed to validate query '"+query.Mrn+"'"))
+			}
+
+			code[query.Mrn] = codeBundle
+		}
+	}
+
+	res := p.ToMap()
+	res.Code = code
+
+	if len(warnings) != 0 {
+		var msg strings.Builder
+		for i := range warnings {
+			msg.WriteString(warnings[i].Error())
+			msg.WriteString("\n")
+		}
+		return res, errors.New(msg.String())
+	}
+
+	return res, nil
+}

explorer/datalake.go

@@ -0,0 +1,30 @@
+package explorer
+
+import "context"
+
+// DataLake provides additional database calls, that are not accessible to
+// external users. We use them with specialized tools only. This limits the
+// potential exposure to underlying data and reduces the surface for breaking
+// changes.
+type DataLake interface {
+	// GetQuery retrieves a given query
+	GetQuery(ctx context.Context, mrn string) (*Mquery, error)
+	// SetQuery stores a given query
+	// Note: the query must be defined, it cannot be nil
+	SetQuery(ctx context.Context, mrn string, query *Mquery) error
+
+	// SetQueryPack stores a given pack in the data lake
+	SetQueryPack(ctx context.Context, querypack *QueryPack, filters []*Mquery) error
+	// GetQueryPack retrieves and if necessary updates the pack
+	GetQueryPack(ctx context.Context, mrn string) (*QueryPack, error)
+	// DeleteQueryPack removes a given pack
+	// Note: the MRN has to be valid
+	DeleteQueryPack(ctx context.Context, mrn string) error
+	// GetBundle retrieves and if necessary updates the bundle
+	GetBundle(ctx context.Context, mrn string) (*Bundle, error)
+	// List all packs for a given owner
+	// Note: Owner MRN is required
+	ListQueryPacks(ctx context.Context, ownerMrn string, name string) ([]*QueryPack, error)
+	// GetQueryPackFilters retrieves the list of asset filters for a pack (fast)
+	GetQueryPackFilters(ctx context.Context, mrn string) ([]*Mquery, error)
+}

explorer/explorer.go

@@ -0,0 +1,3 @@
+package explorer
+
+//go:generate protoc --proto_path=../:. --go_out=. --go_opt=paths=source_relative --rangerrpc_out=. explorer.proto