diff --git a/.vscode/launch.json b/.vscode/launch.json index f6348ed803..1d14786269 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -112,6 +112,19 @@ "-c", "gcp.project.kms.keyrings" ], + }, + { + "name": "cnquery-gcp-assets", + "type": "go", + "request": "launch", + "program": "${workspaceRoot}/apps/cnquery/cnquery.go", + "cwd": "${workspaceRoot}/", + "args": [ + "scan", + "gcp", + "--discover", + "compute-images" + ], } ] } \ No newline at end of file diff --git a/motor/discovery/gcp/discovery.go b/motor/discovery/gcp/discovery.go index fa5bc6ba05..a264e3e498 100644 --- a/motor/discovery/gcp/discovery.go +++ b/motor/discovery/gcp/discovery.go @@ -1,6 +1,7 @@ package gcp const ( - DiscoveryInstances = "instances" - DiscoveryProjects = "projects" + DiscoveryInstances = "instances" + DiscoveryProjects = "projects" + DiscoveryComputeImages = "compute-images" ) diff --git a/motor/discovery/gcp/mql_asset_objects.go b/motor/discovery/gcp/mql_asset_objects.go new file mode 100644 index 0000000000..03624e5bdb --- /dev/null +++ b/motor/discovery/gcp/mql_asset_objects.go @@ -0,0 +1,46 @@ +package gcp + +import ( + "github.com/cockroachdb/errors" + "go.mondoo.com/cnquery/motor/asset" + "go.mondoo.com/cnquery/motor/providers" +) + +func getTitleFamily(o gcpObject) (gcpObjectPlatformInfo, error) { + switch o.service { + case "compute": + if o.objectType == "image" { + return gcpObjectPlatformInfo{title: "GCP Compute Image", platform: "gcp-compute-image"}, nil + } + } + return gcpObjectPlatformInfo{}, errors.Newf("missing runtime info for gcp object service %s type %s", o.service, o.objectType) +} + +func computeImages(m *MqlDiscovery, project string, tc *providers.Config) []*asset.Asset { + assets := []*asset.Asset{} + images := m.GetList("gcp.project.compute.images { id name labels }") + for i := range images { + b := images[i].(map[string]interface{}) + id := b["id"].(string) + name := b["name"].(string) + tags := b["labels"].(map[string]interface{}) + stringLabels := make(map[string]string) + for k, v := range tags { + stringLabels[k] = v.(string) + } + + assets = append(assets, MqlObjectToAsset(project, + mqlObject{ + name: name, labels: stringLabels, + gcpObject: gcpObject{ + project: project, + region: "global", // Not region-based + name: name, + id: id, + service: "compute", + objectType: "image", + }, + }, tc)) + } + return assets +} diff --git a/motor/discovery/gcp/mql_assets.go b/motor/discovery/gcp/mql_assets.go new file mode 100644 index 0000000000..f678082ff9 --- /dev/null +++ b/motor/discovery/gcp/mql_assets.go @@ -0,0 +1,156 @@ +package gcp + +import ( + "errors" + + "github.com/mitchellh/mapstructure" + "github.com/rs/zerolog/log" + "go.mondoo.com/cnquery" + "go.mondoo.com/cnquery/llx" + "go.mondoo.com/cnquery/motor" + "go.mondoo.com/cnquery/motor/asset" + "go.mondoo.com/cnquery/motor/discovery/common" + "go.mondoo.com/cnquery/motor/platform" + "go.mondoo.com/cnquery/motor/providers" + gcpprovider "go.mondoo.com/cnquery/motor/providers/google" + "go.mondoo.com/cnquery/mql" + "go.mondoo.com/cnquery/resources" + resource_pack "go.mondoo.com/cnquery/resources/packs/gcp" +) + +const RegionLabel string = "mondoo.com/region" + +type MqlDiscovery struct { + rt *resources.Runtime +} + +func NewMQLAssetsDiscovery(provider *gcpprovider.Provider) (*MqlDiscovery, error) { + m, err := motor.New(provider) + if err != nil { + return nil, err + } + rt := resources.NewRuntime(resource_pack.Registry, m) + return &MqlDiscovery{rt: rt}, nil +} + +func (md *MqlDiscovery) Close() { + if md.rt != nil && md.rt.Motor != nil { + md.rt.Motor.Close() + } +} + +func (md *MqlDiscovery) GetList(query string) []interface{} { + mqlExecutor := mql.New(md.rt, cnquery.DefaultFeatures) + value, err := mqlExecutor.Exec(query, map[string]*llx.Primitive{}) + if err != nil { + return nil + } + + a := []interface{}{} + d, _ := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ + Result: &a, + }) + d.Decode(value.Value) + return a +} + +func GatherMQLObjects(tc *providers.Config, project string) ([]*asset.Asset, error) { + assets := []*asset.Asset{} + pCfg := tc.Clone() + at, err := gcpprovider.New(pCfg) + if err != nil { + return nil, err + } + m, err := NewMQLAssetsDiscovery(at) + if err != nil { + return nil, err + } + if tc.IncludesOneOfDiscoveryTarget(common.DiscoveryAll, common.DiscoveryAuto, DiscoveryComputeImages) { + assets = append(assets, computeImages(m, project, tc)...) + } + + return assets, nil +} + +type mqlObject struct { + name string + labels map[string]string + gcpObject gcpObject +} + +type gcpObject struct { + project string + region string + id string + service string + objectType string + name string +} + +type gcpObjectPlatformInfo struct { + title string + platform string +} + +func MondooObjectID(o gcpObject) string { + return "//platformid.api.mondoo.app/runtime/gcp/" + o.service + "/v1/projects/" + o.project + "/regions/" + o.region + "/" + o.objectType + "/" + o.name +} + +func MqlObjectToAsset(account string, mqlObject mqlObject, tc *providers.Config) *asset.Asset { + if mqlObject.name == "" { + mqlObject.name = mqlObject.gcpObject.id + } + if err := validate(mqlObject); err != nil { + log.Error().Err(err).Msg("missing values in mql object to asset translation") + return nil + } + info, err := getTitleFamily(mqlObject.gcpObject) + if err != nil { + log.Error().Err(err).Msg("missing runtime info") + return nil + } + platformid := MondooObjectID(mqlObject.gcpObject) + t := tc.Clone() + t.PlatformId = platformid + return &asset.Asset{ + PlatformIds: []string{platformid, mqlObject.gcpObject.id}, + Name: mqlObject.name, + Platform: &platform.Platform{ + Name: info.platform, + Title: info.title, + Kind: providers.Kind_KIND_GCP_OBJECT, + Runtime: providers.RUNTIME_GCP, + }, + State: asset.State_STATE_ONLINE, + Labels: addInformationalLabels(mqlObject.labels, mqlObject), + Connections: []*providers.Config{t}, + } +} + +func validate(m mqlObject) error { + if m.name == "" { + return errors.New("name required for mql gcp object to asset translation") + } + if m.gcpObject.id == "" { + return errors.New("id required for mql gcp object to asset translation") + } + if m.gcpObject.region == "" { + return errors.New("region required for mql gcp object to asset translation") + } + if m.gcpObject.project == "" { + return errors.New("project required for mql gcp object to asset translation") + } + if m.gcpObject.name == "" { + return errors.New("name required for mql gcp object to asset translation") + } + return nil +} + +func addInformationalLabels(l map[string]string, o mqlObject) map[string]string { + if l == nil { + l = make(map[string]string) + } + l[RegionLabel] = o.gcpObject.region + l[common.ParentId] = o.gcpObject.project + return l +} diff --git a/motor/discovery/gcp/resolver_gcp.go b/motor/discovery/gcp/resolver_gcp.go index 421e9f707a..4d4ecbbb90 100644 --- a/motor/discovery/gcp/resolver_gcp.go +++ b/motor/discovery/gcp/resolver_gcp.go @@ -17,7 +17,7 @@ func (k *GcpResolver) Name() string { } func (r *GcpResolver) AvailableDiscoveryTargets() []string { - return []string{common.DiscoveryAuto, common.DiscoveryAll, DiscoveryProjects, DiscoveryInstances} + return []string{common.DiscoveryAuto, common.DiscoveryAll, DiscoveryProjects, DiscoveryInstances, DiscoveryComputeImages} } func (r *GcpResolver) Resolve(ctx context.Context, root *asset.Asset, tc *providers.Config, cfn common.CredentialFn, sfn common.QuerySecretFn, userIdDetectors ...providers.PlatformIdDetector) ([]*asset.Asset, error) { diff --git a/motor/discovery/gcp/resolver_project.go b/motor/discovery/gcp/resolver_project.go index c76c78b787..78760963cf 100644 --- a/motor/discovery/gcp/resolver_project.go +++ b/motor/discovery/gcp/resolver_project.go @@ -21,7 +21,7 @@ func (k *GcpProjectResolver) Name() string { } func (r *GcpProjectResolver) AvailableDiscoveryTargets() []string { - return []string{common.DiscoveryAuto, common.DiscoveryAll, DiscoveryProjects, DiscoveryInstances} + return []string{common.DiscoveryAuto, common.DiscoveryAll, DiscoveryProjects, DiscoveryInstances, DiscoveryComputeImages} } func (r *GcpProjectResolver) Resolve(ctx context.Context, tc *providers.Config, cfn common.CredentialFn, sfn common.QuerySecretFn, userIdDetectors ...providers.PlatformIdDetector) ([]*asset.Asset, error) { @@ -65,8 +65,9 @@ func (r *GcpProjectResolver) Resolve(ctx context.Context, tc *providers.Config, } // ^^ + var resolvedRoot *asset.Asset if tc.IncludesOneOfDiscoveryTarget(common.DiscoveryAuto, common.DiscoveryAll, DiscoveryProjects) { - resolved = append(resolved, &asset.Asset{ + resolvedRoot = &asset.Asset{ PlatformIds: []string{identifier}, Name: "GCP project " + project, Platform: pf, @@ -74,7 +75,22 @@ func (r *GcpProjectResolver) Resolve(ctx context.Context, tc *providers.Config, Labels: map[string]string{ common.ParentId: project, }, - }) + } + resolved = append(resolved, resolvedRoot) + } + + if tc.IncludesOneOfDiscoveryTarget(common.DiscoveryAll, common.DiscoveryAuto, DiscoveryComputeImages) { + assetList, err := GatherMQLObjects(tc, project) + if err != nil { + return nil, err + } + for i := range assetList { + a := assetList[i] + if resolvedRoot != nil { + a.RelatedAssets = append(a.RelatedAssets, resolvedRoot) + } + resolved = append(resolved, a) + } } // discover compute instances diff --git a/motor/providers/provider.pb.go b/motor/providers/provider.pb.go index 60a886eb35..22ee1e619b 100644 --- a/motor/providers/provider.pb.go +++ b/motor/providers/provider.pb.go @@ -174,6 +174,7 @@ const ( Kind_KIND_BARE_METAL Kind = 9 Kind_KIND_NETWORK Kind = 10 Kind_KIND_K8S_OBJECT Kind = 11 + Kind_KIND_GCP_OBJECT Kind = 13 ) // Enum value maps for Kind. @@ -191,6 +192,7 @@ var ( 9: "KIND_BARE_METAL", 10: "KIND_NETWORK", 11: "KIND_K8S_OBJECT", + 13: "KIND_GCP_OBJECT", } Kind_value = map[string]int32{ "KIND_UNKNOWN": 0, @@ -205,6 +207,7 @@ var ( "KIND_BARE_METAL": 9, "KIND_NETWORK": 10, "KIND_K8S_OBJECT": 11, + "KIND_GCP_OBJECT": 13, } ) @@ -595,7 +598,7 @@ var file_provider_proto_rawDesc = []byte{ 0x10, 0x1d, 0x12, 0x14, 0x0a, 0x10, 0x47, 0x4f, 0x4f, 0x47, 0x4c, 0x45, 0x5f, 0x57, 0x4f, 0x52, 0x4b, 0x53, 0x50, 0x41, 0x43, 0x45, 0x10, 0x1e, 0x12, 0x09, 0x0a, 0x05, 0x53, 0x4c, 0x41, 0x43, 0x4b, 0x10, 0x1f, 0x12, 0x07, 0x0a, 0x03, 0x56, 0x43, 0x44, 0x10, 0x20, 0x22, 0x04, 0x08, 0x0b, - 0x10, 0x0b, 0x2a, 0xfd, 0x01, 0x0a, 0x04, 0x4b, 0x69, 0x6e, 0x64, 0x12, 0x10, 0x0a, 0x0c, 0x4b, + 0x10, 0x0b, 0x2a, 0x92, 0x02, 0x0a, 0x04, 0x4b, 0x69, 0x6e, 0x64, 0x12, 0x10, 0x0a, 0x0c, 0x4b, 0x49, 0x4e, 0x44, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x1e, 0x0a, 0x1a, 0x4b, 0x49, 0x4e, 0x44, 0x5f, 0x56, 0x49, 0x52, 0x54, 0x55, 0x41, 0x4c, 0x5f, 0x4d, 0x41, 0x43, 0x48, 0x49, 0x4e, 0x45, 0x5f, 0x49, 0x4d, 0x41, 0x47, 0x45, 0x10, 0x01, 0x12, 0x18, 0x0a, @@ -611,10 +614,11 @@ var file_provider_proto_rawDesc = []byte{ 0x41, 0x52, 0x45, 0x5f, 0x4d, 0x45, 0x54, 0x41, 0x4c, 0x10, 0x09, 0x12, 0x10, 0x0a, 0x0c, 0x4b, 0x49, 0x4e, 0x44, 0x5f, 0x4e, 0x45, 0x54, 0x57, 0x4f, 0x52, 0x4b, 0x10, 0x0a, 0x12, 0x13, 0x0a, 0x0f, 0x4b, 0x49, 0x4e, 0x44, 0x5f, 0x4b, 0x38, 0x53, 0x5f, 0x4f, 0x42, 0x4a, 0x45, 0x43, 0x54, - 0x10, 0x0b, 0x42, 0x27, 0x5a, 0x25, 0x67, 0x6f, 0x2e, 0x6d, 0x6f, 0x6e, 0x64, 0x6f, 0x6f, 0x2e, - 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x6e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x2f, 0x6d, 0x6f, 0x74, 0x6f, - 0x72, 0x2f, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x73, 0x62, 0x06, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x33, + 0x10, 0x0b, 0x12, 0x13, 0x0a, 0x0f, 0x4b, 0x49, 0x4e, 0x44, 0x5f, 0x47, 0x43, 0x50, 0x5f, 0x4f, + 0x42, 0x4a, 0x45, 0x43, 0x54, 0x10, 0x0d, 0x42, 0x27, 0x5a, 0x25, 0x67, 0x6f, 0x2e, 0x6d, 0x6f, + 0x6e, 0x64, 0x6f, 0x6f, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x6e, 0x71, 0x75, 0x65, 0x72, 0x79, + 0x2f, 0x6d, 0x6f, 0x74, 0x6f, 0x72, 0x2f, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x73, + 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/motor/providers/provider.proto b/motor/providers/provider.proto index f752af798d..4af5b148aa 100644 --- a/motor/providers/provider.proto +++ b/motor/providers/provider.proto @@ -58,6 +58,7 @@ enum Kind { KIND_BARE_METAL = 9; KIND_NETWORK = 10; KIND_K8S_OBJECT = 11; + KIND_GCP_OBJECT = 13; } message Config {