Skip to content

Commit

Permalink
⭐️ Backport GCP org and folder scanning to v7 (#958)
Browse files Browse the repository at this point in the history
- #954 
- #892 
- #960

---------

Signed-off-by: Ivan Milchev <[email protected]>
  • Loading branch information
imilchev authored Feb 22, 2023
1 parent d1fb6ac commit 10ca6d6
Show file tree
Hide file tree
Showing 19 changed files with 1,984 additions and 270 deletions.
58 changes: 58 additions & 0 deletions apps/cnquery/cmd/builder/builder.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,9 @@ const (
Ec2ebsInstanceAssetType
Ec2ebsVolumeAssetType
Ec2ebsSnapshotAssetType
GcpOrganizationAssetType
GcpProjectAssetType
GcpFolderAssetType
GcrContainerRegistryAssetType
GithubOrganizationAssetType
GithubRepositoryAssetType
Expand Down Expand Up @@ -121,6 +124,9 @@ func buildCmd(baseCmd *cobra.Command, commonCmdFlags commonFlagsFn, preRun commo
gcpCmd := scanGcpCmd(commonCmdFlags, preRun, runFn, docs)
gcpGcrCmd := scanGcpGcrCmd(commonCmdFlags, preRun, runFn, docs)
gcpCmd.AddCommand(gcpGcrCmd)
gcpCmd.AddCommand(scanGcpOrgCmd(commonCmdFlags, preRun, runFn, docs))
gcpCmd.AddCommand(scanGcpProjectCmd(commonCmdFlags, preRun, runFn, docs))
gcpCmd.AddCommand(scanGcpFolderCmd(commonCmdFlags, preRun, runFn, docs))

// vsphere subcommand
vsphereCmd := vsphereProviderCmd(commonCmdFlags, preRun, runFn, docs)
Expand Down Expand Up @@ -620,6 +626,58 @@ func scanGcpCmd(commonCmdFlags commonFlagsFn, preRun commonPreRunFn, runFn runFn
return cmd
}

func scanGcpOrgCmd(commonCmdFlags commonFlagsFn, preRun commonPreRunFn, runFn runFn, docs CommandsDocs) *cobra.Command {
cmd := &cobra.Command{
Use: "org ORGANIZATION-ID",
Aliases: []string{"organization"},
Short: docs.GetShort("gcp-org"),
Long: docs.GetLong("gcp-org"),
Args: cobra.ExactArgs(1),
PreRun: func(cmd *cobra.Command, args []string) {
preRun(cmd, args)
},
Run: func(cmd *cobra.Command, args []string) {
runFn(cmd, args, providers.ProviderType_GCP, GcpOrganizationAssetType)
},
}
commonCmdFlags(cmd)
return cmd
}

func scanGcpProjectCmd(commonCmdFlags commonFlagsFn, preRun commonPreRunFn, runFn runFn, docs CommandsDocs) *cobra.Command {
cmd := &cobra.Command{
Use: "project PROJECT-ID",
Short: docs.GetShort("gcp-project"),
Long: docs.GetLong("gcp-project"),
Args: cobra.ExactArgs(1),
PreRun: func(cmd *cobra.Command, args []string) {
preRun(cmd, args)
},
Run: func(cmd *cobra.Command, args []string) {
runFn(cmd, args, providers.ProviderType_GCP, GcpProjectAssetType)
},
}
commonCmdFlags(cmd)
return cmd
}

func scanGcpFolderCmd(commonCmdFlags commonFlagsFn, preRun commonPreRunFn, runFn runFn, docs CommandsDocs) *cobra.Command {
cmd := &cobra.Command{
Use: "folder FOLDER-ID",
Short: docs.GetShort("gcp-folder"),
Long: docs.GetLong("gcp-folder"),
Args: cobra.ExactArgs(1),
PreRun: func(cmd *cobra.Command, args []string) {
preRun(cmd, args)
},
Run: func(cmd *cobra.Command, args []string) {
runFn(cmd, args, providers.ProviderType_GCP, GcpFolderAssetType)
},
}
commonCmdFlags(cmd)
return cmd
}

func scanGcpGcrCmd(commonCmdFlags commonFlagsFn, preRun commonPreRunFn, runFn runFn, docs CommandsDocs) *cobra.Command {
cmd := &cobra.Command{
Use: "gcr PROJECT",
Expand Down
57 changes: 36 additions & 21 deletions apps/cnquery/cmd/builder/parse.go
Original file line number Diff line number Diff line change
Expand Up @@ -389,30 +389,45 @@ func ParseTargetAsset(cmd *cobra.Command, args []string, providerType providers.
case providers.ProviderType_GCP:
connection.Backend = providerType

// deprecated, remove in v8
if project, err := cmd.Flags().GetString("project"); err != nil {
log.Fatal().Err(err).Msg("cannot parse --project value")
} else if project != "" {
connection.Options["project-id"] = project
}
switch assetType {
case DefaultAssetType:
// deprecated, remove in v8
if project, err := cmd.Flags().GetString("project"); err != nil {
log.Fatal().Err(err).Msg("cannot parse --project value")
} else if project != "" {
log.Warn().Msg("--project flag is deprecated, use `scan gcp project` instead")
connection.Options["project-id"] = project
}

// deprecated, remove in v8
if organization, err := cmd.Flags().GetString("organization"); err != nil {
log.Fatal().Err(err).Msg("cannot parse --organization value")
} else if organization != "" {
connection.Options["organization-id"] = organization
}
// deprecated, remove in v8
if organization, err := cmd.Flags().GetString("organization"); err != nil {
log.Fatal().Err(err).Msg("cannot parse --organization value")
} else if organization != "" {
log.Warn().Msg("--organization flag is deprecated, use `scan gcp org` instead")
connection.Options["organization-id"] = organization
}

if project, err := cmd.Flags().GetString("project-id"); err != nil {
log.Fatal().Err(err).Msg("cannot parse --project value")
} else if project != "" {
connection.Options["project-id"] = project
}
// deprecated, remove in v9
if project, err := cmd.Flags().GetString("project-id"); err != nil {
log.Fatal().Err(err).Msg("cannot parse --project value")
} else if project != "" {
log.Warn().Msg("--organization-id flag is deprecated, use `scan gcp project` instead")
connection.Options["project-id"] = project
}

if organization, err := cmd.Flags().GetString("organization-id"); err != nil {
log.Fatal().Err(err).Msg("cannot parse --organization value")
} else if organization != "" {
connection.Options["organization-id"] = organization
// deprecated, remove in v9
if organization, err := cmd.Flags().GetString("organization-id"); err != nil {
log.Fatal().Err(err).Msg("cannot parse --organization value")
} else if organization != "" {
log.Warn().Msg("--organization-id flag is deprecated, use `scan gcp org` instead")
connection.Options["organization-id"] = organization
}
case GcpOrganizationAssetType:
connection.Options["organization-id"] = args[0]
case GcpProjectAssetType:
connection.Options["project-id"] = args[0]
case GcpFolderAssetType:
connection.Options["folder-id"] = args[0]
}

case providers.ProviderType_VSPHERE:
Expand Down
11 changes: 10 additions & 1 deletion apps/cnquery/cmd/scan.go
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,16 @@ configuration for the account scan. To scan Azure virtual machines, you must
configure your Azure credentials and have SSH access to the virtual machines.`,
},
"gcp": {
Short: "Scan a Google Cloud Platform (GCP) organization or project.",
Short: "Scan a Google Cloud Platform (GCP) organization, project or folder.",
},
"gcp-org": {
Short: "Scan a Google Cloud Platform (GCP) organization.",
},
"gcp-project": {
Short: "Scan a Google Cloud Platform (GCP) project.",
},
"gcp-folder": {
Short: "Scan a Google Cloud Platform (GCP) folder.",
},
"gcp-gcr": {
Short: "Scan a Google Container Registry (GCR).",
Expand Down
2 changes: 2 additions & 0 deletions motor/discovery/gcp/consts.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ package gcp

const (
// Discovery flags
DiscoveryOrganization = "organization"
DiscoveryFolders = "folders"
DiscoveryInstances = "instances"
DiscoveryProjects = "projects"
DiscoveryComputeImages = "compute-images"
Expand Down
16 changes: 8 additions & 8 deletions motor/discovery/gcp/mql_asset_objects.go
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ func computeInstances(m *MqlDiscovery, project string, tc *providers.Config, sfn
}
}

a := MqlObjectToAsset(project,
a := MqlObjectToAsset(
mqlObject{
name: i.Name, labels: stringLabels,
gcpObject: gcpObject{
Expand Down Expand Up @@ -132,7 +132,7 @@ func computeImages(m *MqlDiscovery, project string, tc *providers.Config) ([]*as
return nil, err
}
for _, i := range images {
assets = append(assets, MqlObjectToAsset(project,
assets = append(assets, MqlObjectToAsset(
mqlObject{
name: i.Name, labels: i.Labels,
gcpObject: gcpObject{
Expand All @@ -159,7 +159,7 @@ func computeNetworks(m *MqlDiscovery, project string, tc *providers.Config) ([]*
return nil, err
}
for _, n := range networks {
assets = append(assets, MqlObjectToAsset(project,
assets = append(assets, MqlObjectToAsset(
mqlObject{
name: n.Name,
gcpObject: gcpObject{
Expand Down Expand Up @@ -189,7 +189,7 @@ func computeSubnetworks(m *MqlDiscovery, project string, tc *providers.Config) (
for _, s := range subnets {
region := gcp.RegionNameFromRegionUrl(s.RegionUrl)

assets = append(assets, MqlObjectToAsset(project,
assets = append(assets, MqlObjectToAsset(
mqlObject{
name: s.Name,
gcpObject: gcpObject{
Expand All @@ -216,7 +216,7 @@ func computeFirewalls(m *MqlDiscovery, project string, tc *providers.Config) ([]
return nil, err
}
for _, f := range firewalls {
assets = append(assets, MqlObjectToAsset(project,
assets = append(assets, MqlObjectToAsset(
mqlObject{
name: f.Name,
gcpObject: gcpObject{
Expand Down Expand Up @@ -245,7 +245,7 @@ func gkeClusters(m *MqlDiscovery, project string, tc *providers.Config) ([]*asse
return nil, err
}
for _, c := range clusters {
assets = append(assets, MqlObjectToAsset(project,
assets = append(assets, MqlObjectToAsset(
mqlObject{
name: c.Name, labels: c.ResourceLabels,
gcpObject: gcpObject{
Expand Down Expand Up @@ -274,7 +274,7 @@ func storageBuckets(m *MqlDiscovery, project string, tc *providers.Config) ([]*a
return nil, err
}
for _, b := range buckets {
assets = append(assets, MqlObjectToAsset(project,
assets = append(assets, MqlObjectToAsset(
mqlObject{
name: b.Name, labels: b.Labels,
gcpObject: gcpObject{
Expand Down Expand Up @@ -302,7 +302,7 @@ func bigQueryDatasets(m *MqlDiscovery, project string, tc *providers.Config) ([]
return nil, err
}
for _, d := range datasets {
assets = append(assets, MqlObjectToAsset(project,
assets = append(assets, MqlObjectToAsset(
mqlObject{
name: d.Id, labels: d.Labels,
gcpObject: gcpObject{
Expand Down
18 changes: 17 additions & 1 deletion motor/discovery/gcp/mql_assets.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,22 @@ func (md *MqlDiscovery) Close() {
}
}

func GetValue[T any](md *MqlDiscovery, query string) (T, error) {
mqlExecutor := mql.New(md.rt, cnquery.DefaultFeatures)
value, err := mqlExecutor.Exec(query, map[string]*llx.Primitive{})
if err != nil {
return *new(T), err
}
if value.Error != nil {
return *new(T), value.Error
}
var out T
if err := mapstructure.Decode(value.Value, &out); err != nil {
return *new(T), err
}
return out, nil
}

func GetList[T any](md *MqlDiscovery, query string) ([]T, error) {
mqlExecutor := mql.New(md.rt, cnquery.DefaultFeatures)
value, err := mqlExecutor.Exec(query, map[string]*llx.Primitive{})
Expand Down Expand Up @@ -158,7 +174,7 @@ func GcpPlatformID(o gcpObject) string {
return "//platformid.api.mondoo.app/runtime/gcp/" + o.service + "/v1/projects/" + o.project + "/regions/" + o.region + "/" + o.objectType + "/" + o.name
}

func MqlObjectToAsset(account string, mqlObject mqlObject, tc *providers.Config) *asset.Asset {
func MqlObjectToAsset(mqlObject mqlObject, tc *providers.Config) *asset.Asset {
if mqlObject.name == "" {
mqlObject.name = mqlObject.gcpObject.id
}
Expand Down
108 changes: 108 additions & 0 deletions motor/discovery/gcp/resolver_folder.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
package gcp

import (
"context"
"fmt"

"github.com/cockroachdb/errors"
"go.mondoo.com/cnquery/motor/asset"
"go.mondoo.com/cnquery/motor/discovery/common"
"go.mondoo.com/cnquery/motor/platform/detector"
"go.mondoo.com/cnquery/motor/providers"
gcp_provider "go.mondoo.com/cnquery/motor/providers/google"
"go.mondoo.com/cnquery/motor/providers/resolver"
"go.mondoo.com/cnquery/motor/vault"
)

var FolderDiscoveryTargets = append(ProjectDiscoveryTargets)

type GcpFolderResolver struct{}

func (k *GcpFolderResolver) Name() string {
return "GCP Folder Resolver"
}

func (r *GcpFolderResolver) AvailableDiscoveryTargets() []string {
return append(FolderDiscoveryTargets, common.DiscoveryAuto, common.DiscoveryAll, DiscoveryFolders)
}

func (r *GcpFolderResolver) Resolve(ctx context.Context, tc *providers.Config, credsResolver vault.Resolver, sfn common.QuerySecretFn, userIdDetectors ...providers.PlatformIdDetector) ([]*asset.Asset, error) {
resolved := []*asset.Asset{}
if tc == nil || tc.Options["folder-id"] == "" {
return resolved, nil
}

// Note: we use the resolver instead of the direct gcp_provider.New to resolve credentials properly
m, err := resolver.NewMotorConnection(ctx, tc, credsResolver)
if err != nil {
return nil, err
}
defer m.Close()
provider, ok := m.Provider.(*gcp_provider.Provider)
if !ok {
return nil, errors.New("could not create gcp provider")
}

identifier, err := provider.Identifier()
if err != nil {
return nil, err
}

// detect platform info for the asset
detector := detector.New(provider)
pf, err := detector.Platform()
if err != nil {
return nil, err
}

folderId := tc.Options["folder-id"]
md, err := NewMQLAssetsDiscovery(provider)
if err != nil {
return nil, err
}

folder, err := GetValue[string](md, fmt.Sprintf("return gcp.folder(id: '%s').name", folderId))
if err != nil {
return nil, err
}

var resolvedRoot *asset.Asset
if tc.IncludesOneOfDiscoveryTarget(DiscoveryFolders) {
pf.Name = "gcp-folder"
resolvedRoot = &asset.Asset{
PlatformIds: []string{identifier},
Name: "GCP folder " + folder,
Platform: pf,
Connections: []*providers.Config{tc}, // pass-in the current config
}
resolved = append(resolved, resolvedRoot)
}

if tc.IncludesOneOfDiscoveryTarget(common.DiscoveryAuto, common.DiscoveryAll,
DiscoveryInstances, DiscoveryComputeImages, DiscoveryComputeNetworks, DiscoveryComputeSubnetworks, DiscoveryComputeFirewalls,
DiscoveryGkeClusters,
DiscoveryStorageBuckets,
DiscoveryBigQueryDatasets) {
type project struct {
Id string
}
projects, err := GetList[project](md, fmt.Sprintf("return gcp.folder(id: '%s').projects { id }", folderId))
if err != nil {
return nil, err
}

for _, p := range projects {
projectConfig := tc.Clone()
projectConfig.Options = map[string]string{
"project-id": p.Id,
}

assets, err := (&GcpProjectResolver{}).Resolve(ctx, projectConfig, credsResolver, sfn, userIdDetectors...)
if err != nil {
return nil, err
}
resolved = append(resolved, assets...)
}
}
return resolved, nil
}
Loading

0 comments on commit 10ca6d6

Please sign in to comment.