Fix paging for listing resources for all tables. Fixes #250 (#254)

.github/workflows/golangci-lint.yml

@@ -14,20 +14,9 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v2
 
-      - name: Hook private repo
-        run: git config --global url."https://${{ secrets.GH_ACCESS_TOKEN }}:[email protected]".insteadOf "https://github.com"
-
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v2
         with:
           # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
           version: v1.29
-
-          # Optional: working directory, useful for monorepos
-          # working-directory: somedir
-
-          # Optional: golangci-lint command line arguments.
-          # args: --issues-exit-code=0
-
-          # Optional: show only new issues if it's a pull request. The default value is `false`.
-          # only-new-issues: true
+          args: --timeout=5m

azure/service.go

@@ -33,31 +33,29 @@ type Session struct {
 func GetNewSession(ctx context.Context, d *plugin.QueryData, tokenAudience string) (session *Session, err error) {
 	azureConfig := GetConfig(d.Connection)
 
-	if &azureConfig != nil {
-		if azureConfig.TenantID != nil {
-			os.Setenv("AZURE_TENANT_ID", *azureConfig.TenantID)
-		}
-		if azureConfig.SubscriptionID != nil {
-			os.Setenv("AZURE_SUBSCRIPTION_ID", *azureConfig.SubscriptionID)
-		}
-		if azureConfig.ClientID != nil {
-			os.Setenv("AZURE_CLIENT_ID", *azureConfig.ClientID)
-		}
-		if azureConfig.ClientSecret != nil {
-			os.Setenv("AZURE_CLIENT_SECRET", *azureConfig.ClientSecret)
-		}
-		if azureConfig.CertificatePath != nil {
-			os.Setenv("AZURE_CERTIFICATE_PATH", *azureConfig.CertificatePath)
-		}
-		if azureConfig.CertificatePassword != nil {
-			os.Setenv("AZURE_CERTIFICATE_PASSWORD", *azureConfig.CertificatePassword)
-		}
-		if azureConfig.Username != nil {
-			os.Setenv("AZURE_USERNAME", *azureConfig.Username)
-		}
-		if azureConfig.Username != nil {
-			os.Setenv("AZURE_PASSWORD", *azureConfig.Password)
-		}
+	if azureConfig.TenantID != nil {
+		os.Setenv("AZURE_TENANT_ID", *azureConfig.TenantID)
+	}
+	if azureConfig.SubscriptionID != nil {
+		os.Setenv("AZURE_SUBSCRIPTION_ID", *azureConfig.SubscriptionID)
+	}
+	if azureConfig.ClientID != nil {
+		os.Setenv("AZURE_CLIENT_ID", *azureConfig.ClientID)
+	}
+	if azureConfig.ClientSecret != nil {
+		os.Setenv("AZURE_CLIENT_SECRET", *azureConfig.ClientSecret)
+	}
+	if azureConfig.CertificatePath != nil {
+		os.Setenv("AZURE_CERTIFICATE_PATH", *azureConfig.CertificatePath)
+	}
+	if azureConfig.CertificatePassword != nil {
+		os.Setenv("AZURE_CERTIFICATE_PASSWORD", *azureConfig.CertificatePassword)
+	}
+	if azureConfig.Username != nil {
+		os.Setenv("AZURE_USERNAME", *azureConfig.Username)
+	}
+	if azureConfig.Username != nil {
+		os.Setenv("AZURE_PASSWORD", *azureConfig.Password)
 	}
 
 	logger := plugin.Logger(ctx)

azure/table_azure_ad_group.go

@@ -103,18 +103,23 @@ func listAdGroups(ctx context.Context, d *plugin.QueryData, _ *plugin.HydrateDat
 	graphClient := graphrbac.NewGroupsClient(tenantID)
 	graphClient.Authorizer = session.Authorizer
 
-	pagesLeft := true
-	for pagesLeft {
-		result, err := graphClient.List(ctx, "")
+	result, err := graphClient.List(ctx, "")
+	if err != nil {
+		return nil, err
+	}
+	for _, group := range result.Values() {
+		d.StreamListItem(ctx, group)
+	}
+
+	for result.NotDone() {
+		err = result.NextWithContext(ctx)
 		if err != nil {
 			return nil, err
 		}
 
 		for _, group := range result.Values() {
 			d.StreamListItem(ctx, group)
 		}
-		result.NextWithContext(context.Background())
-		pagesLeft = result.NotDone()
 	}
 
 	return nil, err

azure/table_azure_ad_service_principal.go

@@ -154,11 +154,16 @@ func listAdServicePrincipals(ctx context.Context, d *plugin.QueryData, _ *plugin
 	if err != nil {
 		return nil, err
 	}
+	for _, servicePrincipal := range result.Values() {
+		d.StreamListItem(ctx, servicePrincipal)
+	}
 
-	for result.Response().OdataNextLink != nil && *result.Response().OdataNextLink != "" {
-		if err := result.NextWithContext(ctx); err != nil {
+	for result.NotDone() {
+		err = result.NextWithContext(ctx)
+		if err != nil {
 			return nil, err
 		}
+
 		for _, servicePrincipal := range result.Values() {
 			d.StreamListItem(ctx, servicePrincipal)
 		}

azure/table_azure_ad_user.go

@@ -135,18 +135,22 @@ func listAdUsers(ctx context.Context, d *plugin.QueryData, _ *plugin.HydrateData
 	graphClient := graphrbac.NewUsersClient(tenantID)
 	graphClient.Authorizer = session.Authorizer
 
-	pagesLeft := true
-	for pagesLeft {
-		result, err := graphClient.List(ctx, "", "")
+	result, err := graphClient.List(ctx, "", "")
+	if err != nil {
+		return nil, err
+	}
+	for _, user := range result.Values() {
+		d.StreamListItem(ctx, user)
+	}
+
+	for result.NotDone() {
+		err = result.NextWithContext(ctx)
 		if err != nil {
 			return nil, err
 		}
-
 		for _, user := range result.Values() {
 			d.StreamListItem(ctx, user)
 		}
-		result.NextWithContext(context.Background())
-		pagesLeft = result.NotDone()
 	}
 
 	return nil, err

azure/table_azure_api_management.go

@@ -231,18 +231,23 @@ func listAPIManagements(ctx context.Context, d *plugin.QueryData, _ *plugin.Hydr
 	apiManagementClient := apimanagement.NewServiceClient(subscriptionID)
 	apiManagementClient.Authorizer = session.Authorizer
 
-	pagesLeft := true
-	for pagesLeft {
-		result, err := apiManagementClient.List(ctx)
+	result, err := apiManagementClient.List(ctx)
+	if err != nil {
+		return nil, err
+	}
+	for _, apiManagement := range result.Values() {
+		d.StreamListItem(ctx, apiManagement)
+	}
+
+	for result.NotDone() {
+		err = result.NextWithContext(ctx)
 		if err != nil {
 			return nil, err
 		}
 
 		for _, apiManagement := range result.Values() {
 			d.StreamListItem(ctx, apiManagement)
 		}
-		result.NextWithContext(context.Background())
-		pagesLeft = result.NotDone()
 	}
 
 	return nil, err

azure/table_azure_app_service_environment.go

@@ -175,18 +175,24 @@ func listAppServiceEnvironments(ctx context.Context, d *plugin.QueryData, _ *plu
 	webClient := web.NewAppServiceEnvironmentsClient(subscriptionID)
 	webClient.Authorizer = session.Authorizer
 
-	pagesLeft := true
-	for pagesLeft {
-		result, err := webClient.List(ctx)
+	result, err := webClient.List(ctx)
+	if err != nil {
+		return nil, err
+	}
+	for _, environment := range result.Values() {
+		d.StreamListItem(ctx, environment)
+	}
+
+	for result.NotDone() {
+		err = result.NextWithContext(ctx)
 		if err != nil {
 			return nil, err
 		}
 
 		for _, environment := range result.Values() {
 			d.StreamListItem(ctx, environment)
 		}
-		result.NextWithContext(context.Background())
-		pagesLeft = result.NotDone()
+
 	}
 	return nil, err
 }

azure/table_azure_app_service_function_app.go

@@ -188,9 +188,19 @@ func listAppServiceFunctionApps(ctx context.Context, d *plugin.QueryData, _ *plu
 	webClient := web.NewAppsClient(subscriptionID)
 	webClient.Authorizer = session.Authorizer
 
-	pagesLeft := true
-	for pagesLeft {
-		result, err := webClient.List(ctx)
+	result, err := webClient.List(ctx)
+	if err != nil {
+		return nil, err
+	}
+	for _, functionApp := range result.Values() {
+		// Filtering out all the web apps
+		if strings.Contains(string(*functionApp.Kind), "functionapp") {
+			d.StreamListItem(ctx, functionApp)
+		}
+	}
+
+	for result.NotDone() {
+		err := result.NextWithContext(ctx)
 		if err != nil {
 			return nil, err
 		}
@@ -201,8 +211,7 @@ func listAppServiceFunctionApps(ctx context.Context, d *plugin.QueryData, _ *plu
 				d.StreamListItem(ctx, functionApp)
 			}
 		}
-		result.NextWithContext(context.Background())
-		pagesLeft = result.NotDone()
+
 	}
 	return nil, err
 }

azure/table_azure_app_service_plan.go

@@ -4,6 +4,7 @@ import (
 	"context"
 
 	"github.com/Azure/azure-sdk-for-go/services/web/mgmt/2020-06-01/web"
+	"github.com/turbot/go-kit/types"
 	"github.com/turbot/steampipe-plugin-sdk/grpc/proto"
 	"github.com/turbot/steampipe-plugin-sdk/plugin/transform"
 
@@ -188,19 +189,23 @@ func listAppServicePlans(ctx context.Context, d *plugin.QueryData, _ *plugin.Hyd
 	webClient := web.NewAppServicePlansClient(subscriptionID)
 	webClient.Authorizer = session.Authorizer
 
-	pagesLeft := true
-	for pagesLeft {
-		param := true
-		result, err := webClient.List(ctx, &param)
+	result, err := webClient.List(ctx, types.Bool(true))
+	if err != nil {
+		return nil, err
+	}
+	for _, servicePlan := range result.Values() {
+		d.StreamListItem(ctx, servicePlan)
+	}
+
+	for result.NotDone() {
+		err = result.NextWithContext(ctx)
 		if err != nil {
 			return nil, err
 		}
 
 		for _, servicePlan := range result.Values() {
 			d.StreamListItem(ctx, servicePlan)
 		}
-		result.NextWithContext(context.Background())
-		pagesLeft = result.NotDone()
 	}
 	return nil, err
 }

azure/table_azure_app_service_web_app.go

@@ -4,6 +4,7 @@ import (
 	"context"
 
 	"github.com/Azure/azure-sdk-for-go/services/web/mgmt/2020-06-01/web"
+	"github.com/turbot/go-kit/types"
 	"github.com/turbot/steampipe-plugin-sdk/grpc/proto"
 	"github.com/turbot/steampipe-plugin-sdk/plugin/transform"
 
@@ -206,9 +207,19 @@ func listAppServiceWebApps(ctx context.Context, d *plugin.QueryData, _ *plugin.H
 	webClient := web.NewAppsClient(subscriptionID)
 	webClient.Authorizer = session.Authorizer
 
-	pagesLeft := true
-	for pagesLeft {
-		result, err := webClient.List(ctx)
+	result, err := webClient.List(ctx)
+	if err != nil {
+		return nil, err
+	}
+	for _, webApp := range result.Values() {
+		// Filtering out all the function apps
+		if string(*webApp.Kind) != "functionapp" {
+			d.StreamListItem(ctx, webApp)
+		}
+	}
+
+	for result.NotDone() {
+		err = result.NextWithContext(ctx)
 		if err != nil {
 			return nil, err
 		}
@@ -219,8 +230,6 @@ func listAppServiceWebApps(ctx context.Context, d *plugin.QueryData, _ *plugin.H
 				d.StreamListItem(ctx, webApp)
 			}
 		}
-		result.NextWithContext(context.Background())
-		pagesLeft = result.NotDone()
 	}
 	return nil, err
 }
@@ -356,7 +365,7 @@ func webAppIdentity(ctx context.Context, d *transform.TransformData) (interface{
 	data := d.HydrateItem.(web.Site)
 	objectMap := make(map[string]interface{})
 	if data.Identity != nil {
-		if &data.Identity.Type != nil {
+		if types.SafeString(data.Identity.Type) != "" {
 			objectMap["Type"] = data.Identity.Type
 		}
 		if data.Identity.TenantID != nil {

azure/table_azure_application_security_group.go

@@ -111,18 +111,24 @@ func listApplicationSecurityGroups(ctx context.Context, d *plugin.QueryData, _ *
 	applicationSecurityGroupClient := network.NewApplicationSecurityGroupsClient(subscriptionID)
 	applicationSecurityGroupClient.Authorizer = session.Authorizer
 
-	pagesLeft := true
-	for pagesLeft {
-		result, err := applicationSecurityGroupClient.ListAll(ctx)
+	result, err := applicationSecurityGroupClient.ListAll(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, applicationSecurityGroup := range result.Values() {
+		d.StreamListItem(ctx, applicationSecurityGroup)
+	}
+
+	for result.NotDone() {
+		err = result.NextWithContext(ctx)
 		if err != nil {
 			return nil, err
 		}
 
 		for _, applicationSecurityGroup := range result.Values() {
 			d.StreamListItem(ctx, applicationSecurityGroup)
 		}
-		result.NextWithContext(context.Background())
-		pagesLeft = result.NotDone()
 	}
 	return nil, err
 }