feat: adding support for bq (#1878)

Co-authored-by: Andrew Peabody <[email protected]>
Co-authored-by: Awais Malik <[email protected]>

infra/blueprint-test/pkg/bq/bq.go

@@ -0,0 +1,130 @@
+/**
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Package bq provides a set of helpers to interact with bq tool (part of CloudSDK)
+package bq
+
+import (
+	"strings"
+
+	"github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/utils"
+	"github.com/gruntwork-io/terratest/modules/logger"
+	"github.com/gruntwork-io/terratest/modules/shell"
+	"github.com/mitchellh/go-testing-interface"
+	"github.com/tidwall/gjson"
+)
+
+type CmdCfg struct {
+	bqBinary   string         // path to bq binary
+	commonArgs []string       // common arguments to pass to bq calls
+	logger     *logger.Logger // custom logger
+}
+
+type cmdOption func(*CmdCfg)
+
+func WithBinary(bqBinary string) cmdOption {
+	return func(f *CmdCfg) {
+		f.bqBinary = bqBinary
+	}
+}
+
+func WithCommonArgs(commonArgs []string) cmdOption {
+	return func(f *CmdCfg) {
+		f.commonArgs = commonArgs
+	}
+}
+
+func WithLogger(logger *logger.Logger) cmdOption {
+	return func(f *CmdCfg) {
+		f.logger = logger
+	}
+}
+
+// newCmdConfig sets defaults and validates values for bq Options.
+func newCmdConfig(opts ...cmdOption) (*CmdCfg, error) {
+	gOpts := &CmdCfg{}
+	// apply options
+	for _, opt := range opts {
+		opt(gOpts)
+	}
+	if gOpts.bqBinary == "" {
+		err := utils.BinaryInPath("bq")
+		if err != nil {
+			return nil, err
+		}
+		gOpts.bqBinary = "bq"
+	}
+	if gOpts.commonArgs == nil {
+		gOpts.commonArgs = []string{"--format", "json"}
+	}
+	if gOpts.logger == nil {
+		gOpts.logger = utils.GetLoggerFromT()
+	}
+	return gOpts, nil
+}
+
+// RunCmd executes a bq command and fails test if there are any errors.
+func RunCmd(t testing.TB, cmd string, opts ...cmdOption) string {
+	op, err := RunCmdE(t, cmd, opts...)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return op
+}
+
+// RunCmdE executes a bq command and return output.
+func RunCmdE(t testing.TB, cmd string, opts ...cmdOption) (string, error) {
+	gOpts, err := newCmdConfig(opts...)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// split command into args
+	args := strings.Fields(cmd)
+	bqCmd := shell.Command{
+		Command: "bq",
+		Args:    append(args, gOpts.commonArgs...),
+		Logger:  gOpts.logger,
+	}
+	return shell.RunCommandAndGetStdOutE(t, bqCmd)
+}
+
+// Run executes a bq command and returns value as gjson.Result.
+// It fails the test if there are any errors executing the bq command or parsing the output value.
+func Run(t testing.TB, cmd string, opts ...cmdOption) gjson.Result {
+	op := RunCmd(t, cmd, opts...)
+	if !gjson.Valid(op) {
+		t.Fatalf("Error parsing output, invalid json: %s", op)
+	}
+	return gjson.Parse(op)
+}
+
+// RunWithCmdOptsf executes a bq command and returns value as gjson.Result.
+//
+// RunWithCmdOptsf(t, ops.., "ls --datasets --project_id=%s", "projectId")
+//
+// It fails the test if there are any errors executing the bq command or parsing the output value.
+func RunWithCmdOptsf(t testing.TB, opts []cmdOption, cmd string, args ...interface{}) gjson.Result {
+	return Run(t, utils.StringFromTextAndArgs(append([]interface{}{cmd}, args...)...), opts...)
+}
+
+// Runf executes a bq command and returns value as gjson.Result.
+//
+// Runf(t, "ls --datasets --project_id=%s", "projectId")
+//
+// It fails the test if there are any errors executing the bq command or parsing the output value.
+func Runf(t testing.TB, cmd string, args ...interface{}) gjson.Result {
+	return Run(t, utils.StringFromTextAndArgs(append([]interface{}{cmd}, args...)...))
+}

infra/blueprint-test/pkg/bq/bq_test.go

@@ -0,0 +1,51 @@
+/**
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Package bq provides a set of helpers to interact with bq tool (part of CloudSDK)
+package bq
+
+import (
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestRunf(t *testing.T) {
+	tests := []struct {
+		name            string
+		cmd             string
+		projectIdEnvVar string
+	}{
+		{
+			name:            "Runf",
+			cmd:             "ls --datasets --project_id=%s",
+			projectIdEnvVar: "bigquery-public-data",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if projectName, present := os.LookupEnv(tt.projectIdEnvVar); present {
+				op := Runf(t, tt.cmd, projectName)
+				assert := assert.New(t)
+				assert.Equal("bigquery#dataset", op.Array()[0].Get("kind").String())
+			} else {
+				t.Logf("Skipping test, %s envvar not set", tt.projectIdEnvVar)
+				t.Skip()
+			}
+		})
+	}
+}

infra/blueprint-test/pkg/gcloud/gcloud.go

@@ -130,7 +130,7 @@ func TFVet(t testing.TB, planFilePath string, policyLibraryPath, terraformVetPro
 //
 // It fails the test if there are any errors executing the gcloud command or parsing the output value.
 func RunWithCmdOptsf(t testing.TB, opts []cmdOption, cmd string, args ...interface{}) gjson.Result {
-	return Run(t, stringFromTextAndArgs(append([]interface{}{cmd}, args...)...), opts...)
+	return Run(t, utils.StringFromTextAndArgs(append([]interface{}{cmd}, args...)...), opts...)
 }
 
 // Runf executes a gcloud command and returns value as gjson.Result.
@@ -139,25 +139,7 @@ func RunWithCmdOptsf(t testing.TB, opts []cmdOption, cmd string, args ...interfa
 //
 // It fails the test if there are any errors executing the gcloud command or parsing the output value.
 func Runf(t testing.TB, cmd string, args ...interface{}) gjson.Result {
-	return Run(t, stringFromTextAndArgs(append([]interface{}{cmd}, args...)...))
-}
-
-// stringFromTextAndArgs convert msg and args to formatted text
-func stringFromTextAndArgs(msgAndArgs ...interface{}) string {
-	if len(msgAndArgs) == 0 || msgAndArgs == nil {
-		return ""
-	}
-	if len(msgAndArgs) == 1 {
-		msg := msgAndArgs[0]
-		if msgAsStr, ok := msg.(string); ok {
-			return msgAsStr
-		}
-		return fmt.Sprintf("%+v", msg)
-	}
-	if len(msgAndArgs) > 1 {
-		return fmt.Sprintf(msgAndArgs[0].(string), msgAndArgs[1:]...)
-	}
-	return ""
+	return Run(t, utils.StringFromTextAndArgs(append([]interface{}{cmd}, args...)...))
 }
 
 // ActivateCredsAndEnvVars activates credentials and exports auth related envvars.

infra/blueprint-test/pkg/utils/string_formatter.go

@@ -0,0 +1,37 @@
+/**
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package utils
+
+import "fmt"
+
+// StringFromTextAndArgs converts msg and args to formatted text
+func StringFromTextAndArgs(msgAndArgs ...interface{}) string {
+	if len(msgAndArgs) == 0 || msgAndArgs == nil {
+		return ""
+	}
+	if len(msgAndArgs) == 1 {
+		msg := msgAndArgs[0]
+		if msgAsStr, ok := msg.(string); ok {
+			return msgAsStr
+		}
+		return fmt.Sprintf("%+v", msg)
+	}
+	if len(msgAndArgs) > 1 {
+		return fmt.Sprintf(msgAndArgs[0].(string), msgAndArgs[1:]...)
+	}
+	return ""
+}

infra/blueprint-test/pkg/utils/string_formatter_test.go

@@ -0,0 +1,46 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package utils
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestStringFromTextAndArgs(t *testing.T) {
+	tests := []struct {
+		name   string
+		cmd    string
+		args   []interface{}
+		output string
+	}{
+		{
+			name:   "one arg",
+			cmd:    "project list --filter=%s",
+			args:   []interface{}{"TEST_PROJECT"},
+			output: "project list --filter=TEST_PROJECT",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			assert := assert.New(t)
+			funcOut := StringFromTextAndArgs(append([]interface{}{tt.cmd}, tt.args...)...)
+			assert.Equal(tt.output, funcOut)
+		})
+	}
+}