feat(gcloud)!: add support to seed data when using RunBigQueryContainer

docs/modules/gcloud.md

@@ -17,7 +17,7 @@ go get github.com/testcontainers/testcontainers-go/modules/gcloud
 ## Usage example
 
 !!!info
-    By default, the all the emulators use `gcr.io/google.com/cloudsdktool/cloud-sdk:367.0.0-emulators` as the default Docker image, except for the BigQuery emulator, which uses `ghcr.io/goccy/bigquery-emulator:0.4.3`, and Spanner, which uses `gcr.io/cloud-spanner-emulator/emulator:1.4.0`.
+    By default, the all the emulators use `gcr.io/google.com/cloudsdktool/cloud-sdk:367.0.0-emulators` as the default Docker image, except for the BigQuery emulator, which uses `ghcr.io/goccy/bigquery-emulator:0.6.1`, and Spanner, which uses `gcr.io/cloud-spanner-emulator/emulator:1.4.0`.
 
 ### BigQuery
 
@@ -28,6 +28,17 @@ go get github.com/testcontainers/testcontainers-go/modules/gcloud
 
 It's important to set the `option.WithEndpoint()` option using the container's URI, as shown in the client example above.
 
+#### Data Yaml (Seed File)
+
+If you would like to do additional initialization in the BigQuery container, add a `data.yaml` file to the container request with the `WithDataYamlFile` function.
+Those files will be copied after the container is created but before it's started. The startup command then used will look like `--project test --data-from-yaml /data.yaml`.
+
+An example of a `data.yaml` file that seeds the BigQuery instance with datasets and tables is shown below:
+
+<!--codeinclude-->
+[Data Yaml content](../../modules/gcloud/testdata/data.yaml)
+<!--/codeinclude-->
+
 ### BigTable
 
 <!--codeinclude-->

modules/gcloud/bigquery.go

@@ -13,7 +13,7 @@ import (
 func RunBigQueryContainer(ctx context.Context, opts ...testcontainers.ContainerCustomizer) (*GCloudContainer, error) {
 	req := testcontainers.GenericContainerRequest{
 		ContainerRequest: testcontainers.ContainerRequest{
-			Image:        "ghcr.io/goccy/bigquery-emulator:0.4.3",
+			Image:        "ghcr.io/goccy/bigquery-emulator:0.6.1",
 			ExposedPorts: []string{"9050/tcp", "9060/tcp"},
 			WaitingFor:   wait.ForHTTP("/discovery/v1/apis/bigquery/v2/rest").WithPort("9050/tcp").WithStartupTimeout(time.Second * 5),
 		},
@@ -27,18 +27,24 @@ func RunBigQueryContainer(ctx context.Context, opts ...testcontainers.ContainerC
 
 	req.Cmd = []string{"--project", settings.ProjectID}
 
+	for _, opt := range opts {
+		if err := opt.Customize(&req); err != nil {
+			return nil, err
+		}
+	}
+
 	container, err := testcontainers.GenericContainer(ctx, req)
 	if err != nil {
 		return nil, err
 	}
 
-	spannerContainer, err := newGCloudContainer(ctx, 9050, container, settings)
+	bigqueryContainer, err := newGCloudContainer(ctx, 9050, container, settings)
 	if err != nil {
 		return nil, err
 	}
 
 	// always prepend http:// to the URI
-	spannerContainer.URI = "http://" + spannerContainer.URI
+	bigqueryContainer.URI = "http://" + bigqueryContainer.URI
 
-	return spannerContainer, nil
+	return bigqueryContainer, nil
 }

modules/gcloud/bigquery_test.go

@@ -5,6 +5,8 @@ import (
 	"errors"
 	"fmt"
 	"log"
+	"path/filepath"
+	"testing"
 
 	"cloud.google.com/go/bigquery"
 	"google.golang.org/api/iterator"
@@ -13,17 +15,19 @@ import (
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/insecure"
 
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 	"github.com/testcontainers/testcontainers-go"
 	"github.com/testcontainers/testcontainers-go/modules/gcloud"
 )
 
-func ExampleRunBigQueryContainer() {
+func TestBigQueryContainer(t *testing.T) {
 	// runBigQueryContainer {
 	ctx := context.Background()
 
 	bigQueryContainer, err := gcloud.RunBigQueryContainer(
 		ctx,
-		testcontainers.WithImage("ghcr.io/goccy/bigquery-emulator:0.4.3"),
+		testcontainers.WithImage("ghcr.io/goccy/bigquery-emulator:0.6.1"),
 		gcloud.WithProjectID("bigquery-project"),
 	)
 	if err != nil {
@@ -78,8 +82,85 @@ func ExampleRunBigQueryContainer() {
 		}
 	}
 
-	fmt.Println(val)
+	// Output:
+	// [30]
+	expectedValue := int64(30)
+	actualValue := val[0]
+	fmt.Println(val[0])
+
+	require.NoError(t, err)
+	if assert.NotNil(t, val) {
+		assert.Equal(t, expectedValue, actualValue)
+	}
+}
+
+func TestBigQueryWithDataYamlFile(t *testing.T) {
+	// runBigQueryContainer {
+	ctx := context.Background()
+
+	absPath, err := filepath.Abs(filepath.Join(".", "testdata", "data.yaml"))
+	if err != nil {
+		log.Fatalf("failed to run container: %v", err)
+	}
+
+	bigQueryContainer, err := gcloud.RunBigQueryContainer(
+		ctx,
+		testcontainers.WithImage("ghcr.io/goccy/bigquery-emulator:0.6.1"),
+		gcloud.WithProjectID("test"),
+		gcloud.WithDataYamlFile(absPath),
+	)
+	if err != nil {
+		log.Fatalf("failed to run container: %v", err)
+	}
+
+	// Clean up the container
+	defer func() {
+		if err := bigQueryContainer.Terminate(ctx); err != nil {
+			log.Fatalf("failed to terminate container: %v", err)
+		}
+	}()
+	// }
+
+	// bigQueryClient {
+	projectID := bigQueryContainer.Settings.ProjectID
+
+	opts := []option.ClientOption{
+		option.WithEndpoint(bigQueryContainer.URI),
+		option.WithGRPCDialOption(grpc.WithTransportCredentials(insecure.NewCredentials())),
+		option.WithoutAuthentication(),
+		internaloption.SkipDialSettingsValidation(),
+	}
+
+	client, err := bigquery.NewClient(ctx, projectID, opts...)
+	if err != nil {
+		log.Fatalf("failed to create bigquery client: %v", err) // nolint:gocritic
+	}
+	defer client.Close()
+	// }
+
+	selectQuery := client.Query("SELECT * FROM dataset1.table_a where name = @name")
+	selectQuery.QueryConfig.Parameters = []bigquery.QueryParameter{
+		{Name: "name", Value: "bob"},
+	}
+	it, err := selectQuery.Read(ctx)
+	if err != nil {
+		log.Fatalf("failed to read query: %v", err)
+	}
+
+	var val []bigquery.Value
+	for {
+		err := it.Next(&val)
+		if errors.Is(err, iterator.Done) {
+			break
+		}
+		if err != nil {
+			log.Fatalf("failed to iterate: %v", err)
+		}
+	}
 
 	// Output:
 	// [30]
+	expectedValue := int64(30)
+	actualValue := val[0]
+	assert.Equal(t, expectedValue, actualValue)
 }

modules/gcloud/gcloud.go

@@ -41,12 +41,14 @@ func newGCloudContainer(ctx context.Context, port int, c testcontainers.Containe
 }
 
 type options struct {
-	ProjectID string
+	ProjectID    string
+	DataYamlFile string
 }
 
 func defaultOptions() options {
 	return options{
-		ProjectID: defaultProjectID,
+		ProjectID:    defaultProjectID,
+		DataYamlFile: "/data.yaml",
 	}
 }
 
@@ -69,6 +71,22 @@ func WithProjectID(projectID string) Option {
 	}
 }
 
+// WithDataYamlFile seeds the Bigquery project for the GCloud container.
+func WithDataYamlFile(dataYamlFile string) testcontainers.CustomizeRequestOption {
+	return func(req *testcontainers.GenericContainerRequest) error {
+		dataFile := testcontainers.ContainerFile{
+			HostFilePath:      dataYamlFile,
+			ContainerFilePath: "/data.yaml",
+			FileMode:          0o755,
+		}
+
+		req.Files = append(req.Files, dataFile)
+		req.Cmd = append(req.Cmd, "--data-from-yaml", "/data.yaml")
+
+		return nil
+	}
+}
+
 // applyOptions applies the options to the container request and returns the settings.
 func applyOptions(req *testcontainers.GenericContainerRequest, opts []testcontainers.ContainerCustomizer) (options, error) {
 	settings := defaultOptions()

modules/gcloud/go.mod

@@ -10,6 +10,7 @@ require (
 	cloud.google.com/go/pubsub v1.36.2
 	cloud.google.com/go/spanner v1.57.0
 	github.com/docker/go-connections v0.5.0
+	github.com/stretchr/testify v1.9.0
 	github.com/testcontainers/testcontainers-go v0.30.0
 	google.golang.org/api v0.168.0
 	google.golang.org/grpc v1.62.0
@@ -34,6 +35,7 @@ require (
 	github.com/containerd/containerd v1.7.15 // indirect
 	github.com/containerd/log v0.1.0 // indirect
 	github.com/cpuguy83/dockercfg v0.3.1 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/distribution/reference v0.5.0 // indirect
 	github.com/docker/docker v25.0.5+incompatible // indirect
 	github.com/docker/go-units v0.5.0 // indirect
@@ -68,6 +70,7 @@ require (
 	github.com/opencontainers/image-spec v1.1.0 // indirect
 	github.com/pierrec/lz4/v4 v4.1.18 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
 	github.com/shirou/gopsutil/v3 v3.23.12 // indirect
 	github.com/shoenig/go-m1cpu v0.1.6 // indirect
@@ -97,6 +100,7 @@ require (
 	google.golang.org/genproto/googleapis/api v0.0.0-20240221002015-b0ce06bbee7c // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240304161311-37d4d3c04a78 // indirect
 	google.golang.org/protobuf v1.33.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
 
 replace github.com/testcontainers/testcontainers-go => ../..

modules/gcloud/go.sum

@@ -149,6 +149,10 @@ github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGC
 github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
 github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg=
 github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4=
 github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
 github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY=
@@ -181,6 +185,8 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
 github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw=
 github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
+github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
 github.com/shirou/gopsutil/v3 v3.23.12 h1:z90NtUkp3bMtmICZKpC4+WaknU1eXtp5vtbQ11DgpE4=
 github.com/shirou/gopsutil/v3 v3.23.12/go.mod h1:1FrWgea594Jp7qmjHUUPlJDTPgcsb9mGnXDxavtikzM=
 github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM=
@@ -191,8 +197,9 @@ github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ
 github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
-github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
+github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
@@ -362,6 +369,8 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ
 google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
 google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

modules/gcloud/testdata/data.yaml

@@ -0,0 +1,20 @@
+projects:
+  - id: test
+    datasets:
+      - id: dataset1
+        tables:
+          - id: table_a
+            columns:
+              - name: id
+                type: INTEGER
+              - name: name
+                type: STRING
+              - name: createdAt
+                type: TIMESTAMP
+            data:
+              - id: 1
+                name: alice
+                createdAt: "2022-10-21T00:00:00"
+              - id: 30
+                name: bob
+                createdAt: "2022-10-21T00:00:00"