chore: bump bigquery to v1.61.0 (#78)

# Description

bump bigquery to 1.61.0

## Security

- [x] The code changed/added as part of this pull request won't create
any security issues with how the software is being used.

go.mod

@@ -6,21 +6,20 @@ toolchain go1.22.2
 
 require (
 	cloud.google.com/go v0.113.0
-	cloud.google.com/go/bigquery v1.60.0
+	cloud.google.com/go/bigquery v1.61.0
 	github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5
 	github.com/aws/aws-sdk-go-v2 v1.27.0
 	github.com/aws/aws-sdk-go-v2/config v1.27.11
 	github.com/aws/aws-sdk-go-v2/credentials v1.17.15
 	github.com/aws/aws-sdk-go-v2/service/redshiftdata v1.25.7
-	github.com/cenkalti/backoff/v4 v4.3.0
 	github.com/databricks/databricks-sql-go v1.5.5
 	github.com/dlclark/regexp2 v1.11.0
 	github.com/gliderlabs/ssh v0.3.7
 	github.com/go-sql-driver/mysql v1.8.1
 	github.com/google/uuid v1.6.0
 	github.com/lib/pq v1.10.9
 	github.com/ory/dockertest/v3 v3.10.0
-	github.com/rudderlabs/rudder-go-kit v0.30.0
+	github.com/rudderlabs/rudder-go-kit v0.31.0
 	github.com/rudderlabs/sql-tunnels v0.1.6
 	github.com/samber/lo v1.39.0
 	github.com/sirupsen/logrus v1.9.3
@@ -30,7 +29,6 @@ require (
 	github.com/tidwall/sjson v1.2.5
 	github.com/trinodb/trino-go-client v0.315.0
 	golang.org/x/crypto v0.23.0
-	golang.org/x/sync v0.7.0
 	google.golang.org/api v0.181.0
 )
 
@@ -51,7 +49,6 @@ require (
 	github.com/Microsoft/go-winio v0.6.1 // indirect
 	github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect
 	github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect
-	github.com/apache/arrow/go/v14 v14.0.2 // indirect
 	github.com/apache/arrow/go/v15 v15.0.2 // indirect
 	github.com/apache/arrow/go/v16 v16.0.0 // indirect
 	github.com/apache/thrift v0.19.0 // indirect
@@ -71,6 +68,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.24.2 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sts v1.28.9 // indirect
 	github.com/aws/smithy-go v1.20.2 // indirect
+	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
 	github.com/containerd/continuity v0.4.3 // indirect
 	github.com/coreos/go-oidc/v3 v3.5.0 // indirect
 	github.com/danieljoos/wincred v1.1.2 // indirect
@@ -131,13 +129,14 @@ require (
 	go.opencensus.io v0.24.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
-	go.opentelemetry.io/otel v1.25.0 // indirect
-	go.opentelemetry.io/otel/metric v1.25.0 // indirect
-	go.opentelemetry.io/otel/trace v1.25.0 // indirect
+	go.opentelemetry.io/otel v1.26.0 // indirect
+	go.opentelemetry.io/otel/metric v1.26.0 // indirect
+	go.opentelemetry.io/otel/trace v1.26.0 // indirect
 	golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 // indirect
 	golang.org/x/mod v0.17.0 // indirect
 	golang.org/x/net v0.25.0 // indirect
 	golang.org/x/oauth2 v0.20.0 // indirect
+	golang.org/x/sync v0.7.0 // indirect
 	golang.org/x/sys v0.20.0 // indirect
 	golang.org/x/term v0.20.0 // indirect
 	golang.org/x/text v0.15.0 // indirect

go.sum

@@ -5,8 +5,8 @@ cloud.google.com/go/auth v0.4.1 h1:Z7YNIhlWRtrnKlZke7z3GMqzvuYzdc2z98F9D1NV5Hg=
 cloud.google.com/go/auth v0.4.1/go.mod h1:QVBuVEKpCn4Zp58hzRGvL0tjRGU0YqdRTdCHM1IHnro=
 cloud.google.com/go/auth/oauth2adapt v0.2.2 h1:+TTV8aXpjeChS9M+aTtN/TjdQnzJvmzKFt//oWu7HX4=
 cloud.google.com/go/auth/oauth2adapt v0.2.2/go.mod h1:wcYjgpZI9+Yu7LyYBg4pqSiaRkfEK3GQcpb7C/uyF1Q=
-cloud.google.com/go/bigquery v1.60.0 h1:kA96WfgvCbkqfLnr7xI5uEfJ4h4FrnkdEb0yty0KSZo=
-cloud.google.com/go/bigquery v1.60.0/go.mod h1:Clwk2OeC0ZU5G5LDg7mo+h8U7KlAa5v06z5rptKdM3g=
+cloud.google.com/go/bigquery v1.61.0 h1:w2Goy9n6gh91LVi6B2Sc+HpBl8WbWhIyzdvVvrAuEIw=
+cloud.google.com/go/bigquery v1.61.0/go.mod h1:PjZUje0IocbuTOdq4DBOJLNYB0WF3pAKBHzAYyxCwFo=
 cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k=
 cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc=
 cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k=
@@ -51,8 +51,6 @@ github.com/ahmetb/dlog v0.0.0-20170105205344-4fb5f8204f26 h1:3YVZUqkoev4mL+aCwVO
 github.com/ahmetb/dlog v0.0.0-20170105205344-4fb5f8204f26/go.mod h1:ymXt5bw5uSNu4jveerFxE0vNYxF8ncqbptntMaFMg3k=
 github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8=
 github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4=
-github.com/apache/arrow/go/v14 v14.0.2 h1:N8OkaJEOfI3mEZt07BIkvo4sC6XDbL+48MBPWO5IONw=
-github.com/apache/arrow/go/v14 v14.0.2/go.mod h1:u3fgh3EdgN/YQ8cVQRguVW3R+seMybFg8QBQ5LU+eBY=
 github.com/apache/arrow/go/v15 v15.0.2 h1:60IliRbiyTWCWjERBCkO1W4Qun9svcYoZrSLcyOsMLE=
 github.com/apache/arrow/go/v15 v15.0.2/go.mod h1:DGXsR3ajT524njufqf95822i+KTh+yea1jass9YXgjA=
 github.com/apache/arrow/go/v16 v16.0.0 h1:qRLbJRPj4zaseZrjbDHa7mUoZDDIU+4pu+mE2Lucs5g=
@@ -295,8 +293,8 @@ github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/f
 github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
 github.com/rs/zerolog v1.28.0 h1:MirSo27VyNi7RJYP3078AA1+Cyzd2GB66qy3aUHvsWY=
 github.com/rs/zerolog v1.28.0/go.mod h1:NILgTygv/Uej1ra5XxGf82ZFSLk58MFGAUS2o6usyD0=
-github.com/rudderlabs/rudder-go-kit v0.30.0 h1:JV5NlnvAboh//2mo2sV3PDXuHPX5Y6CtjZXibMc1wWM=
-github.com/rudderlabs/rudder-go-kit v0.30.0/go.mod h1:Io1s43ApHrIMUUrz0Z2j+zb9p8r49NKT5YUzMIh62Q8=
+github.com/rudderlabs/rudder-go-kit v0.31.0 h1:7bHJMogiMvkrlGAIy/FrDfR8eUIuJBfGbZcnolkmvW4=
+github.com/rudderlabs/rudder-go-kit v0.31.0/go.mod h1:EJSKYDnbUjJzBP8ZhniW6GkfxfK2nBTaxiL5KBYvo08=
 github.com/rudderlabs/sql-tunnels v0.1.6 h1:v2KA2cq8ZV5LXRJQpqigq1Q4V64oDL+XlfckW/0K2/4=
 github.com/rudderlabs/sql-tunnels v0.1.6/go.mod h1:Jew/XwojzFTK4KTDj/wvChI7iZCgAKYyKjFK1nnHlNM=
 github.com/samber/lo v1.39.0 h1:4gTz1wUhNYLhFSKl6O+8peW0v2F4BCY034GRpU9WnuA=
@@ -353,14 +351,14 @@ go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.4
 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0/go.mod h1:Mjt1i1INqiaoZOMGR1RIUJN+i3ChKoFRqzrRQhlkbs0=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw=
-go.opentelemetry.io/otel v1.25.0 h1:gldB5FfhRl7OJQbUHt/8s0a7cE8fbsPAtdpRaApKy4k=
-go.opentelemetry.io/otel v1.25.0/go.mod h1:Wa2ds5NOXEMkCmUou1WA7ZBfLTHWIsp034OVD7AO+Vg=
-go.opentelemetry.io/otel/metric v1.25.0 h1:LUKbS7ArpFL/I2jJHdJcqMGxkRdxpPHE0VU/D4NuEwA=
-go.opentelemetry.io/otel/metric v1.25.0/go.mod h1:rkDLUSd2lC5lq2dFNrX9LGAbINP5B7WBkC78RXCpH5s=
-go.opentelemetry.io/otel/sdk v1.25.0 h1:PDryEJPC8YJZQSyLY5eqLeafHtG+X7FWnf3aXMtxbqo=
-go.opentelemetry.io/otel/sdk v1.25.0/go.mod h1:oFgzCM2zdsxKzz6zwpTZYLLQsFwc+K0daArPdIhuxkw=
-go.opentelemetry.io/otel/trace v1.25.0 h1:tqukZGLwQYRIFtSQM2u2+yfMVTgGVeqRLPUYx1Dq6RM=
-go.opentelemetry.io/otel/trace v1.25.0/go.mod h1:hCCs70XM/ljO+BeQkyFnbK28SBIJ/Emuha+ccrCRT7I=
+go.opentelemetry.io/otel v1.26.0 h1:LQwgL5s/1W7YiiRwxf03QGnWLb2HW4pLiAhaA5cZXBs=
+go.opentelemetry.io/otel v1.26.0/go.mod h1:UmLkJHUAidDval2EICqBMbnAd0/m2vmpf/dAM+fvFs4=
+go.opentelemetry.io/otel/metric v1.26.0 h1:7S39CLuY5Jgg9CrnA9HHiEjGMF/X2VHvoXGgSllRz30=
+go.opentelemetry.io/otel/metric v1.26.0/go.mod h1:SY+rHOI4cEawI9a7N1A4nIg/nTQXe1ccCNWYOJUrpX4=
+go.opentelemetry.io/otel/sdk v1.26.0 h1:Y7bumHf5tAiDlRYFmGqetNcLaVUZmh4iYfmGxtmz7F8=
+go.opentelemetry.io/otel/sdk v1.26.0/go.mod h1:0p8MXpqLeJ0pzcszQQN4F0S5FVjBLgypeGSngLsmirs=
+go.opentelemetry.io/otel/trace v1.26.0 h1:1ieeAUb4y0TE26jUFrCIXKpTuVK7uJGN9/Z/2LP5sQA=
+go.opentelemetry.io/otel/trace v1.26.0/go.mod h1:4iDxvGDQuUkHve82hJJ8UqrwswHYsZuWCBllGV2U2y0=
 go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
 go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=

sqlconnect/internal/bigquery/db.go

@@ -30,9 +30,6 @@ func NewDB(configJSON json.RawMessage) (*DB, error) {
 
 	db := sql.OpenDB(driver.NewConnector(
 		config.ProjectID,
-		driver.Config{
-			JobRateLimitExceededRetryEnabled: true,
-		},
 		option.WithCredentialsJSON([]byte(config.CredentialsJSON))),
 	)
 

sqlconnect/internal/bigquery/driver/connection.go

@@ -5,16 +5,12 @@ import (
 	"database/sql/driver"
 	"errors"
 	"fmt"
-	"strings"
-	"time"
 
 	"cloud.google.com/go/bigquery"
-	"github.com/cenkalti/backoff/v4"
 	"google.golang.org/api/iterator"
 )
 
 type bigQueryConnection struct {
-	config Config
 	ctx    context.Context
 	client *bigquery.Client
 	closed bool
@@ -80,28 +76,3 @@ func (bigQueryConnection) CheckNamedValue(*driver.NamedValue) error {
 func (connection *bigQueryConnection) BigqueryClient() *bigquery.Client {
 	return connection.client
 }
-
-// readWithBackoff will retry the read operation if the error is [jobRateLimitExceeded] and the config is set to retry rate limit errors
-//
-// TODO: this should no longer be needed once this fix is released:
-// https://github.com/googleapis/google-cloud-go/pull/9726
-func (connection *bigQueryConnection) readWithBackoff(ctx context.Context, query *bigquery.Query) (it *bigquery.RowIterator, err error) {
-	if !connection.config.JobRateLimitExceededRetryEnabled {
-		return query.Read(ctx)
-	}
-	// mimicking google's own retry backoff settings
-	// https://github.com/googleapis/google-cloud-go/blob/b2e704d9d287445304d2b6030b6e35a4eb8be80a/bigquery/bigquery.go#L236
-	retry := backoff.WithContext(backoff.NewExponentialBackOff(
-		backoff.WithInitialInterval(1*time.Second),
-		backoff.WithMaxInterval(32*time.Second),
-		backoff.WithMultiplier(2),
-	), ctx)
-	_ = backoff.Retry(func() error {
-		it, err = query.Read(ctx)
-		if err != nil && (strings.Contains(err.Error(), "jobRateLimitExceeded")) {
-			return err
-		}
-		return nil
-	}, retry)
-	return it, err
-}

sqlconnect/internal/bigquery/driver/connector.go

@@ -8,21 +8,15 @@ import (
 	"google.golang.org/api/option"
 )
 
-type Config struct {
-	JobRateLimitExceededRetryEnabled bool // Enable jobRateLimitExceeded retries: default false
-}
-
-func NewConnector(projectID string, config Config, opts ...option.ClientOption) driver.Connector {
+func NewConnector(projectID string, opts ...option.ClientOption) driver.Connector {
 	return &bigQueryConnector{
 		projectID: projectID,
-		config:    config,
 		opts:      opts,
 	}
 }
 
 type bigQueryConnector struct {
 	projectID string
-	config    Config
 	opts      []option.ClientOption
 }
 
@@ -33,7 +27,6 @@ func (c *bigQueryConnector) Connect(ctx context.Context) (driver.Conn, error) {
 	}
 
 	return &bigQueryConnection{
-		config: c.config,
 		ctx:    ctx,
 		client: client,
 	}, nil

sqlconnect/internal/bigquery/driver/driver_test.go

@@ -12,7 +12,6 @@ import (
 	"time"
 
 	"github.com/stretchr/testify/require"
-	"golang.org/x/sync/errgroup"
 	"google.golang.org/api/option"
 
 	"github.com/rudderlabs/rudder-go-kit/testhelper/rand"
@@ -34,7 +33,7 @@ func TestBigqueryDriver(t *testing.T) {
 	require.NoError(t, json.Unmarshal([]byte(configJSON), &c))
 
 	t.Run("OpenDB", func(t *testing.T) {
-		db := sql.OpenDB(driver.NewConnector(c.ProjectID, driver.Config{}, option.WithCredentialsJSON([]byte(c.CredentialsJSON))))
+		db := sql.OpenDB(driver.NewConnector(c.ProjectID, option.WithCredentialsJSON([]byte(c.CredentialsJSON))))
 		t.Cleanup(func() {
 			require.NoError(t, db.Close(), "it should be able to close the database connection")
 		})
@@ -224,51 +223,6 @@ func TestBigqueryDriver(t *testing.T) {
 			require.NoError(t, rows.Err())
 		})
 	})
-
-	// TODO: this test should start failing once this fix is released:
-	// https://github.com/googleapis/google-cloud-go/pull/9726
-	t.Run("jobRateLimitExceeded", func(t *testing.T) {
-		t.Run("selecting from information schema from multiple go-routines should lead to a jobRateLimitExceeded error", func(t *testing.T) {
-			db := sql.OpenDB(driver.NewConnector(c.ProjectID, driver.Config{}, option.WithCredentialsJSON([]byte(c.CredentialsJSON))))
-			t.Cleanup(func() {
-				require.NoError(t, db.Close(), "it should be able to close the database connection")
-			})
-			ctx, cancel := context.WithCancel(context.Background())
-			t.Cleanup(cancel)
-
-			g, ctx := errgroup.WithContext(ctx)
-			g.SetLimit(10)
-			for i := 0; i < 200; i++ {
-				g.Go(func() error {
-					_, err := db.ExecContext(ctx, "SELECT * FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME = 'invalid'")
-					return err
-				})
-			}
-			err := g.Wait()
-			require.Error(t, err, "it should return an error")
-			require.Contains(t, err.Error(), "jobRateLimitExceeded", "it should return a jobRateLimitExceeded error")
-		})
-
-		t.Run("selecting from information schema from multiple go-routines and having rate limit retries enabled, shoudln't lead to a jobRateLimitExceeded error", func(t *testing.T) {
-			db := sql.OpenDB(driver.NewConnector(c.ProjectID, driver.Config{JobRateLimitExceededRetryEnabled: true}, option.WithCredentialsJSON([]byte(c.CredentialsJSON))))
-			t.Cleanup(func() {
-				require.NoError(t, db.Close(), "it should be able to close the database connection")
-			})
-			ctx, cancel := context.WithCancel(context.Background())
-			t.Cleanup(cancel)
-
-			g, ctx := errgroup.WithContext(ctx)
-			g.SetLimit(10)
-			for i := 0; i < 50; i++ {
-				g.Go(func() error {
-					_, err := db.ExecContext(ctx, "SELECT * FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME = 'invalid'")
-					return err
-				})
-			}
-			err := g.Wait()
-			require.NoError(t, err, "it shouldn't return an error")
-		})
-	})
 }
 
 type config struct {

sqlconnect/internal/bigquery/driver/statement.go

@@ -41,7 +41,7 @@ func (statement *bigQueryStatement) ExecContext(ctx context.Context, args []driv
 		return nil, err
 	}
 
-	rowIterator, err := statement.connection.readWithBackoff(ctx, query)
+	rowIterator, err := query.Read(ctx)
 	if err != nil {
 		return nil, err
 	}
@@ -55,7 +55,7 @@ func (statement *bigQueryStatement) QueryContext(ctx context.Context, args []dri
 		return nil, err
 	}
 
-	rowIterator, err := statement.connection.readWithBackoff(ctx, query)
+	rowIterator, err := query.Read(ctx)
 	if err != nil {
 		return nil, err
 	}