Remove vtworker stragglers

Signed-off-by: Matt Lord <[email protected]>

Makefile

@@ -107,21 +107,21 @@ endif
 install: build
 	# binaries
 	mkdir -p "$${PREFIX}/bin"
-	cp "$${VTROOT}/bin/"{mysqlctl,mysqlctld,vtorc,vtadmin,vtctld,vtctlclient,vtctldclient,vtgate,vttablet,vtworker,vtbackup} "$${PREFIX}/bin/"
+	cp "$${VTROOT}/bin/"{mysqlctl,mysqlctld,vtorc,vtadmin,vtctld,vtctlclient,vtctldclient,vtgate,vttablet,vtbackup} "$${PREFIX}/bin/"
 
 # Will only work inside the docker bootstrap for now
 cross-install: cross-build
 	# binaries
 	mkdir -p "$${PREFIX}/bin"
 	# Still no vtorc for cross-compile
-	cp "/go/bin/${GOOS}_${GOARCH}/"{mysqlctl,mysqlctld,vtadmin,vtctld,vtctlclient,vtctldclient,vtgate,vttablet,vtworker,vtbackup} "$${PREFIX}/bin/"
+	cp "/go/bin/${GOOS}_${GOARCH}/"{mysqlctl,mysqlctld,vtadmin,vtctld,vtctlclient,vtctldclient,vtgate,vttablet,vtbackup} "$${PREFIX}/bin/"
 
 # Install local install the binaries needed to run vitess locally
 # Usage: make install-local PREFIX=/path/to/install/root
 install-local: build
 	# binaries
 	mkdir -p "$${PREFIX}/bin"
-	cp "$${VTROOT}/bin/"{mysqlctl,mysqlctld,vtorc,vtadmin,vtctl,vtctld,vtctlclient,vtctldclient,vtgate,vttablet,vtworker,vtbackup} "$${PREFIX}/bin/"
+	cp "$${VTROOT}/bin/"{mysqlctl,mysqlctld,vtorc,vtadmin,vtctl,vtctld,vtctlclient,vtctldclient,vtgate,vttablet,vtbackup} "$${PREFIX}/bin/"
 
 
 # install copies the files needed to run test Vitess using vtcombo into the given directory tree.

docker/k8s/Dockerfile

@@ -49,7 +49,6 @@ COPY --from=base /vt/bin/vtctl /vt/bin/
 COPY --from=base /vt/bin/vtctlclient /vt/bin/
 COPY --from=base /vt/bin/vtgate /vt/bin/
 COPY --from=base /vt/bin/vttablet /vt/bin/
-COPY --from=base /vt/bin/vtworker /vt/bin/
 COPY --from=base /vt/bin/vtbackup /vt/bin/
 COPY --from=base /vt/bin/vtadmin /vt/bin/
 

docker/release.sh

@@ -56,11 +56,6 @@ do
   docker push vitess/vtctld:$vt_base_version-$debian_version
   if [[ $debian_version == $default_debian_version ]]; then docker push vitess/vtctld:$vt_base_version; fi
 
-  docker build --platform linux/amd64 --build-arg VT_BASE_VER=$vt_base_version --build-arg DEBIAN_VER=$debian_version-slim -t vitess/vtworker:$vt_base_version-$debian_version k8s/vtworker
-  docker tag vitess/vtworker:$vt_base_version-$debian_version vitess/vtworker:$vt_base_version
-  docker push vitess/vtworker:$vt_base_version-$debian_version
-  if [[ $debian_version == $default_debian_version ]]; then docker push vitess/vtworker:$vt_base_version; fi
-
   docker build --platform linux/amd64 --build-arg VT_BASE_VER=$vt_base_version --build-arg DEBIAN_VER=$debian_version-slim -t vitess/logrotate:$vt_base_version-$debian_version k8s/logrotate
   docker tag vitess/logrotate:$vt_base_version-$debian_version vitess/logrotate:$vt_base_version
   docker push vitess/logrotate:$vt_base_version-$debian_version

examples/compose/docker-compose.yml

@@ -393,17 +393,4 @@ services:
     - "3306"
     volumes:
     - .:/script
-  vtwork:
-    command:
-    - sh
-    - -c
-    - '/vt/bin/vtworker -topo_implementation consul -topo_global_server_address consul1:8500
-      -topo_global_root vitess/global -cell test -logtostderr=true -service_map ''grpc-vtworker''
-      -port 8080 -grpc_port 15999 -use_v3_resharding_mode=true '
-    depends_on:
-    - vtctld
-    image: vitess/lite:${VITESS_TAG:-latest}
-    ports:
-    - "8080"
-    - "15999"
 version: "2.1"

examples/compose/vtcompose/docker-compose.test.yml

@@ -301,18 +301,4 @@ services:
       - "3306"
     volumes:
       - .:/script
-  vtwork:
-    command:
-      - sh
-      - -c
-      - '$$VTROOT/bin/vtworker -topo_implementation consul -topo_global_server_address
-      consul1:8500 -topo_global_root vitess/global -cell test -logtostderr=true -service_map
-      ''grpc-vtworker'' -port 8080 -grpc_port 15999 -use_v3_resharding_mode=true -pid_file
-      $$VTDATAROOT/tmp/vtwork.pid '
-    depends_on:
-      - vtctld
-    image: vitess/base
-    ports:
-      - 15100:8080
-      - "15999"
 version: "2.1"

examples/compose/vtcompose/vtcompose.go

@@ -478,7 +478,6 @@ func applyDefaultDockerPatches(
 
 	dockerComposeFile = applyInMemoryPatch(dockerComposeFile, generateVtctld(opts))
 	dockerComposeFile = applyInMemoryPatch(dockerComposeFile, generateVtgate(opts))
-	dockerComposeFile = applyInMemoryPatch(dockerComposeFile, generateVtwork(opts))
 	dockerComposeFile = applyInMemoryPatch(dockerComposeFile, generateVreplication(dbInfo, opts))
 	dockerComposeFile = applyInMemoryPatch(dockerComposeFile, generateVtorc(dbInfo, opts))
 	return dockerComposeFile
@@ -721,29 +720,6 @@ func generateVtgate(opts vtOptions) string {
 `, opts.webPort, opts.gRpcPort, opts.mySqlPort, opts.topologyFlags, opts.cell)
 }
 
-func generateVtwork(opts vtOptions) string {
-	return fmt.Sprintf(`
-- op: add
-  path: /services/vtwork
-  value:
-    image: vitess/lite:${VITESS_TAG:-latest}
-    ports:
-      - "%[1]d"
-      - "%[2]d"
-    command: ["sh", "-c", "/vt/bin/vtworker \
-        %[3]s \
-        -cell %[4]s \
-        -logtostderr=true \
-        -service_map 'grpc-vtworker' \
-        -port %[1]d \
-        -grpc_port %[2]d \
-        -use_v3_resharding_mode=true \
-        "]
-    depends_on:
-      - vtctld
-`, opts.webPort, opts.gRpcPort, opts.topologyFlags, opts.cell)
-}
-
 func generateVtorc(dbInfo externalDbInfo, opts vtOptions) string {
 	externalDb := "0"
 	if dbInfo.dbName != "" {

go/flags/endtoend/vtgate.txt

@@ -278,7 +278,7 @@ Usage of vtgate:
   --security_policy string
 	the name of a registered security policy to use for controlling access to URLs - empty means allow all for anyone (built-in policies: deny-all, read-only)
   --service_map value
-	comma separated list of services to enable (or disable if prefixed with '-') Example: grpc-vtworker
+	comma separated list of services to enable (or disable if prefixed with '-') Example: grpc-vtgateservice
   --sql-max-length-errors int
 	truncate queries in error logs to the given length (default unlimited)
   --sql-max-length-ui int

go/flags/endtoend/vttablet.txt

@@ -748,7 +748,7 @@ Usage of vttablet:
   --security_policy string
 	the name of a registered security policy to use for controlling access to URLs - empty means allow all for anyone (built-in policies: deny-all, read-only)
   --service_map value
-	comma separated list of services to enable (or disable if prefixed with '-') Example: grpc-vtworker
+	comma separated list of services to enable (or disable if prefixed with '-') Example: grpc-vtgateservice
   --serving_state_grace_period duration
 	how long to pause after broadcasting health to vtgate, before enforcing a new serving state
   --shard_sync_retry_delay duration

go/test/endtoend/cluster/vtgate_process.go

@@ -236,7 +236,7 @@ func VtgateProcessInstance(
 		Binary:                "vtgate",
 		FileToLogQueries:      path.Join(tmpDirectory, "/vtgate_querylog.txt"),
 		Directory:             os.Getenv("VTDATAROOT"),
-		ServiceMap:            "grpc-tabletmanager,grpc-throttler,grpc-queryservice,grpc-updatestream,grpc-vtctl,grpc-vtworker,grpc-vtgateservice",
+		ServiceMap:            "grpc-tabletmanager,grpc-throttler,grpc-queryservice,grpc-updatestream,grpc-vtctl,grpc-vtgateservice",
 		LogDir:                tmpDirectory,
 		Port:                  port,
 		GrpcPort:              grpcPort,

go/test/fuzzing/autogenerate/convert_grep_to_fuzzer.go

@@ -53,7 +53,6 @@ var importPathShort = map[string]string{
 	"vitess.io/vitess/go/vt/proto/vschema":           "vschema",
 	"vitess.io/vitess/go/vt/proto/mysqlctl":          "mysqlctl",
 	"vitess.io/vitess/go/vt/proto/vtadmin":           "vtadmin",
-	"vitess.io/vitess/go/vt/proto/vtworkerdata":      "vtworkerdata",
 	"vitess.io/vitess/go/vt/proto/throttlerdata":     "throttlerdata",
 	"vitess.io/vitess/go/vt/proto/topodata":          "topodata",
 }
@@ -77,7 +76,6 @@ var pathToImportPath = map[string]string{
 	"./proto/vschema/vschema_vtproto.pb.go":                     "vitess.io/vitess/go/vt/proto/vschema",
 	"./proto/mysqlctl/mysqlctl_vtproto.pb.go":                   "vitess.io/vitess/go/vt/proto/mysqlctl",
 	"./proto/vtadmin/vtadmin_vtproto.pb.go":                     "vitess.io/vitess/go/vt/proto/vtadmin",
-	"./proto/vtworkerdata/vtworkerdata_vtproto.pb.go":           "vitess.io/vitess/go/vt/proto/vtworkerdata",
 	"./proto/throttlerdata/throttlerdata_vtproto.pb.go":         "vitess.io/vitess/go/vt/proto/throttlerdata",
 	"./proto/topodata/topodata_vtproto.pb.go":                   "vitess.io/vitess/go/vt/proto/topodata",
 }

go/vt/discovery/utils.go

@@ -33,10 +33,6 @@ func RemoveUnhealthyTablets(tabletStatsList []LegacyTabletStats) []LegacyTabletS
 	result := make([]LegacyTabletStats, 0, len(tabletStatsList))
 	for _, ts := range tabletStatsList {
 		// Note we do not check the 'Serving' flag here.
-		// This is mainly to avoid the case where we run a vtworker Diff between a
-		// source and destination, and the source is not serving (disabled by
-		// TabletControl). When we switch the tablet to 'worker', it will
-		// go back to serving state.
 		if ts.Stats == nil || ts.Stats.HealthError != "" || ts.LastError != nil || LegacyIsReplicationLagHigh(&ts) {
 			continue
 		}

go/vt/mysqlctl/mysqld.go

@@ -59,8 +59,6 @@ var (
 	// 2. in vtctld so it can be exported to the UI (different
 	// package, that's why it's exported). That way we can disable
 	// menu items there, using features.
-	// 3. prevents the vtworker from updating replication topology
-	// after restarting replication after a split clone/diff.
 	DisableActiveReparents = flag.Bool("disable_active_reparents", false, "if set, do not allow active reparents. Use this to protect a cluster using external reparents.")
 
 	dbaPoolSize = flag.Int("dba_pool_size", 20, "Size of the connection pool for dba connections")

go/vt/servenv/service_map.go

@@ -34,7 +34,7 @@ var (
 )
 
 func init() {
-	flag.Var(&serviceMapFlag, "service_map", "comma separated list of services to enable (or disable if prefixed with '-') Example: grpc-vtworker")
+	flag.Var(&serviceMapFlag, "service_map", "comma separated list of services to enable (or disable if prefixed with '-') Example: grpc-vtgateservice")
 	OnInit(func() {
 		updateServiceMap()
 	})

go/vt/vtctl/fakevtctlclient/fake_loggerevent_streamingclient.go

@@ -28,7 +28,7 @@ import (
 	logutilpb "vitess.io/vitess/go/vt/proto/logutil"
 )
 
-// FakeLoggerEventStreamingClient is the base for the fakes for the vtctlclient and vtworkerclient.
+// FakeLoggerEventStreamingClient is the base for the fakes for vtctlclient.
 // It allows to register a (multi-)line string for a given command and return the result as channel which streams it back.
 type FakeLoggerEventStreamingClient struct {
 	results map[string]*result

go/vt/vtctl/vtctl.go

@@ -63,9 +63,7 @@ COMMAND ARGUMENT DEFINITIONS
              for backup purposes
   -- batch: A replicated copy of data for OLAP load patterns (typically for
             MapReduce jobs)
-  -- drained: A tablet that is reserved for a background process. For example,
-              a tablet used by a vtworker process, where the tablet is likely
-              lagging in replication.
+  -- drained: A tablet that is reserved for a background process.
   -- experimental: A replica copy of data that is ready but not serving query
                    traffic. The value indicates a special characteristic of
                    the tablet that indicates the tablet should not be

go/vt/vttablet/tabletmanager/rpc_actions.go

@@ -87,9 +87,7 @@ func (tm *TabletManager) ChangeType(ctx context.Context, tabletType topodatapb.T
 
 // ChangeType changes the tablet type
 func (tm *TabletManager) changeTypeLocked(ctx context.Context, tabletType topodatapb.TabletType, action DBAction, semiSync SemiSyncAction) error {
-	// We don't want to allow multiple callers to claim a tablet as drained. There is a race that could happen during
-	// horizontal resharding where two vtworkers will try to DRAIN the same tablet. This check prevents that race from
-	// causing errors.
+	// We don't want to allow multiple callers to claim a tablet as drained.
 	if tabletType == topodatapb.TabletType_DRAINED && tm.Tablet().Type == topodatapb.TabletType_DRAINED {
 		return fmt.Errorf("Tablet: %v, is already drained", tm.tabletAlias)
 	}

go/vt/vttablet/tabletserver/query_executor.go

@@ -414,7 +414,6 @@ func (qre *QueryExecutor) checkPermissions() error {
 	}
 
 	// Skip the ACL check if the connecting user is an exempted superuser.
-	// Necessary to whitelist e.g. direct vtworker access.
 	if qre.tsv.qe.exemptACL != nil && qre.tsv.qe.exemptACL.IsMember(&querypb.VTGateCallerID{Username: username}) {
 		qre.tsv.qe.tableaclExemptCount.Add(1)
 		return nil

go/vt/wrangler/keyspace.go

@@ -120,43 +120,6 @@ func (wr *Wrangler) printShards(ctx context.Context, si []*topo.ShardInfo) error
 	return nil
 }
 
-// findSourceDest derives the source and destination from the overlapping shards.
-// Whichever side has SourceShards is a destination.
-func (wr *Wrangler) findSourceDest(ctx context.Context, os *topotools.OverlappingShards) (sourceShards, destinationShards []*topo.ShardInfo, err error) {
-	// It's possible that both source and destination have source shards because of reversible replication.
-	// If so, the Frozen flag in the tablet control record dictates the direction.
-	// So, check that first.
-	for _, left := range os.Left {
-		tc := left.GetTabletControl(topodatapb.TabletType_PRIMARY)
-		if tc == nil {
-			continue
-		}
-		if tc.Frozen {
-			return os.Left, os.Right, nil
-		}
-	}
-	for _, right := range os.Right {
-		tc := right.GetTabletControl(topodatapb.TabletType_PRIMARY)
-		if tc == nil {
-			continue
-		}
-		if tc.Frozen {
-			return os.Right, os.Left, nil
-		}
-	}
-	for _, left := range os.Left {
-		if len(left.SourceShards) != 0 {
-			return os.Right, os.Left, nil
-		}
-	}
-	for _, right := range os.Right {
-		if len(right.SourceShards) != 0 {
-			return os.Left, os.Right, nil
-		}
-	}
-	return nil, nil, fmt.Errorf("neither Shard '%v' nor Shard '%v' have a 'SourceShards' entry. Did you successfully run vtworker SplitClone before? Or did you already migrate the MASTER type?", os.Left[0].ShardName(), os.Right[0].ShardName())
-}
-
 func (wr *Wrangler) getPrimaryPositions(ctx context.Context, shards []*topo.ShardInfo) (map[*topo.ShardInfo]string, error) {
 	mu := sync.Mutex{}
 	result := make(map[*topo.ShardInfo]string)

go/vt/wrangler/schema.go

@@ -261,8 +261,7 @@ func (wr *Wrangler) CopySchemaShard(ctx context.Context, sourceTabletAlias *topo
 	// where the database already existed on the destination, but with different
 	// options e.g. a different character set.
 	// In that case, MySQL would have skipped our CREATE DATABASE IF NOT EXISTS
-	// statement. We want to fail early in this case because vtworker SplitDiff
-	// fails in case of such an inconsistency as well.
+	// statement.
 	if !skipVerify {
 		diffs, err = schematools.CompareSchemas(ctx, wr.ts, wr.tmc, sourceTabletAlias, destShardInfo.PrimaryAlias, tables, excludeTables, includeViews)
 		if err != nil {

proto/throttlerservice.proto

@@ -15,8 +15,7 @@ limitations under the License.
 */
 
 // gRPC RPC interface for the internal resharding throttler (go/vt/throttler)
-// which is used by the resharding clone process (vtworker) and filtered
-// replication (vttablet).
+// which is used vreplication.
 
 syntax = "proto3";
 option go_package = "vitess.io/vitess/go/vt/proto/throttlerservice";

tools/make-release-packages.sh

@@ -35,7 +35,7 @@ mkdir -p releases
 
 # Copy a subset of binaries from issue #5421
 mkdir -p "${RELEASE_DIR}/bin"
-for binary in vttestserver mysqlctl mysqlctld query_analyzer topo2topo vtaclcheck vtadmin vtbackup vtbench vtclient vtcombo vtctl vtctldclient vtctlclient vtctld vtexplain vtgate vttablet vtorc vtworker vtworkerclient zk zkctl zkctld; do
+for binary in vttestserver mysqlctl mysqlctld query_analyzer topo2topo vtaclcheck vtadmin vtbackup vtbench vtclient vtcombo vtctl vtctldclient vtctlclient vtctld vtexplain vtgate vttablet vtorc zk zkctl zkctld; do
  cp "bin/$binary" "${RELEASE_DIR}/bin/"
 done;
 

vitess-mixin/config.libsonnet

@@ -5,7 +5,6 @@
     vtctldSelector: 'job="vitess-vtctld"',
     vtgateSelector: 'job="vitess-vtgate"',
     vttabletSelector: 'job="vitess-vttablet"',
-    vtworkerSelector: 'job="vitess-vtworker"',
     vtgateNodeSelector: 'job="node-exporter-vitess-vtgate"',
     mysqlSelector: 'job="mysql"',
     defaultTimeFrom: 'now-30m',

vitess-mixin/dashboards/layouts/cluster_overview.libsonnet

@@ -29,7 +29,6 @@ local config = import '../../config.libsonnet';
         singlestats.vtgateUp { gridPos: { h: 2, w: 2, x: 20, y: 3 } },
         singlestats.vtctldUp { gridPos: { h: 2, w: 2, x: 20, y: 5 } },
         singlestats.vttabletUp { gridPos: { h: 2, w: 2, x: 22, y: 3 } },
-        singlestats.vtworkerUp { gridPos: { h: 2, w: 2, x: 22, y: 5 } },
 
         helpers.vtgate.getPanel(config.vtgate.panels.vtgateRequests) { gridPos: { h: 6, w: 8, x: 0, y: 7 } },
         helpers.vtgate.getPanel(config.vtgate.panels.vtgateErrorRate) { gridPos: { h: 6, w: 8, x: 8, y: 7 } },

vitess-mixin/dashboards/resources/grafonnet/panels.libsonnet

@@ -6,7 +6,7 @@ local graphPanel = grafana.graphPanel;
 local prometheus = grafana.prometheus;
 
 // TODO: add description for each panel.
-// TODO: create a _helper _config file for each group [vtctld,vtworker],
+// TODO: create a _helper _config file for each group [vtctld],
 {
   //        _       _   _     _
   // __   _| |_ ___| |_| | __| |
@@ -15,13 +15,6 @@ local prometheus = grafana.prometheus;
   //   \_/  \__\___|\__|_|\__,_|
   //
 
-  //        _                      _
-  // __   _| |___      _____  _ __| | _____ _ __
-  // \ \ / / __\ \ /\ / / _ \| '__| |/ / _ \ '__|
-  //  \ V /| |_ \ V  V / (_) | |  |   <  __/ |
-  //   \_/  \__| \_/\_/ \___/|_|  |_|\_\___|_|
-  //
-
   //            _
   //  _ __ ___ (_)___  ___
   // | '_ ` _ \| / __|/ __|

vitess-mixin/dashboards/resources/grafonnet/singlestats.libsonnet

@@ -179,27 +179,6 @@ local prometheus = grafana.prometheus;
       )
     ),
 
-  vtworkerUp::
-    singlestat.new(
-      'vtworker',
-      datasource='%(dataSource)s' % config._config,
-      valueFontSize='50%',
-      valueName='current',
-    )
-    .addTarget(
-      prometheus.target(
-        |||
-          sum(
-            up{
-              %(vtworkerSelector)s
-            }
-          )
-        ||| % config._config,
-        instant=true,
-        intervalFactor=1
-      )
-    ),
-
   mysqlQPS::
     singlestat.new(
       'QPS - MySQL',

vitess-mixin/e2e/cypress/integration/cluster_overview_spec.js

@@ -41,7 +41,6 @@ describe('vitess-mixin: Cluster Overview Dashboard Test', function () {
         'keyspace',
         'shard',
         'vtctld',
-        'vtworker',
         'Requests',
         'Error rate',
         'Duration 99th quantile',
@@ -54,4 +53,4 @@ describe('vitess-mixin: Cluster Overview Dashboard Test', function () {
       ])
     })
   })
-})
+})