Vectors: Online DDL support (#5461)

* Vectors: Online DDL support

Signed-off-by: Shlomi Noach <[email protected]>

* Add support for vectors in vreplication workflows. unit and e2e tests. Update cnf file to add loose_with_ps_vectors

Signed-off-by: Rohit Nayak <[email protected]>

* Self review fixes. Add e2e TestVector to CI

Signed-off-by: Rohit Nayak <[email protected]>

* Simplify unit test CI workflow to debug failure

Signed-off-by: Rohit Nayak <[email protected]>

* Skip test on errors on setting with_ps_vectors

Signed-off-by: Rohit Nayak <[email protected]>

* Revert debug mods to unit_race.yml

Signed-off-by: Rohit Nayak <[email protected]>

* Unit test race is again failing with no logs. So modify CI workflow temporarily to get full detail to debug random failures apparently unrelated to this PR

Signed-off-by: Rohit Nayak <[email protected]>

* Setup templating for vectors vreplication build

Signed-off-by: Dirkjan Bussink <[email protected]>

* Add vcopier unit test. Insert additional rows in e2e test after copy phase to test replication phase

Signed-off-by: Rohit Nayak <[email protected]>

* Switch shard for vector test to correct one

Signed-off-by: Rohit Nayak <[email protected]>

* Use vectors build for unit tests

Signed-off-by: Dirkjan Bussink <[email protected]>

* Refactor tests: move vector unit tests to separate files

Signed-off-by: Rohit Nayak <[email protected]>

* endtoend suite: filter with ONLINEDDL_SUITE_TEST_FILTER

Signed-off-by: Shlomi Noach <[email protected]>

* recognize ERVectorDimensionsMismatch      = ErrorCode(4501) as unrecoverable

Signed-off-by: Shlomi Noach <[email protected]>

* add expected failign vector test. Rename vector tests

Signed-off-by: Shlomi Noach <[email protected]>

* recognize ERVectorDimensionsMismatch      = ErrorCode(4501) as unrecoverable

Signed-off-by: Shlomi Noach <[email protected]>

* Unit race still failing without details:-(. Commenting out the summary code which seems to be redacting the details

Signed-off-by: Rohit Nayak <[email protected]>

* 'onlineddl_vrepl_suite' now requires a vectors build

Signed-off-by: Shlomi Noach <[email protected]>

* use vectors build

Signed-off-by: Shlomi Noach <[email protected]>

* update mysql version

Signed-off-by: Shlomi Noach <[email protected]>

---------

Signed-off-by: Shlomi Noach <[email protected]>
Signed-off-by: Rohit Nayak <[email protected]>
Signed-off-by: Dirkjan Bussink <[email protected]>
Co-authored-by: Rohit Nayak <[email protected]>
Co-authored-by: Dirkjan Bussink <[email protected]>

.github/workflows/cluster_endtoend_onlineddl_vrepl_suite.yml

@@ -87,8 +87,8 @@ jobs:
         sudo apt-get -qq install -y make unzip g++ etcd curl git wget awscli eatmydata xz-utils libncurses5
         sudo service etcd stop
 
-        # Pin this to 8.0.34
-        LAST_BUILD="mysql/main/8.0.34.20240216-ps-6ce6a79427b/dist/mysql-8.0.34.20240216-ps-6ce6a79427b-jammy-linux-x86_64.tar.gz"
+        # Using the 8.0.37 MySQL Vectors build
+        LAST_BUILD="mysql/main/8.0.37.20240627-ps-c583fd90bb9/dist/mysql-8.0.37.20240627-ps-c583fd90bb9-jammy-linux-x86_64.tar.gz"
 
         echo "installing psdb mysql $LAST_BUILD"
         aws s3 cp "s3://planetscale-mysql-server-private-ci-artifacts/${LAST_BUILD}" .

.github/workflows/cluster_endtoend_vreplication_vector.yml

@@ -87,7 +87,7 @@ jobs:
         sudo service etcd stop
 
         # Using the 8.0.37 MySQL Vectors build
-        LAST_BUILD="mysql/main/8.0.37.20240624-ps-212d2140ff5/dist/mysql-8.0.37.20240624-ps-212d2140ff5-jammy-linux-x86_64.tar.gz"
+        LAST_BUILD="mysql/main/8.0.37.20240627-ps-c583fd90bb9/dist/mysql-8.0.37.20240627-ps-c583fd90bb9-jammy-linux-x86_64.tar.gz"
 
         echo "installing psdb mysql $LAST_BUILD"
         aws s3 cp "s3://planetscale-mysql-server-private-ci-artifacts/${LAST_BUILD}" .

config/mycnf/mysql80.cnf

@@ -61,3 +61,4 @@ super-read-only
 slave_net_timeout = 8
 loose_with_ps_vectors = 1
 
+loose_with_ps_vectors = 1

go/test/endtoend/onlineddl/vrepl_suite/onlineddl_vrepl_suite_test.go

@@ -56,16 +56,21 @@ var (
 	beforeTableName       = `onlineddl_test_before`
 	afterTableName        = `onlineddl_test_after`
 	eventName             = `onlineddl_test`
+
+	testsFilter = ""
 )
 
 const (
-	testDataPath = "testdata"
+	testDataPath     = "testdata"
+	testFilterEnvVar = "ONLINEDDL_SUITE_TEST_FILTER"
 )
 
 func TestMain(m *testing.M) {
 	defer cluster.PanicHandler(nil)
 	flag.Parse()
 
+	testsFilter = os.Getenv(testFilterEnvVar)
+
 	exitcode, err := func() (int, error) {
 		clusterInstance = cluster.NewCluster(cell, hostname)
 		schemaChangeDirectory = path.Join("/tmp", fmt.Sprintf("schema_change_dir_%d", clusterInstance.GetAndReserveTabletUID()))
@@ -183,6 +188,10 @@ func readTestFile(t *testing.T, testName string, fileName string) (content strin
 // testSingle is the main testing function for a single test in the suite.
 // It prepares the grounds, creates the test data, runs a migration, expects results/error, cleans up.
 func testSingle(t *testing.T, testName string, fkOnlineDDLPossible bool) {
+	if !strings.Contains(testName, testsFilter) {
+		t.Skipf("Skipping test %s due to filter: %s=%s", testName, testFilterEnvVar, testsFilter)
+		return
+	}
 	if _, exists := readTestFile(t, testName, "require_rename_table_preserve_foreign_key"); exists {
 		if !fkOnlineDDLPossible {
 			t.Skipf("Skipping test due to require_rename_table_preserve_foreign_key")

go/test/endtoend/onlineddl/vrepl_suite/testdata/add-2nd-vector-key/alter

@@ -0,0 +1 @@
+add vector key v1_idx (v1)

go/test/endtoend/onlineddl/vrepl_suite/testdata/add-2nd-vector-key/create.sql

@@ -0,0 +1,25 @@
+drop table if exists onlineddl_test;
+create table onlineddl_test (
+  id int auto_increment,
+  v1 vector(5),
+  v2 vector(5),
+  primary key(id),
+  vector key v2_idx (v2)
+) auto_increment=1;
+
+insert into onlineddl_test values (null, VEC_FromText('[1,1,1,1,1]'), VEC_FromText('[2,2,2,2,2]'));
+
+drop event if exists onlineddl_test;
+delimiter ;;
+create event onlineddl_test
+  on schedule every 1 second
+  starts current_timestamp
+  ends current_timestamp + interval 60 second
+  on completion not preserve
+  enable
+  do
+begin
+  insert into onlineddl_test values (null, VEC_FromText('[3,3,3,3,3]'), VEC_FromText('[4,4,4,4,4]'));
+  insert into onlineddl_test values (null, VEC_FromText('[5,5,5,5,5]'), VEC_FromText('[6,6,6,6,6]'));
+  insert into onlineddl_test values (null, VEC_FromText('[7,7,7,7,7]'), VEC_FromText('[8,8,8,8,8]'));
+end ;;

go/test/endtoend/onlineddl/vrepl_suite/testdata/add-two-vector-keys/alter

@@ -0,0 +1 @@
+add vector key v1_idx (v1), add vector key v2_idx (v2)

go/test/endtoend/onlineddl/vrepl_suite/testdata/add-two-vector-keys/create.sql

@@ -0,0 +1,24 @@
+drop table if exists onlineddl_test;
+create table onlineddl_test (
+  id int auto_increment,
+  v1 vector(5),
+  v2 vector(5),
+  primary key(id)
+) auto_increment=1;
+
+insert into onlineddl_test values (null, VEC_FromText('[1,1,1,1,1]'), VEC_FromText('[2,2,2,2,2]'));
+
+drop event if exists onlineddl_test;
+delimiter ;;
+create event onlineddl_test
+  on schedule every 1 second
+  starts current_timestamp
+  ends current_timestamp + interval 60 second
+  on completion not preserve
+  enable
+  do
+begin
+  insert into onlineddl_test values (null, VEC_FromText('[3,3,3,3,3]'), VEC_FromText('[3,3,3,3,3]'));
+  insert into onlineddl_test values (null, VEC_FromText('[4,4,4,4,4]'), VEC_FromText('[5,5,5,5,5]'));
+  insert into onlineddl_test values (null, VEC_FromText('[6,6,6,6,6]'), VEC_FromText('[7,7,7,7,7]'));
+end ;;

go/test/endtoend/onlineddl/vrepl_suite/testdata/add-vector-key/alter

@@ -0,0 +1 @@
+add vector key v1_idx (v1)

go/test/endtoend/onlineddl/vrepl_suite/testdata/add-vector-key/create.sql

@@ -0,0 +1,24 @@
+drop table if exists onlineddl_test;
+create table onlineddl_test (
+  id int auto_increment,
+  v1 vector(5),
+  v2 vector(5),
+  primary key(id)
+) auto_increment=1;
+
+insert into onlineddl_test values (null, VEC_FromText('[1,1,1,1,1]'), VEC_FromText('[2,2,2,2,2]'));
+
+drop event if exists onlineddl_test;
+delimiter ;;
+create event onlineddl_test
+  on schedule every 1 second
+  starts current_timestamp
+  ends current_timestamp + interval 60 second
+  on completion not preserve
+  enable
+  do
+begin
+  insert into onlineddl_test values (null, VEC_FromText('[3,3,3,3,3]'), VEC_FromText('[4,4,4,4,4]'));
+  insert into onlineddl_test values (null, VEC_FromText('[5,5,5,5,5]'), VEC_FromText('[6,6,6,6,6]'));
+  insert into onlineddl_test values (null, VEC_FromText('[7,7,7,7,7]'), VEC_FromText('[8,8,8,8,8]'));
+end ;;

go/test/endtoend/onlineddl/vrepl_suite/testdata/fail-modify-vector-length/alter

@@ -0,0 +1 @@
+modify v1 vector(6)

go/test/endtoend/onlineddl/vrepl_suite/testdata/fail-modify-vector-length/create.sql

@@ -0,0 +1,24 @@
+drop table if exists onlineddl_test;
+create table onlineddl_test (
+  id int auto_increment,
+  v1 vector(5),
+  v2 vector(5),
+  primary key(id)
+) auto_increment=1;
+
+insert into onlineddl_test values (null, VEC_FromText('[1,1,1,1,1]'), VEC_FromText('[2,2,2,2,2]'));
+
+drop event if exists onlineddl_test;
+delimiter ;;
+create event onlineddl_test
+  on schedule every 1 second
+  starts current_timestamp
+  ends current_timestamp + interval 60 second
+  on completion not preserve
+  enable
+  do
+begin
+  insert into onlineddl_test values (null, VEC_FromText('[3,3,3,3,3]'), VEC_FromText('[4,4,4,4,4]'));
+  insert into onlineddl_test values (null, VEC_FromText('[5,5,5,5,5]'), VEC_FromText('[6,6,6,6,6]'));
+  insert into onlineddl_test values (null, VEC_FromText('[7,7,7,7,7]'), VEC_FromText('[8,8,8,8,8]'));
+end ;;

go/test/endtoend/onlineddl/vrepl_suite/testdata/fail-modify-vector-length/expect_failure

@@ -0,0 +1 @@
+Vector object dimensions must match

go/vt/vttablet/onlineddl/executor.go

@@ -1332,6 +1332,7 @@ func (e *Executor) validateAndEditCreateTableStatement(onlineDDL *schema.OnlineD
 // validateAndEditAlterTableStatement inspects the AlterTable statement and:
 // - modifies any CONSTRAINT name according to given name mapping
 // - explode ADD FULLTEXT KEY into multiple statements
+// - explode ADD VECTOR KEY into multiple statements
 func (e *Executor) validateAndEditAlterTableStatement(capableOf capabilities.CapableOf, onlineDDL *schema.OnlineDDL, alterTable *sqlparser.AlterTable, constraintMap map[string]string) (alters []*sqlparser.AlterTable, err error) {
 	capableOfInstantDDLXtrabackup, err := capableOf(capabilities.InstantDDLXtrabackupCapability)
 	if err != nil {
@@ -1374,7 +1375,9 @@ func (e *Executor) validateAndEditAlterTableStatement(capableOf capabilities.Cap
 				continue
 			}
 		case *sqlparser.AddIndexDefinition:
-			if opt.IndexDefinition.Info.Type == sqlparser.IndexTypeFullText {
+			// FULLTEXT and VECTOR indexes both share the same limitations: you can't add more
+			// than one in a single ALTER TABLE statement.
+			if opt.IndexDefinition.Info.Type == sqlparser.IndexTypeFullText || opt.IndexDefinition.Info.Type == sqlparser.IndexTypeVector {
 				countAddFullTextStatements++
 				if countAddFullTextStatements > 1 {
 					// We've already got one ADD FULLTEXT KEY. We can't have another

test/ci_workflow_gen.go

@@ -137,6 +137,7 @@ var (
 		"vtorc",
 	}
 	clustersRequiringVectors = []string{
+		"onlineddl_vrepl_suite",
 		"vreplication_vector",
 	}
 )

test/templates/cluster_endtoend_test.tpl

@@ -102,7 +102,7 @@ jobs:
 
 {{ if .Vectors }}
         # Using the 8.0.37 MySQL Vectors build
-        LAST_BUILD="mysql/main/8.0.37.20240624-ps-212d2140ff5/dist/mysql-8.0.37.20240624-ps-212d2140ff5-jammy-linux-x86_64.tar.gz"
+        LAST_BUILD="mysql/main/8.0.37.20240627-ps-c583fd90bb9/dist/mysql-8.0.37.20240627-ps-c583fd90bb9-jammy-linux-x86_64.tar.gz"
 {{ else }}
         # Pin this to 8.0.34
         LAST_BUILD="mysql/main/8.0.34.20240216-ps-6ce6a79427b/dist/mysql-8.0.34.20240216-ps-6ce6a79427b-jammy-linux-x86_64.tar.gz"