From 489eb3c4ac1d26f187b69583ff0056c7be448c0a Mon Sep 17 00:00:00 2001 From: Rafael Chacon Date: Thu, 3 Oct 2019 15:21:50 -0700 Subject: [PATCH 01/28] Adds proto definition to store dml extracted from Query Signed-off-by: Rafael Chacon --- go/vt/proto/binlogdata/binlogdata.pb.go | 202 ++++++++++++------------ proto/binlogdata.proto | 1 + py/vtproto/binlogdata_pb2.py | 41 +++-- py/vtproto/vtrpc_pb2.py | 85 +++++----- 4 files changed, 172 insertions(+), 157 deletions(-) diff --git a/go/vt/proto/binlogdata/binlogdata.pb.go b/go/vt/proto/binlogdata/binlogdata.pb.go index 5909c807611..53663737594 100644 --- a/go/vt/proto/binlogdata/binlogdata.pb.go +++ b/go/vt/proto/binlogdata/binlogdata.pb.go @@ -1144,6 +1144,7 @@ type VEvent struct { FieldEvent *FieldEvent `protobuf:"bytes,6,opt,name=field_event,json=fieldEvent,proto3" json:"field_event,omitempty"` Vgtid *VGtid `protobuf:"bytes,7,opt,name=vgtid,proto3" json:"vgtid,omitempty"` Journal *Journal `protobuf:"bytes,8,opt,name=journal,proto3" json:"journal,omitempty"` + Dml string `protobuf:"bytes,9,opt,name=dml,proto3" json:"dml,omitempty"` // current_time specifies the current time to handle clock skew. CurrentTime int64 `protobuf:"varint,20,opt,name=current_time,json=currentTime,proto3" json:"current_time,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` @@ -1232,6 +1233,13 @@ func (m *VEvent) GetJournal() *Journal { return nil } +func (m *VEvent) GetDml() string { + if m != nil { + return m.Dml + } + return "" +} + func (m *VEvent) GetCurrentTime() int64 { if m != nil { return m.CurrentTime @@ -1527,103 +1535,103 @@ func init() { func init() { proto.RegisterFile("binlogdata.proto", fileDescriptor_5fd02bcb2e350dad) } var fileDescriptor_5fd02bcb2e350dad = []byte{ - // 1558 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xd4, 0x57, 0xcb, 0x72, 0xdb, 0xca, - 0x11, 0x35, 0x09, 0xf0, 0xd5, 0x90, 0x28, 0x68, 0xf4, 0x08, 0xa3, 0x8a, 0x53, 0x0a, 0x2a, 0x8e, + // 1568 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xd4, 0x57, 0x4b, 0x72, 0xdb, 0xcc, + 0x11, 0x36, 0x09, 0xf0, 0xd5, 0x90, 0x28, 0x68, 0xf4, 0x08, 0xa3, 0x8a, 0x53, 0x0a, 0x2a, 0x8e, 0x14, 0x55, 0x85, 0x72, 0x98, 0xc4, 0x59, 0x39, 0x0e, 0x1f, 0xb0, 0x4c, 0x09, 0x22, 0xe5, 0x21, - 0x24, 0xa7, 0xbc, 0x41, 0x41, 0xc4, 0x48, 0x42, 0x04, 0x02, 0x34, 0x30, 0xa4, 0xa2, 0x0f, 0x48, - 0xe5, 0x03, 0xb2, 0xcd, 0x0f, 0x64, 0x9f, 0x6d, 0xb6, 0xd9, 0xe7, 0x0b, 0xb2, 0xca, 0x7f, 0xdc, - 0x9a, 0x07, 0x40, 0x42, 0xf6, 0xb5, 0xe5, 0x5b, 0x75, 0x17, 0x77, 0xc3, 0xea, 0xe9, 0xe9, 0xe7, - 0x41, 0x4f, 0x77, 0x13, 0xf4, 0x4b, 0x3f, 0x0c, 0xa2, 0x6b, 0xcf, 0xa5, 0x6e, 0x73, 0x1a, 0x47, - 0x34, 0x42, 0xb0, 0xe0, 0xec, 0x68, 0x73, 0x1a, 0x4f, 0xc7, 0xe2, 0x62, 0x47, 0xfb, 0x30, 0x23, - 0xf1, 0xbd, 0x3c, 0xd4, 0x69, 0x34, 0x8d, 0x16, 0x5a, 0xc6, 0x29, 0x54, 0xba, 0x37, 0x6e, 0x9c, - 0x10, 0x8a, 0xb6, 0xa1, 0x3c, 0x0e, 0x7c, 0x12, 0xd2, 0x46, 0x61, 0xb7, 0xb0, 0x5f, 0xc2, 0xf2, - 0x84, 0x10, 0xa8, 0xe3, 0x28, 0x0c, 0x1b, 0x45, 0xce, 0xe5, 0x34, 0x93, 0x4d, 0x48, 0x3c, 0x27, - 0x71, 0x43, 0x11, 0xb2, 0xe2, 0x64, 0xfc, 0x5f, 0x81, 0xf5, 0x0e, 0x8f, 0xc3, 0x8e, 0xdd, 0x30, - 0x71, 0xc7, 0xd4, 0x8f, 0x42, 0x74, 0x04, 0x90, 0x50, 0x97, 0x92, 0x09, 0x09, 0x69, 0xd2, 0x28, - 0xec, 0x2a, 0xfb, 0x5a, 0x6b, 0xaf, 0xb9, 0x94, 0xc1, 0x47, 0x2a, 0xcd, 0x51, 0x2a, 0x8f, 0x97, - 0x54, 0x51, 0x0b, 0x34, 0x32, 0x27, 0x21, 0x75, 0x68, 0x74, 0x4b, 0xc2, 0x86, 0xba, 0x5b, 0xd8, - 0xd7, 0x5a, 0xeb, 0x4d, 0x91, 0xa0, 0xc9, 0x6e, 0x6c, 0x76, 0x81, 0x81, 0x64, 0xf4, 0xce, 0x7f, - 0x8a, 0x50, 0xcb, 0xac, 0x21, 0x0b, 0xaa, 0x63, 0x97, 0x92, 0xeb, 0x28, 0xbe, 0xe7, 0x69, 0xd6, - 0x5b, 0xcf, 0x1f, 0x19, 0x48, 0xb3, 0x2b, 0xf5, 0x70, 0x66, 0x01, 0xfd, 0x0a, 0x2a, 0x63, 0x81, - 0x1e, 0x47, 0x47, 0x6b, 0x6d, 0x2c, 0x1b, 0x93, 0xc0, 0xe2, 0x54, 0x06, 0xe9, 0xa0, 0x24, 0x1f, - 0x02, 0x0e, 0xd9, 0x0a, 0x66, 0xa4, 0xf1, 0xcf, 0x02, 0x54, 0x53, 0xbb, 0x68, 0x03, 0xd6, 0x3a, - 0x96, 0x73, 0x3e, 0xc0, 0x66, 0x77, 0x78, 0x34, 0xe8, 0xbf, 0x37, 0x7b, 0xfa, 0x13, 0xb4, 0x02, - 0xd5, 0x8e, 0xe5, 0x74, 0xcc, 0xa3, 0xfe, 0x40, 0x2f, 0xa0, 0x55, 0xa8, 0x75, 0x2c, 0xa7, 0x3b, - 0x3c, 0x3d, 0xed, 0xdb, 0x7a, 0x11, 0xad, 0x81, 0xd6, 0xb1, 0x1c, 0x3c, 0xb4, 0xac, 0x4e, 0xbb, - 0x7b, 0xa2, 0x2b, 0x68, 0x0b, 0xd6, 0x3b, 0x96, 0xd3, 0x3b, 0xb5, 0x9c, 0x9e, 0x79, 0x86, 0xcd, - 0x6e, 0xdb, 0x36, 0x7b, 0xba, 0x8a, 0x00, 0xca, 0x8c, 0xdd, 0xb3, 0xf4, 0x92, 0xa4, 0x47, 0xa6, - 0xad, 0x97, 0xa5, 0xb9, 0xfe, 0x60, 0x64, 0x62, 0x5b, 0xaf, 0xc8, 0xe3, 0xf9, 0x59, 0xaf, 0x6d, - 0x9b, 0x7a, 0x55, 0x1e, 0x7b, 0xa6, 0x65, 0xda, 0xa6, 0x5e, 0x3b, 0x56, 0xab, 0x45, 0x5d, 0x39, - 0x56, 0xab, 0x8a, 0xae, 0x1a, 0x7f, 0x2f, 0xc0, 0xd6, 0x88, 0xc6, 0xc4, 0x9d, 0x9c, 0x90, 0x7b, - 0xec, 0x86, 0xd7, 0x04, 0x93, 0x0f, 0x33, 0x92, 0x50, 0xb4, 0x03, 0xd5, 0x69, 0x94, 0xf8, 0x0c, - 0x3b, 0x0e, 0x70, 0x0d, 0x67, 0x67, 0x74, 0x08, 0xb5, 0x5b, 0x72, 0xef, 0xc4, 0x4c, 0x5e, 0x02, - 0x86, 0x9a, 0x59, 0x41, 0x66, 0x96, 0xaa, 0xb7, 0x92, 0x5a, 0xc6, 0x57, 0xf9, 0x32, 0xbe, 0xc6, - 0x15, 0x6c, 0x3f, 0x0c, 0x2a, 0x99, 0x46, 0x61, 0x42, 0x90, 0x05, 0x48, 0x28, 0x3a, 0x74, 0xf1, - 0x6d, 0x79, 0x7c, 0x5a, 0xeb, 0xe9, 0x67, 0x0b, 0x00, 0xaf, 0x5f, 0x3e, 0x64, 0x19, 0x7f, 0x81, - 0x0d, 0xe1, 0xc7, 0x76, 0x2f, 0x03, 0x92, 0x3c, 0x26, 0xf5, 0x6d, 0x28, 0x53, 0x2e, 0xdc, 0x28, - 0xee, 0x2a, 0xfb, 0x35, 0x2c, 0x4f, 0x5f, 0x9b, 0xa1, 0x07, 0x9b, 0x79, 0xcf, 0xdf, 0x4b, 0x7e, - 0xbf, 0x05, 0x15, 0xcf, 0x02, 0x82, 0x36, 0xa1, 0x34, 0x71, 0xe9, 0xf8, 0x46, 0x66, 0x23, 0x0e, - 0x2c, 0x95, 0x2b, 0x3f, 0xa0, 0x24, 0xe6, 0x9f, 0xb0, 0x86, 0xe5, 0xc9, 0x78, 0x0e, 0xe5, 0xd7, - 0x9c, 0x42, 0xbf, 0x80, 0x52, 0x3c, 0x63, 0xb9, 0x8a, 0xa7, 0xae, 0x2f, 0x07, 0xc0, 0x0c, 0x63, - 0x71, 0x6d, 0xfc, 0xa3, 0x08, 0x2b, 0x22, 0xa0, 0x51, 0x34, 0x8b, 0xc7, 0x84, 0x21, 0x78, 0x4b, - 0xee, 0x93, 0xa9, 0x3b, 0x26, 0x29, 0x82, 0xe9, 0x99, 0x05, 0x93, 0xdc, 0xb8, 0xb1, 0x27, 0xbd, - 0x8a, 0x03, 0xfa, 0x1d, 0x68, 0x1c, 0x49, 0xea, 0xd0, 0xfb, 0x29, 0xe1, 0x18, 0xd6, 0x5b, 0x9b, - 0x8b, 0xa2, 0xe2, 0x38, 0x51, 0xfb, 0x7e, 0x4a, 0x30, 0xd0, 0x8c, 0xce, 0x57, 0xa2, 0xfa, 0x88, - 0x4a, 0x5c, 0x7c, 0xbf, 0x52, 0xee, 0xfb, 0x1d, 0x64, 0x60, 0x94, 0xa5, 0x95, 0xa5, 0x5c, 0x05, - 0x1c, 0x29, 0x40, 0xa8, 0x09, 0xe5, 0x28, 0x74, 0x3c, 0x2f, 0x68, 0x54, 0x78, 0x98, 0x3f, 0x5a, - 0x96, 0x1d, 0x86, 0xbd, 0x9e, 0xd5, 0x16, 0x9f, 0xa4, 0x14, 0x85, 0x3d, 0x2f, 0x30, 0xde, 0x42, - 0x0d, 0x47, 0x77, 0xdd, 0x1b, 0x1e, 0x80, 0x01, 0xe5, 0x4b, 0x72, 0x15, 0xc5, 0x44, 0x7e, 0x55, - 0x90, 0x5d, 0x0f, 0x47, 0x77, 0x58, 0xde, 0xa0, 0x5d, 0x28, 0xb9, 0x57, 0xe9, 0x87, 0xc9, 0x8b, - 0x88, 0x0b, 0xc3, 0x85, 0x2a, 0x8e, 0xee, 0x78, 0xa7, 0x44, 0x4f, 0x41, 0x20, 0xe2, 0x84, 0xee, - 0x24, 0x85, 0xbb, 0xc6, 0x39, 0x03, 0x77, 0x42, 0xd0, 0x0b, 0xd0, 0xe2, 0xe8, 0xce, 0x19, 0x73, - 0xf7, 0xa2, 0x6c, 0xb5, 0xd6, 0x56, 0xee, 0x53, 0xa6, 0xc1, 0x61, 0x88, 0x53, 0x32, 0x31, 0xde, + 0x24, 0xa7, 0xbc, 0x41, 0x41, 0xc4, 0x48, 0x42, 0x04, 0x02, 0x34, 0x30, 0xa4, 0xa2, 0x03, 0xa4, + 0x72, 0x80, 0x6c, 0x73, 0x81, 0xec, 0xb3, 0xcd, 0x36, 0xfb, 0x1c, 0x22, 0x07, 0xc8, 0x0d, 0xfe, + 0x9a, 0x07, 0x40, 0x42, 0xf6, 0x6f, 0xcb, 0x7f, 0xd5, 0xbf, 0xf8, 0x37, 0xa8, 0x9e, 0x9e, 0x7e, + 0x7e, 0xd3, 0x33, 0xdd, 0x00, 0xfd, 0xd2, 0x0f, 0x83, 0xe8, 0xda, 0x73, 0xa9, 0xdb, 0x9c, 0xc6, + 0x11, 0x8d, 0x10, 0x2c, 0x38, 0x3b, 0xda, 0x9c, 0xc6, 0xd3, 0xb1, 0xd8, 0xd8, 0xd1, 0x3e, 0xcc, + 0x48, 0x7c, 0x2f, 0x17, 0x75, 0x1a, 0x4d, 0xa3, 0x85, 0x96, 0x71, 0x0a, 0x95, 0xee, 0x8d, 0x1b, + 0x27, 0x84, 0xa2, 0x6d, 0x28, 0x8f, 0x03, 0x9f, 0x84, 0xb4, 0x51, 0xd8, 0x2d, 0xec, 0x97, 0xb0, + 0x5c, 0x21, 0x04, 0xea, 0x38, 0x0a, 0xc3, 0x46, 0x91, 0x73, 0x39, 0xcd, 0x64, 0x13, 0x12, 0xcf, + 0x49, 0xdc, 0x50, 0x84, 0xac, 0x58, 0x19, 0xff, 0x53, 0x60, 0xbd, 0xc3, 0xe3, 0xb0, 0x63, 0x37, + 0x4c, 0xdc, 0x31, 0xf5, 0xa3, 0x10, 0x1d, 0x01, 0x24, 0xd4, 0xa5, 0x64, 0x42, 0x42, 0x9a, 0x34, + 0x0a, 0xbb, 0xca, 0xbe, 0xd6, 0xda, 0x6b, 0x2e, 0x65, 0xf0, 0x91, 0x4a, 0x73, 0x94, 0xca, 0xe3, + 0x25, 0x55, 0xd4, 0x02, 0x8d, 0xcc, 0x49, 0x48, 0x1d, 0x1a, 0xdd, 0x92, 0xb0, 0xa1, 0xee, 0x16, + 0xf6, 0xb5, 0xd6, 0x7a, 0x53, 0x24, 0x68, 0xb2, 0x1d, 0x9b, 0x6d, 0x60, 0x20, 0x19, 0xbd, 0xf3, + 0x9f, 0x22, 0xd4, 0x32, 0x6b, 0xc8, 0x82, 0xea, 0xd8, 0xa5, 0xe4, 0x3a, 0x8a, 0xef, 0x79, 0x9a, + 0xf5, 0xd6, 0xf3, 0x47, 0x06, 0xd2, 0xec, 0x4a, 0x3d, 0x9c, 0x59, 0x40, 0xbf, 0x82, 0xca, 0x58, + 0xa0, 0xc7, 0xd1, 0xd1, 0x5a, 0x1b, 0xcb, 0xc6, 0x24, 0xb0, 0x38, 0x95, 0x41, 0x3a, 0x28, 0xc9, + 0x87, 0x80, 0x43, 0xb6, 0x82, 0x19, 0x69, 0xfc, 0xb3, 0x00, 0xd5, 0xd4, 0x2e, 0xda, 0x80, 0xb5, + 0x8e, 0xe5, 0x9c, 0x0f, 0xb0, 0xd9, 0x1d, 0x1e, 0x0d, 0xfa, 0xef, 0xcd, 0x9e, 0xfe, 0x04, 0xad, + 0x40, 0xb5, 0x63, 0x39, 0x1d, 0xf3, 0xa8, 0x3f, 0xd0, 0x0b, 0x68, 0x15, 0x6a, 0x1d, 0xcb, 0xe9, + 0x0e, 0x4f, 0x4f, 0xfb, 0xb6, 0x5e, 0x44, 0x6b, 0xa0, 0x75, 0x2c, 0x07, 0x0f, 0x2d, 0xab, 0xd3, + 0xee, 0x9e, 0xe8, 0x0a, 0xda, 0x82, 0xf5, 0x8e, 0xe5, 0xf4, 0x4e, 0x2d, 0xa7, 0x67, 0x9e, 0x61, + 0xb3, 0xdb, 0xb6, 0xcd, 0x9e, 0xae, 0x22, 0x80, 0x32, 0x63, 0xf7, 0x2c, 0xbd, 0x24, 0xe9, 0x91, + 0x69, 0xeb, 0x65, 0x69, 0xae, 0x3f, 0x18, 0x99, 0xd8, 0xd6, 0x2b, 0x72, 0x79, 0x7e, 0xd6, 0x6b, + 0xdb, 0xa6, 0x5e, 0x95, 0xcb, 0x9e, 0x69, 0x99, 0xb6, 0xa9, 0xd7, 0x8e, 0xd5, 0x6a, 0x51, 0x57, + 0x8e, 0xd5, 0xaa, 0xa2, 0xab, 0xc6, 0xdf, 0x0b, 0xb0, 0x35, 0xa2, 0x31, 0x71, 0x27, 0x27, 0xe4, + 0x1e, 0xbb, 0xe1, 0x35, 0xc1, 0xe4, 0xc3, 0x8c, 0x24, 0x14, 0xed, 0x40, 0x75, 0x1a, 0x25, 0x3e, + 0xc3, 0x8e, 0x03, 0x5c, 0xc3, 0xd9, 0x1a, 0x1d, 0x42, 0xed, 0x96, 0xdc, 0x3b, 0x31, 0x93, 0x97, + 0x80, 0xa1, 0x66, 0x56, 0x90, 0x99, 0xa5, 0xea, 0xad, 0xa4, 0x96, 0xf1, 0x55, 0xbe, 0x8c, 0xaf, + 0x71, 0x05, 0xdb, 0x0f, 0x83, 0x4a, 0xa6, 0x51, 0x98, 0x10, 0x64, 0x01, 0x12, 0x8a, 0x0e, 0x5d, + 0x9c, 0x2d, 0x8f, 0x4f, 0x6b, 0x3d, 0xfd, 0x6c, 0x01, 0xe0, 0xf5, 0xcb, 0x87, 0x2c, 0xe3, 0x2f, + 0xb0, 0x21, 0xfc, 0xd8, 0xee, 0x65, 0x40, 0x92, 0xc7, 0xa4, 0xbe, 0x0d, 0x65, 0xca, 0x85, 0x1b, + 0xc5, 0x5d, 0x65, 0xbf, 0x86, 0xe5, 0xea, 0x6b, 0x33, 0xf4, 0x60, 0x33, 0xef, 0xf9, 0x7b, 0xc9, + 0xef, 0xb7, 0xa0, 0xe2, 0x59, 0x40, 0xd0, 0x26, 0x94, 0x26, 0x2e, 0x1d, 0xdf, 0xc8, 0x6c, 0xc4, + 0x82, 0xa5, 0x72, 0xe5, 0x07, 0x94, 0xc4, 0xfc, 0x08, 0x6b, 0x58, 0xae, 0x8c, 0xe7, 0x50, 0x7e, + 0xcd, 0x29, 0xf4, 0x0b, 0x28, 0xc5, 0x33, 0x96, 0xab, 0xb8, 0xea, 0xfa, 0x72, 0x00, 0xcc, 0x30, + 0x16, 0xdb, 0xc6, 0x3f, 0x8a, 0xb0, 0x22, 0x02, 0x1a, 0x45, 0xb3, 0x78, 0x4c, 0x18, 0x82, 0xb7, + 0xe4, 0x3e, 0x99, 0xba, 0x63, 0x92, 0x22, 0x98, 0xae, 0x59, 0x30, 0xc9, 0x8d, 0x1b, 0x7b, 0xd2, + 0xab, 0x58, 0xa0, 0xdf, 0x81, 0xc6, 0x91, 0xa4, 0x0e, 0xbd, 0x9f, 0x12, 0x8e, 0x61, 0xbd, 0xb5, + 0xb9, 0x28, 0x2a, 0x8e, 0x13, 0xb5, 0xef, 0xa7, 0x04, 0x03, 0xcd, 0xe8, 0x7c, 0x25, 0xaa, 0x8f, + 0xa8, 0xc4, 0xc5, 0xf9, 0x95, 0x72, 0xe7, 0x77, 0x90, 0x81, 0x51, 0x96, 0x56, 0x96, 0x72, 0x15, + 0x70, 0xa4, 0x00, 0xa1, 0x26, 0x94, 0xa3, 0xd0, 0xf1, 0xbc, 0xa0, 0x51, 0xe1, 0x61, 0xfe, 0x68, + 0x59, 0x76, 0x18, 0xf6, 0x7a, 0x56, 0x5b, 0x1c, 0x49, 0x29, 0x0a, 0x7b, 0x5e, 0x60, 0xbc, 0x85, + 0x1a, 0x8e, 0xee, 0xba, 0x37, 0x3c, 0x00, 0x03, 0xca, 0x97, 0xe4, 0x2a, 0x8a, 0x89, 0x3c, 0x55, + 0x90, 0xaf, 0x1e, 0x8e, 0xee, 0xb0, 0xdc, 0x41, 0xbb, 0x50, 0x72, 0xaf, 0xd2, 0x83, 0xc9, 0x8b, + 0x88, 0x0d, 0xc3, 0x85, 0x2a, 0x8e, 0xee, 0xf8, 0x4b, 0x89, 0x9e, 0x82, 0x40, 0xc4, 0x09, 0xdd, + 0x49, 0x0a, 0x77, 0x8d, 0x73, 0x06, 0xee, 0x84, 0xa0, 0x17, 0xa0, 0xc5, 0xd1, 0x9d, 0x33, 0xe6, + 0xee, 0x45, 0xd9, 0x6a, 0xad, 0xad, 0xdc, 0x51, 0xa6, 0xc1, 0x61, 0x88, 0x53, 0x32, 0x31, 0xde, 0x02, 0xbc, 0xf6, 0x49, 0xe0, 0x3d, 0xca, 0xc9, 0xcf, 0x19, 0x7c, 0x24, 0xf0, 0x52, 0xfb, 0x2b, - 0x32, 0x64, 0x6e, 0x01, 0xcb, 0x3b, 0x06, 0xc4, 0x88, 0x7d, 0xed, 0x23, 0xea, 0x7b, 0xdf, 0xa1, - 0x46, 0x10, 0xa8, 0xd7, 0xd4, 0xf7, 0x78, 0x71, 0xd4, 0x30, 0xa7, 0x8d, 0x57, 0x50, 0xba, 0xe0, - 0xe6, 0x5e, 0x80, 0xc6, 0xa5, 0x1c, 0xc6, 0x4e, 0x2b, 0x36, 0x97, 0x66, 0xe6, 0x1a, 0x43, 0x92, - 0x92, 0x89, 0xd1, 0x86, 0xd5, 0x13, 0xe9, 0x96, 0x0b, 0x7c, 0x7d, 0x5c, 0xc6, 0xbf, 0x8a, 0x50, - 0x39, 0x8e, 0x66, 0x71, 0xe8, 0x06, 0xa8, 0x0e, 0x45, 0xdf, 0xe3, 0x7a, 0x0a, 0x2e, 0xfa, 0x1e, - 0xfa, 0x23, 0xd4, 0x27, 0xfe, 0x75, 0xec, 0xb2, 0x7a, 0x10, 0xa5, 0x5d, 0xe4, 0x35, 0xf3, 0xe3, - 0xe5, 0xc8, 0x4e, 0x53, 0x09, 0x5e, 0xdf, 0xab, 0x93, 0xe5, 0xe3, 0x52, 0xc5, 0x2a, 0xb9, 0x8a, - 0x7d, 0x06, 0xf5, 0x20, 0x1a, 0xbb, 0x81, 0x93, 0xf5, 0x2a, 0x95, 0x07, 0xb5, 0xca, 0xb9, 0x67, - 0x69, 0xc3, 0x7a, 0x80, 0x4b, 0xe9, 0x91, 0xb8, 0xa0, 0x97, 0xb0, 0x32, 0x75, 0x63, 0xea, 0x8f, - 0xfd, 0xa9, 0xcb, 0xa6, 0x7d, 0x99, 0x2b, 0xe6, 0xc2, 0xce, 0xe1, 0x86, 0x73, 0xe2, 0xe8, 0x67, - 0xb0, 0x12, 0x93, 0x39, 0x89, 0x13, 0xe2, 0x39, 0xcc, 0x6f, 0x65, 0x57, 0xd9, 0x57, 0xb0, 0x96, - 0xf2, 0xfa, 0x5e, 0x62, 0xfc, 0xaf, 0x08, 0xe5, 0x0b, 0x51, 0x5d, 0x07, 0xa0, 0x72, 0x6c, 0xc4, - 0x24, 0xdf, 0x5e, 0x76, 0x22, 0x24, 0x38, 0x30, 0x5c, 0x06, 0xfd, 0x04, 0x6a, 0xd4, 0x9f, 0x90, - 0x84, 0xba, 0x93, 0x29, 0x07, 0x53, 0xc1, 0x0b, 0xc6, 0xa7, 0x6a, 0x84, 0x8d, 0x6b, 0xf6, 0x58, - 0x05, 0x3c, 0x8c, 0x44, 0xbf, 0x86, 0x1a, 0x7b, 0x13, 0x7c, 0xbb, 0x68, 0x94, 0xf8, 0x23, 0xdb, - 0x7c, 0xf0, 0x22, 0xb8, 0x5b, 0x5c, 0x8d, 0xd3, 0x57, 0xf6, 0x7b, 0xd0, 0x78, 0x15, 0x4b, 0x25, - 0xd1, 0x25, 0xb6, 0xf3, 0x5d, 0x22, 0x7d, 0x2d, 0x18, 0xae, 0x16, 0x2f, 0x67, 0x0f, 0x4a, 0x73, - 0x1e, 0x52, 0x45, 0x6e, 0x39, 0xcb, 0xc9, 0x71, 0xd8, 0xc5, 0x3d, 0x1b, 0x21, 0x7f, 0x16, 0x55, - 0xd4, 0xa8, 0x7e, 0x3c, 0x42, 0x64, 0x81, 0xe1, 0x54, 0x86, 0x21, 0x3c, 0x9e, 0xc5, 0x31, 0xdf, - 0xa2, 0xfc, 0x09, 0x69, 0x6c, 0x72, 0x28, 0x34, 0xc9, 0xb3, 0xfd, 0x09, 0x31, 0xfe, 0x56, 0x84, - 0xfa, 0x85, 0x98, 0x33, 0xe9, 0x6c, 0x7b, 0x05, 0x1b, 0xe4, 0xea, 0x8a, 0x8c, 0xa9, 0x3f, 0x27, - 0xce, 0xd8, 0x0d, 0x02, 0x12, 0x3b, 0xb2, 0x60, 0xb5, 0xd6, 0x5a, 0x53, 0xec, 0x9b, 0x5d, 0xce, - 0xef, 0xf7, 0xf0, 0x7a, 0x26, 0x2b, 0x59, 0x1e, 0x32, 0x61, 0xc3, 0x9f, 0x4c, 0x88, 0xe7, 0xbb, - 0x74, 0xd9, 0x80, 0xe8, 0x54, 0x5b, 0xf2, 0xd9, 0x5f, 0xd8, 0x47, 0x2e, 0x25, 0x0b, 0x33, 0x99, - 0x46, 0x66, 0xe6, 0x19, 0xab, 0xea, 0xf8, 0x3a, 0x1b, 0x97, 0xab, 0x52, 0xd3, 0xe6, 0x4c, 0x2c, - 0x2f, 0x73, 0xa3, 0x58, 0x7d, 0x30, 0x8a, 0x17, 0x2d, 0xbb, 0xf4, 0xa5, 0x96, 0x6d, 0xbc, 0x84, - 0xb5, 0x0c, 0x08, 0x39, 0x6a, 0x0f, 0xa0, 0xcc, 0x3f, 0x65, 0xda, 0x2b, 0xd0, 0xc7, 0x55, 0x87, - 0xa5, 0x84, 0xf1, 0xd7, 0x22, 0xa0, 0x54, 0x3f, 0xba, 0x4b, 0x7e, 0xa0, 0x60, 0x6e, 0x42, 0x89, - 0xf3, 0x25, 0x92, 0xe2, 0xc0, 0x70, 0x08, 0xdc, 0x84, 0x4e, 0x6f, 0x33, 0x18, 0x85, 0xf2, 0x5b, - 0xf6, 0x8b, 0x49, 0x32, 0x0b, 0x28, 0x96, 0x12, 0xc6, 0xbf, 0x0b, 0xb0, 0x91, 0xc3, 0x41, 0x62, - 0xb9, 0x68, 0xff, 0x85, 0x6f, 0x6f, 0xff, 0x68, 0x1f, 0xaa, 0xd3, 0xdb, 0xcf, 0x8c, 0x89, 0xec, - 0xf6, 0x93, 0xaf, 0xf8, 0xa7, 0xa0, 0xc6, 0xd1, 0x5d, 0xd2, 0x50, 0xb9, 0xe6, 0xf2, 0x4c, 0xe4, - 0x7c, 0x36, 0x58, 0x73, 0x79, 0xe4, 0x06, 0xab, 0xb8, 0x39, 0xf8, 0x03, 0x68, 0x4b, 0xf3, 0x99, - 0xad, 0xd0, 0xfd, 0xa3, 0xc1, 0x10, 0x9b, 0xfa, 0x13, 0x54, 0x05, 0x75, 0x64, 0x0f, 0xcf, 0xf4, - 0x02, 0xa3, 0xcc, 0x3f, 0x99, 0x5d, 0xb1, 0x96, 0x33, 0xca, 0x91, 0x42, 0xca, 0xc1, 0x7f, 0x0b, - 0x00, 0x8b, 0x86, 0x84, 0x34, 0xa8, 0x9c, 0x0f, 0x4e, 0x06, 0xc3, 0x77, 0x03, 0x61, 0xe0, 0xc8, - 0xee, 0xf7, 0xf4, 0x02, 0xaa, 0x41, 0x49, 0xec, 0xf9, 0x45, 0xe6, 0x41, 0x2e, 0xf9, 0x0a, 0xfb, - 0x07, 0x90, 0x6d, 0xf8, 0x2a, 0xaa, 0x80, 0x92, 0xed, 0xf1, 0x72, 0x71, 0x2f, 0x33, 0x83, 0xd8, - 0x3c, 0xb3, 0xda, 0x5d, 0x53, 0xaf, 0xb0, 0x8b, 0x6c, 0x85, 0x07, 0x28, 0xa7, 0xfb, 0x3b, 0xd3, - 0x64, 0x5b, 0x3f, 0x30, 0x3f, 0x43, 0xfb, 0x8d, 0x89, 0x75, 0x8d, 0xf1, 0xf0, 0xf0, 0x9d, 0xbe, - 0xc2, 0x78, 0xaf, 0xfb, 0xa6, 0xd5, 0xd3, 0x57, 0xd9, 0xda, 0xff, 0xc6, 0x6c, 0x63, 0xbb, 0x63, - 0xb6, 0x6d, 0xbd, 0xce, 0x6e, 0x2e, 0x78, 0x80, 0x6b, 0xcc, 0xcd, 0xf1, 0xf0, 0x1c, 0x0f, 0xda, - 0x96, 0xae, 0x1f, 0xec, 0xc1, 0x6a, 0x6e, 0xfe, 0x30, 0x5f, 0x76, 0xbb, 0x63, 0x99, 0x23, 0xfd, - 0x09, 0xa3, 0x47, 0x6f, 0xda, 0xb8, 0x37, 0xd2, 0x0b, 0x9d, 0x5f, 0xbe, 0xdf, 0x9b, 0xfb, 0x94, - 0x24, 0x49, 0xd3, 0x8f, 0x0e, 0x05, 0x75, 0x78, 0x1d, 0x1d, 0xce, 0xe9, 0x21, 0xff, 0x0b, 0x7a, - 0xb8, 0x78, 0x3e, 0x97, 0x65, 0xce, 0xf9, 0xcd, 0x37, 0x01, 0x00, 0x00, 0xff, 0xff, 0x2e, 0xb4, - 0x72, 0xde, 0xde, 0x0e, 0x00, 0x00, + 0x32, 0x64, 0x6e, 0x01, 0xcb, 0x3d, 0x06, 0xc4, 0x88, 0x9d, 0xf6, 0x11, 0xf5, 0xbd, 0xef, 0x50, + 0x23, 0x08, 0xd4, 0x6b, 0xea, 0x7b, 0xbc, 0x38, 0x6a, 0x98, 0xd3, 0xc6, 0x2b, 0x28, 0x5d, 0x70, + 0x73, 0x2f, 0x40, 0xe3, 0x52, 0x0e, 0x63, 0xa7, 0x15, 0x9b, 0x4b, 0x33, 0x73, 0x8d, 0x21, 0x49, + 0xc9, 0xc4, 0x68, 0xc3, 0xea, 0x89, 0x74, 0xcb, 0x05, 0xbe, 0x3e, 0x2e, 0xe3, 0x5f, 0x45, 0xa8, + 0x1c, 0x47, 0xb3, 0x38, 0x74, 0x03, 0x54, 0x87, 0xa2, 0xef, 0x71, 0x3d, 0x05, 0x17, 0x7d, 0x0f, + 0xfd, 0x11, 0xea, 0x13, 0xff, 0x3a, 0x76, 0x59, 0x3d, 0x88, 0xd2, 0x2e, 0xf2, 0x9a, 0xf9, 0xf1, + 0x72, 0x64, 0xa7, 0xa9, 0x04, 0xaf, 0xef, 0xd5, 0xc9, 0xf2, 0x72, 0xa9, 0x62, 0x95, 0x5c, 0xc5, + 0x3e, 0x83, 0x7a, 0x10, 0x8d, 0xdd, 0xc0, 0xc9, 0xde, 0x2a, 0x95, 0x07, 0xb5, 0xca, 0xb9, 0x67, + 0xe9, 0x83, 0xf5, 0x00, 0x97, 0xd2, 0x23, 0x71, 0x41, 0x2f, 0x61, 0x65, 0xea, 0xc6, 0xd4, 0x1f, + 0xfb, 0x53, 0x97, 0x75, 0xfb, 0x32, 0x57, 0xcc, 0x85, 0x9d, 0xc3, 0x0d, 0xe7, 0xc4, 0xd1, 0xcf, + 0x60, 0x25, 0x26, 0x73, 0x12, 0x27, 0xc4, 0x73, 0x98, 0xdf, 0xca, 0xae, 0xb2, 0xaf, 0x60, 0x2d, + 0xe5, 0xf5, 0xbd, 0xc4, 0xf8, 0x7f, 0x11, 0xca, 0x17, 0xa2, 0xba, 0x0e, 0x40, 0xe5, 0xd8, 0x88, + 0x4e, 0xbe, 0xbd, 0xec, 0x44, 0x48, 0x70, 0x60, 0xb8, 0x0c, 0xfa, 0x09, 0xd4, 0xa8, 0x3f, 0x21, + 0x09, 0x75, 0x27, 0x53, 0x0e, 0xa6, 0x82, 0x17, 0x8c, 0x4f, 0xd5, 0x08, 0x6b, 0xd7, 0xec, 0xb2, + 0x0a, 0x78, 0x18, 0x89, 0x7e, 0x0d, 0x35, 0x76, 0x27, 0xf8, 0x74, 0xd1, 0x28, 0xf1, 0x4b, 0xb6, + 0xf9, 0xe0, 0x46, 0x70, 0xb7, 0xb8, 0x1a, 0xa7, 0xb7, 0xec, 0xf7, 0xa0, 0xf1, 0x2a, 0x96, 0x4a, + 0xe2, 0x95, 0xd8, 0xce, 0xbf, 0x12, 0xe9, 0x6d, 0xc1, 0x70, 0xb5, 0xb8, 0x39, 0x7b, 0x50, 0x9a, + 0xf3, 0x90, 0x2a, 0x72, 0xca, 0x59, 0x4e, 0x8e, 0xc3, 0x2e, 0xf6, 0x59, 0x0b, 0xf9, 0xb3, 0xa8, + 0xa2, 0x46, 0xf5, 0xe3, 0x16, 0x22, 0x0b, 0x0c, 0xa7, 0x32, 0x3c, 0xab, 0x49, 0xd0, 0xa8, 0xc9, + 0xac, 0x26, 0x01, 0xc3, 0x7c, 0x3c, 0x8b, 0x63, 0x3e, 0x57, 0xf9, 0x13, 0xd2, 0xd8, 0xe4, 0xe0, + 0x68, 0x92, 0x67, 0xfb, 0x13, 0x62, 0xfc, 0xad, 0x08, 0xf5, 0x0b, 0xd1, 0x79, 0xd2, 0x6e, 0xf7, + 0x0a, 0x36, 0xc8, 0xd5, 0x15, 0x19, 0x53, 0x7f, 0x4e, 0x9c, 0xb1, 0x1b, 0x04, 0x24, 0x76, 0x64, + 0x09, 0x6b, 0xad, 0xb5, 0xa6, 0x98, 0x40, 0xbb, 0x9c, 0xdf, 0xef, 0xe1, 0xf5, 0x4c, 0x56, 0xb2, + 0x3c, 0x64, 0xc2, 0x86, 0x3f, 0x99, 0x10, 0xcf, 0x77, 0xe9, 0xb2, 0x01, 0xf1, 0x76, 0x6d, 0xc9, + 0x87, 0xe0, 0xc2, 0x3e, 0x72, 0x29, 0x59, 0x98, 0xc9, 0x34, 0x32, 0x33, 0xcf, 0x58, 0x9d, 0xc7, + 0xd7, 0x59, 0x03, 0x5d, 0x95, 0x9a, 0x36, 0x67, 0x62, 0xb9, 0x99, 0x6b, 0xce, 0xea, 0x83, 0xe6, + 0xbc, 0x78, 0xc4, 0x4b, 0x5f, 0x7a, 0xc4, 0x8d, 0x97, 0xb0, 0x96, 0x01, 0x21, 0x9b, 0xef, 0x01, + 0x94, 0xf9, 0xe1, 0xa6, 0xaf, 0x07, 0xfa, 0xb8, 0x0e, 0xb1, 0x94, 0x30, 0xfe, 0x5a, 0x04, 0x94, + 0xea, 0x47, 0x77, 0xc9, 0x0f, 0x14, 0xcc, 0x4d, 0x28, 0x71, 0xbe, 0x44, 0x52, 0x2c, 0x18, 0x0e, + 0x81, 0x9b, 0xd0, 0xe9, 0x6d, 0x06, 0xa3, 0x50, 0x7e, 0xcb, 0xbe, 0x98, 0x24, 0xb3, 0x80, 0x62, + 0x29, 0x61, 0xfc, 0xbb, 0x00, 0x1b, 0x39, 0x1c, 0x24, 0x96, 0x8b, 0x86, 0x50, 0xf8, 0xf6, 0x86, + 0x80, 0xf6, 0xa1, 0x3a, 0xbd, 0xfd, 0x4c, 0xe3, 0xc8, 0x76, 0x3f, 0x79, 0xaf, 0x7f, 0x0a, 0x6a, + 0x1c, 0xdd, 0x25, 0x0d, 0x95, 0x6b, 0x2e, 0x77, 0x49, 0xce, 0x67, 0xad, 0x36, 0x97, 0x47, 0xae, + 0xd5, 0x8a, 0x9d, 0x83, 0x3f, 0x80, 0xb6, 0xd4, 0xb1, 0xd9, 0x50, 0xdd, 0x3f, 0x1a, 0x0c, 0xb1, + 0xa9, 0x3f, 0x41, 0x55, 0x50, 0x47, 0xf6, 0xf0, 0x4c, 0x2f, 0x30, 0xca, 0xfc, 0x93, 0xd9, 0x15, + 0x83, 0x3a, 0xa3, 0x1c, 0x29, 0xa4, 0x1c, 0xfc, 0xb7, 0x00, 0xb0, 0x78, 0xa2, 0x90, 0x06, 0x95, + 0xf3, 0xc1, 0xc9, 0x60, 0xf8, 0x6e, 0x20, 0x0c, 0x1c, 0xd9, 0xfd, 0x9e, 0x5e, 0x40, 0x35, 0x28, + 0x89, 0xc9, 0xbf, 0xc8, 0x3c, 0xc8, 0xb1, 0x5f, 0x61, 0xff, 0x04, 0xd9, 0xcc, 0xaf, 0xa2, 0x0a, + 0x28, 0xd9, 0x64, 0x2f, 0x47, 0xf9, 0x32, 0x33, 0x88, 0xcd, 0x33, 0xab, 0xdd, 0x35, 0xf5, 0x0a, + 0xdb, 0xc8, 0x86, 0x7a, 0x80, 0x72, 0x3a, 0xd1, 0x33, 0x4d, 0xf6, 0x1f, 0x00, 0xcc, 0xcf, 0xd0, + 0x7e, 0x63, 0x62, 0x5d, 0x63, 0x3c, 0x3c, 0x7c, 0xa7, 0xaf, 0x30, 0xde, 0xeb, 0xbe, 0x69, 0xf5, + 0xf4, 0x55, 0xf6, 0x23, 0xf0, 0xc6, 0x6c, 0x63, 0xbb, 0x63, 0xb6, 0x6d, 0xbd, 0xce, 0x76, 0x2e, + 0x78, 0x80, 0x6b, 0xcc, 0xcd, 0xf1, 0xf0, 0x1c, 0x0f, 0xda, 0x96, 0xae, 0x1f, 0xec, 0xc1, 0x6a, + 0xae, 0x23, 0x31, 0x5f, 0x76, 0xbb, 0x63, 0x99, 0x23, 0xfd, 0x09, 0xa3, 0x47, 0x6f, 0xda, 0xb8, + 0x37, 0xd2, 0x0b, 0x9d, 0x5f, 0xbe, 0xdf, 0x9b, 0xfb, 0x94, 0x24, 0x49, 0xd3, 0x8f, 0x0e, 0x05, + 0x75, 0x78, 0x1d, 0x1d, 0xce, 0xe9, 0x21, 0xff, 0x29, 0x3d, 0x5c, 0x5c, 0x9f, 0xcb, 0x32, 0xe7, + 0xfc, 0xe6, 0x9b, 0x00, 0x00, 0x00, 0xff, 0xff, 0x51, 0x1b, 0x15, 0x13, 0xf0, 0x0e, 0x00, 0x00, } diff --git a/proto/binlogdata.proto b/proto/binlogdata.proto index b28dbc5356f..ba456d43875 100644 --- a/proto/binlogdata.proto +++ b/proto/binlogdata.proto @@ -248,6 +248,7 @@ message VEvent { FieldEvent field_event = 6; VGtid vgtid = 7; Journal journal = 8; + string dml = 9; // current_time specifies the current time to handle clock skew. int64 current_time = 20; } diff --git a/py/vtproto/binlogdata_pb2.py b/py/vtproto/binlogdata_pb2.py index 53eda7da5b1..40082fc73ff 100644 --- a/py/vtproto/binlogdata_pb2.py +++ b/py/vtproto/binlogdata_pb2.py @@ -23,7 +23,7 @@ package='binlogdata', syntax='proto3', serialized_options=_b('Z\'vitess.io/vitess/go/vt/proto/binlogdata'), - serialized_pb=_b('\n\x10\x62inlogdata.proto\x12\nbinlogdata\x1a\x0bvtrpc.proto\x1a\x0bquery.proto\x1a\x0etopodata.proto\"7\n\x07\x43harset\x12\x0e\n\x06\x63lient\x18\x01 \x01(\x05\x12\x0c\n\x04\x63onn\x18\x02 \x01(\x05\x12\x0e\n\x06server\x18\x03 \x01(\x05\"\xb5\x03\n\x11\x42inlogTransaction\x12;\n\nstatements\x18\x01 \x03(\x0b\x32\'.binlogdata.BinlogTransaction.Statement\x12&\n\x0b\x65vent_token\x18\x04 \x01(\x0b\x32\x11.query.EventToken\x1a\xae\x02\n\tStatement\x12\x42\n\x08\x63\x61tegory\x18\x01 \x01(\x0e\x32\x30.binlogdata.BinlogTransaction.Statement.Category\x12$\n\x07\x63harset\x18\x02 \x01(\x0b\x32\x13.binlogdata.Charset\x12\x0b\n\x03sql\x18\x03 \x01(\x0c\"\xa9\x01\n\x08\x43\x61tegory\x12\x13\n\x0f\x42L_UNRECOGNIZED\x10\x00\x12\x0c\n\x08\x42L_BEGIN\x10\x01\x12\r\n\tBL_COMMIT\x10\x02\x12\x0f\n\x0b\x42L_ROLLBACK\x10\x03\x12\x15\n\x11\x42L_DML_DEPRECATED\x10\x04\x12\n\n\x06\x42L_DDL\x10\x05\x12\n\n\x06\x42L_SET\x10\x06\x12\r\n\tBL_INSERT\x10\x07\x12\r\n\tBL_UPDATE\x10\x08\x12\r\n\tBL_DELETE\x10\tJ\x04\x08\x02\x10\x03J\x04\x08\x03\x10\x04\"v\n\x15StreamKeyRangeRequest\x12\x10\n\x08position\x18\x01 \x01(\t\x12%\n\tkey_range\x18\x02 \x01(\x0b\x32\x12.topodata.KeyRange\x12$\n\x07\x63harset\x18\x03 \x01(\x0b\x32\x13.binlogdata.Charset\"S\n\x16StreamKeyRangeResponse\x12\x39\n\x12\x62inlog_transaction\x18\x01 \x01(\x0b\x32\x1d.binlogdata.BinlogTransaction\"]\n\x13StreamTablesRequest\x12\x10\n\x08position\x18\x01 \x01(\t\x12\x0e\n\x06tables\x18\x02 \x03(\t\x12$\n\x07\x63harset\x18\x03 \x01(\x0b\x32\x13.binlogdata.Charset\"Q\n\x14StreamTablesResponse\x12\x39\n\x12\x62inlog_transaction\x18\x01 \x01(\x0b\x32\x1d.binlogdata.BinlogTransaction\"%\n\x04Rule\x12\r\n\x05match\x18\x01 \x01(\t\x12\x0e\n\x06\x66ilter\x18\x02 \x01(\t\")\n\x06\x46ilter\x12\x1f\n\x05rules\x18\x01 \x03(\x0b\x32\x10.binlogdata.Rule\"\xde\x01\n\x0c\x42inlogSource\x12\x10\n\x08keyspace\x18\x01 \x01(\t\x12\r\n\x05shard\x18\x02 \x01(\t\x12)\n\x0btablet_type\x18\x03 \x01(\x0e\x32\x14.topodata.TabletType\x12%\n\tkey_range\x18\x04 \x01(\x0b\x32\x12.topodata.KeyRange\x12\x0e\n\x06tables\x18\x05 \x03(\t\x12\"\n\x06\x66ilter\x18\x06 \x01(\x0b\x32\x12.binlogdata.Filter\x12\'\n\x06on_ddl\x18\x07 \x01(\x0e\x32\x17.binlogdata.OnDDLAction\"B\n\tRowChange\x12\x1a\n\x06\x62\x65\x66ore\x18\x01 \x01(\x0b\x32\n.query.Row\x12\x19\n\x05\x61\x66ter\x18\x02 \x01(\x0b\x32\n.query.Row\"J\n\x08RowEvent\x12\x12\n\ntable_name\x18\x01 \x01(\t\x12*\n\x0brow_changes\x18\x02 \x03(\x0b\x32\x15.binlogdata.RowChange\">\n\nFieldEvent\x12\x12\n\ntable_name\x18\x01 \x01(\t\x12\x1c\n\x06\x66ields\x18\x02 \x03(\x0b\x32\x0c.query.Field\":\n\tShardGtid\x12\x10\n\x08keyspace\x18\x01 \x01(\t\x12\r\n\x05shard\x18\x02 \x01(\t\x12\x0c\n\x04gtid\x18\x03 \x01(\t\"3\n\x05VGtid\x12*\n\x0bshard_gtids\x18\x01 \x03(\x0b\x32\x15.binlogdata.ShardGtid\"0\n\rKeyspaceShard\x12\x10\n\x08keyspace\x18\x01 \x01(\t\x12\r\n\x05shard\x18\x02 \x01(\t\"\xe3\x01\n\x07Journal\x12\n\n\x02id\x18\x01 \x01(\x03\x12\x31\n\x0emigration_type\x18\x02 \x01(\x0e\x32\x19.binlogdata.MigrationType\x12\x0e\n\x06tables\x18\x03 \x03(\t\x12\x16\n\x0elocal_position\x18\x04 \x01(\t\x12*\n\x0bshard_gtids\x18\x05 \x03(\x0b\x32\x15.binlogdata.ShardGtid\x12/\n\x0cparticipants\x18\x06 \x03(\x0b\x32\x19.binlogdata.KeyspaceShard\x12\x14\n\x0creversed_ids\x18\x07 \x03(\x03\"\x90\x02\n\x06VEvent\x12$\n\x04type\x18\x01 \x01(\x0e\x32\x16.binlogdata.VEventType\x12\x11\n\ttimestamp\x18\x02 \x01(\x03\x12\x0c\n\x04gtid\x18\x03 \x01(\t\x12\x0b\n\x03\x64\x64l\x18\x04 \x01(\t\x12\'\n\trow_event\x18\x05 \x01(\x0b\x32\x14.binlogdata.RowEvent\x12+\n\x0b\x66ield_event\x18\x06 \x01(\x0b\x32\x16.binlogdata.FieldEvent\x12 \n\x05vgtid\x18\x07 \x01(\x0b\x32\x11.binlogdata.VGtid\x12$\n\x07journal\x18\x08 \x01(\x0b\x32\x13.binlogdata.Journal\x12\x14\n\x0c\x63urrent_time\x18\x14 \x01(\x03\"\xc7\x01\n\x0eVStreamRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x1d\n\x06target\x18\x03 \x01(\x0b\x32\r.query.Target\x12\x10\n\x08position\x18\x04 \x01(\t\x12\"\n\x06\x66ilter\x18\x05 \x01(\x0b\x32\x12.binlogdata.Filter\"5\n\x0fVStreamResponse\x12\"\n\x06\x65vents\x18\x01 \x03(\x0b\x32\x12.binlogdata.VEvent\"\xc8\x01\n\x12VStreamRowsRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x1d\n\x06target\x18\x03 \x01(\x0b\x32\r.query.Target\x12\r\n\x05query\x18\x04 \x01(\t\x12\"\n\x06lastpk\x18\x05 \x01(\x0b\x32\x12.query.QueryResult\"\x97\x01\n\x13VStreamRowsResponse\x12\x1c\n\x06\x66ields\x18\x01 \x03(\x0b\x32\x0c.query.Field\x12\x1e\n\x08pkfields\x18\x02 \x03(\x0b\x32\x0c.query.Field\x12\x0c\n\x04gtid\x18\x03 \x01(\t\x12\x18\n\x04rows\x18\x04 \x03(\x0b\x32\n.query.Row\x12\x1a\n\x06lastpk\x18\x05 \x01(\x0b\x32\n.query.Row*>\n\x0bOnDDLAction\x12\n\n\x06IGNORE\x10\x00\x12\x08\n\x04STOP\x10\x01\x12\x08\n\x04\x45XEC\x10\x02\x12\x0f\n\x0b\x45XEC_IGNORE\x10\x03*\xd1\x01\n\nVEventType\x12\x0b\n\x07UNKNOWN\x10\x00\x12\x08\n\x04GTID\x10\x01\x12\t\n\x05\x42\x45GIN\x10\x02\x12\n\n\x06\x43OMMIT\x10\x03\x12\x0c\n\x08ROLLBACK\x10\x04\x12\x07\n\x03\x44\x44L\x10\x05\x12\n\n\x06INSERT\x10\x06\x12\x0b\n\x07REPLACE\x10\x07\x12\n\n\x06UPDATE\x10\x08\x12\n\n\x06\x44\x45LETE\x10\t\x12\x07\n\x03SET\x10\n\x12\t\n\x05OTHER\x10\x0b\x12\x07\n\x03ROW\x10\x0c\x12\t\n\x05\x46IELD\x10\r\x12\r\n\tHEARTBEAT\x10\x0e\x12\t\n\x05VGTID\x10\x0f\x12\x0b\n\x07JOURNAL\x10\x10*\'\n\rMigrationType\x12\n\n\x06TABLES\x10\x00\x12\n\n\x06SHARDS\x10\x01\x42)Z\'vitess.io/vitess/go/vt/proto/binlogdatab\x06proto3') + serialized_pb=_b('\n\x10\x62inlogdata.proto\x12\nbinlogdata\x1a\x0bvtrpc.proto\x1a\x0bquery.proto\x1a\x0etopodata.proto\"7\n\x07\x43harset\x12\x0e\n\x06\x63lient\x18\x01 \x01(\x05\x12\x0c\n\x04\x63onn\x18\x02 \x01(\x05\x12\x0e\n\x06server\x18\x03 \x01(\x05\"\xb5\x03\n\x11\x42inlogTransaction\x12;\n\nstatements\x18\x01 \x03(\x0b\x32\'.binlogdata.BinlogTransaction.Statement\x12&\n\x0b\x65vent_token\x18\x04 \x01(\x0b\x32\x11.query.EventToken\x1a\xae\x02\n\tStatement\x12\x42\n\x08\x63\x61tegory\x18\x01 \x01(\x0e\x32\x30.binlogdata.BinlogTransaction.Statement.Category\x12$\n\x07\x63harset\x18\x02 \x01(\x0b\x32\x13.binlogdata.Charset\x12\x0b\n\x03sql\x18\x03 \x01(\x0c\"\xa9\x01\n\x08\x43\x61tegory\x12\x13\n\x0f\x42L_UNRECOGNIZED\x10\x00\x12\x0c\n\x08\x42L_BEGIN\x10\x01\x12\r\n\tBL_COMMIT\x10\x02\x12\x0f\n\x0b\x42L_ROLLBACK\x10\x03\x12\x15\n\x11\x42L_DML_DEPRECATED\x10\x04\x12\n\n\x06\x42L_DDL\x10\x05\x12\n\n\x06\x42L_SET\x10\x06\x12\r\n\tBL_INSERT\x10\x07\x12\r\n\tBL_UPDATE\x10\x08\x12\r\n\tBL_DELETE\x10\tJ\x04\x08\x02\x10\x03J\x04\x08\x03\x10\x04\"v\n\x15StreamKeyRangeRequest\x12\x10\n\x08position\x18\x01 \x01(\t\x12%\n\tkey_range\x18\x02 \x01(\x0b\x32\x12.topodata.KeyRange\x12$\n\x07\x63harset\x18\x03 \x01(\x0b\x32\x13.binlogdata.Charset\"S\n\x16StreamKeyRangeResponse\x12\x39\n\x12\x62inlog_transaction\x18\x01 \x01(\x0b\x32\x1d.binlogdata.BinlogTransaction\"]\n\x13StreamTablesRequest\x12\x10\n\x08position\x18\x01 \x01(\t\x12\x0e\n\x06tables\x18\x02 \x03(\t\x12$\n\x07\x63harset\x18\x03 \x01(\x0b\x32\x13.binlogdata.Charset\"Q\n\x14StreamTablesResponse\x12\x39\n\x12\x62inlog_transaction\x18\x01 \x01(\x0b\x32\x1d.binlogdata.BinlogTransaction\"%\n\x04Rule\x12\r\n\x05match\x18\x01 \x01(\t\x12\x0e\n\x06\x66ilter\x18\x02 \x01(\t\")\n\x06\x46ilter\x12\x1f\n\x05rules\x18\x01 \x03(\x0b\x32\x10.binlogdata.Rule\"\xde\x01\n\x0c\x42inlogSource\x12\x10\n\x08keyspace\x18\x01 \x01(\t\x12\r\n\x05shard\x18\x02 \x01(\t\x12)\n\x0btablet_type\x18\x03 \x01(\x0e\x32\x14.topodata.TabletType\x12%\n\tkey_range\x18\x04 \x01(\x0b\x32\x12.topodata.KeyRange\x12\x0e\n\x06tables\x18\x05 \x03(\t\x12\"\n\x06\x66ilter\x18\x06 \x01(\x0b\x32\x12.binlogdata.Filter\x12\'\n\x06on_ddl\x18\x07 \x01(\x0e\x32\x17.binlogdata.OnDDLAction\"B\n\tRowChange\x12\x1a\n\x06\x62\x65\x66ore\x18\x01 \x01(\x0b\x32\n.query.Row\x12\x19\n\x05\x61\x66ter\x18\x02 \x01(\x0b\x32\n.query.Row\"J\n\x08RowEvent\x12\x12\n\ntable_name\x18\x01 \x01(\t\x12*\n\x0brow_changes\x18\x02 \x03(\x0b\x32\x15.binlogdata.RowChange\">\n\nFieldEvent\x12\x12\n\ntable_name\x18\x01 \x01(\t\x12\x1c\n\x06\x66ields\x18\x02 \x03(\x0b\x32\x0c.query.Field\":\n\tShardGtid\x12\x10\n\x08keyspace\x18\x01 \x01(\t\x12\r\n\x05shard\x18\x02 \x01(\t\x12\x0c\n\x04gtid\x18\x03 \x01(\t\"3\n\x05VGtid\x12*\n\x0bshard_gtids\x18\x01 \x03(\x0b\x32\x15.binlogdata.ShardGtid\"0\n\rKeyspaceShard\x12\x10\n\x08keyspace\x18\x01 \x01(\t\x12\r\n\x05shard\x18\x02 \x01(\t\"\xe3\x01\n\x07Journal\x12\n\n\x02id\x18\x01 \x01(\x03\x12\x31\n\x0emigration_type\x18\x02 \x01(\x0e\x32\x19.binlogdata.MigrationType\x12\x0e\n\x06tables\x18\x03 \x03(\t\x12\x16\n\x0elocal_position\x18\x04 \x01(\t\x12*\n\x0bshard_gtids\x18\x05 \x03(\x0b\x32\x15.binlogdata.ShardGtid\x12/\n\x0cparticipants\x18\x06 \x03(\x0b\x32\x19.binlogdata.KeyspaceShard\x12\x14\n\x0creversed_ids\x18\x07 \x03(\x03\"\x9d\x02\n\x06VEvent\x12$\n\x04type\x18\x01 \x01(\x0e\x32\x16.binlogdata.VEventType\x12\x11\n\ttimestamp\x18\x02 \x01(\x03\x12\x0c\n\x04gtid\x18\x03 \x01(\t\x12\x0b\n\x03\x64\x64l\x18\x04 \x01(\t\x12\'\n\trow_event\x18\x05 \x01(\x0b\x32\x14.binlogdata.RowEvent\x12+\n\x0b\x66ield_event\x18\x06 \x01(\x0b\x32\x16.binlogdata.FieldEvent\x12 \n\x05vgtid\x18\x07 \x01(\x0b\x32\x11.binlogdata.VGtid\x12$\n\x07journal\x18\x08 \x01(\x0b\x32\x13.binlogdata.Journal\x12\x0b\n\x03\x64ml\x18\t \x01(\t\x12\x14\n\x0c\x63urrent_time\x18\x14 \x01(\x03\"\xc7\x01\n\x0eVStreamRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x1d\n\x06target\x18\x03 \x01(\x0b\x32\r.query.Target\x12\x10\n\x08position\x18\x04 \x01(\t\x12\"\n\x06\x66ilter\x18\x05 \x01(\x0b\x32\x12.binlogdata.Filter\"5\n\x0fVStreamResponse\x12\"\n\x06\x65vents\x18\x01 \x03(\x0b\x32\x12.binlogdata.VEvent\"\xc8\x01\n\x12VStreamRowsRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x1d\n\x06target\x18\x03 \x01(\x0b\x32\r.query.Target\x12\r\n\x05query\x18\x04 \x01(\t\x12\"\n\x06lastpk\x18\x05 \x01(\x0b\x32\x12.query.QueryResult\"\x97\x01\n\x13VStreamRowsResponse\x12\x1c\n\x06\x66ields\x18\x01 \x03(\x0b\x32\x0c.query.Field\x12\x1e\n\x08pkfields\x18\x02 \x03(\x0b\x32\x0c.query.Field\x12\x0c\n\x04gtid\x18\x03 \x01(\t\x12\x18\n\x04rows\x18\x04 \x03(\x0b\x32\n.query.Row\x12\x1a\n\x06lastpk\x18\x05 \x01(\x0b\x32\n.query.Row*>\n\x0bOnDDLAction\x12\n\n\x06IGNORE\x10\x00\x12\x08\n\x04STOP\x10\x01\x12\x08\n\x04\x45XEC\x10\x02\x12\x0f\n\x0b\x45XEC_IGNORE\x10\x03*\xd1\x01\n\nVEventType\x12\x0b\n\x07UNKNOWN\x10\x00\x12\x08\n\x04GTID\x10\x01\x12\t\n\x05\x42\x45GIN\x10\x02\x12\n\n\x06\x43OMMIT\x10\x03\x12\x0c\n\x08ROLLBACK\x10\x04\x12\x07\n\x03\x44\x44L\x10\x05\x12\n\n\x06INSERT\x10\x06\x12\x0b\n\x07REPLACE\x10\x07\x12\n\n\x06UPDATE\x10\x08\x12\n\n\x06\x44\x45LETE\x10\t\x12\x07\n\x03SET\x10\n\x12\t\n\x05OTHER\x10\x0b\x12\x07\n\x03ROW\x10\x0c\x12\t\n\x05\x46IELD\x10\r\x12\r\n\tHEARTBEAT\x10\x0e\x12\t\n\x05VGTID\x10\x0f\x12\x0b\n\x07JOURNAL\x10\x10*\'\n\rMigrationType\x12\n\n\x06TABLES\x10\x00\x12\n\n\x06SHARDS\x10\x01\x42)Z\'vitess.io/vitess/go/vt/proto/binlogdatab\x06proto3') , dependencies=[vtrpc__pb2.DESCRIPTOR,query__pb2.DESCRIPTOR,topodata__pb2.DESCRIPTOR,]) @@ -52,8 +52,8 @@ ], containing_type=None, serialized_options=None, - serialized_start=2751, - serialized_end=2813, + serialized_start=2764, + serialized_end=2826, ) _sym_db.RegisterEnumDescriptor(_ONDDLACTION) @@ -135,8 +135,8 @@ ], containing_type=None, serialized_options=None, - serialized_start=2816, - serialized_end=3025, + serialized_start=2829, + serialized_end=3038, ) _sym_db.RegisterEnumDescriptor(_VEVENTTYPE) @@ -158,8 +158,8 @@ ], containing_type=None, serialized_options=None, - serialized_start=3027, - serialized_end=3066, + serialized_start=3040, + serialized_end=3079, ) _sym_db.RegisterEnumDescriptor(_MIGRATIONTYPE) @@ -1031,7 +1031,14 @@ is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( - name='current_time', full_name='binlogdata.VEvent.current_time', index=8, + name='dml', full_name='binlogdata.VEvent.dml', index=8, + number=9, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='current_time', full_name='binlogdata.VEvent.current_time', index=9, number=20, type=3, cpp_type=2, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, @@ -1050,7 +1057,7 @@ oneofs=[ ], serialized_start=1863, - serialized_end=2135, + serialized_end=2148, ) @@ -1108,8 +1115,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=2138, - serialized_end=2337, + serialized_start=2151, + serialized_end=2350, ) @@ -1139,8 +1146,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=2339, - serialized_end=2392, + serialized_start=2352, + serialized_end=2405, ) @@ -1198,8 +1205,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=2395, - serialized_end=2595, + serialized_start=2408, + serialized_end=2608, ) @@ -1257,8 +1264,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=2598, - serialized_end=2749, + serialized_start=2611, + serialized_end=2762, ) _BINLOGTRANSACTION_STATEMENT.fields_by_name['category'].enum_type = _BINLOGTRANSACTION_STATEMENT_CATEGORY diff --git a/py/vtproto/vtrpc_pb2.py b/py/vtproto/vtrpc_pb2.py index 5c9cfc4cc6a..39c07d0a732 100644 --- a/py/vtproto/vtrpc_pb2.py +++ b/py/vtproto/vtrpc_pb2.py @@ -8,7 +8,6 @@ from google.protobuf import message as _message from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() @@ -20,6 +19,7 @@ name='vtrpc.proto', package='vtrpc', syntax='proto3', + serialized_options=_b('\n\017io.vitess.protoZ\"vitess.io/vitess/go/vt/proto/vtrpc'), serialized_pb=_b('\n\x0bvtrpc.proto\x12\x05vtrpc\"F\n\x08\x43\x61llerID\x12\x11\n\tprincipal\x18\x01 \x01(\t\x12\x11\n\tcomponent\x18\x02 \x01(\t\x12\x14\n\x0csubcomponent\x18\x03 \x01(\t\"c\n\x08RPCError\x12+\n\x0blegacy_code\x18\x01 \x01(\x0e\x32\x16.vtrpc.LegacyErrorCode\x12\x0f\n\x07message\x18\x02 \x01(\t\x12\x19\n\x04\x63ode\x18\x03 \x01(\x0e\x32\x0b.vtrpc.Code*\xb6\x02\n\x04\x43ode\x12\x06\n\x02OK\x10\x00\x12\x0c\n\x08\x43\x41NCELED\x10\x01\x12\x0b\n\x07UNKNOWN\x10\x02\x12\x14\n\x10INVALID_ARGUMENT\x10\x03\x12\x15\n\x11\x44\x45\x41\x44LINE_EXCEEDED\x10\x04\x12\r\n\tNOT_FOUND\x10\x05\x12\x12\n\x0e\x41LREADY_EXISTS\x10\x06\x12\x15\n\x11PERMISSION_DENIED\x10\x07\x12\x13\n\x0fUNAUTHENTICATED\x10\x10\x12\x16\n\x12RESOURCE_EXHAUSTED\x10\x08\x12\x17\n\x13\x46\x41ILED_PRECONDITION\x10\t\x12\x0b\n\x07\x41\x42ORTED\x10\n\x12\x10\n\x0cOUT_OF_RANGE\x10\x0b\x12\x11\n\rUNIMPLEMENTED\x10\x0c\x12\x0c\n\x08INTERNAL\x10\r\x12\x0f\n\x0bUNAVAILABLE\x10\x0e\x12\r\n\tDATA_LOSS\x10\x0f*\xe8\x02\n\x0fLegacyErrorCode\x12\x12\n\x0eSUCCESS_LEGACY\x10\x00\x12\x14\n\x10\x43\x41NCELLED_LEGACY\x10\x01\x12\x18\n\x14UNKNOWN_ERROR_LEGACY\x10\x02\x12\x14\n\x10\x42\x41\x44_INPUT_LEGACY\x10\x03\x12\x1c\n\x18\x44\x45\x41\x44LINE_EXCEEDED_LEGACY\x10\x04\x12\x1a\n\x16INTEGRITY_ERROR_LEGACY\x10\x05\x12\x1c\n\x18PERMISSION_DENIED_LEGACY\x10\x06\x12\x1d\n\x19RESOURCE_EXHAUSTED_LEGACY\x10\x07\x12\x1b\n\x17QUERY_NOT_SERVED_LEGACY\x10\x08\x12\x14\n\x10NOT_IN_TX_LEGACY\x10\t\x12\x19\n\x15INTERNAL_ERROR_LEGACY\x10\n\x12\x1a\n\x16TRANSIENT_ERROR_LEGACY\x10\x0b\x12\x1a\n\x16UNAUTHENTICATED_LEGACY\x10\x0c\x42\x35\n\x0fio.vitess.protoZ\"vitess.io/vitess/go/vt/proto/vtrpcb\x06proto3') ) @@ -31,75 +31,75 @@ values=[ _descriptor.EnumValueDescriptor( name='OK', index=0, number=0, - options=None, + serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='CANCELED', index=1, number=1, - options=None, + serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='UNKNOWN', index=2, number=2, - options=None, + serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='INVALID_ARGUMENT', index=3, number=3, - options=None, + serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='DEADLINE_EXCEEDED', index=4, number=4, - options=None, + serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='NOT_FOUND', index=5, number=5, - options=None, + serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='ALREADY_EXISTS', index=6, number=6, - options=None, + serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='PERMISSION_DENIED', index=7, number=7, - options=None, + serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='UNAUTHENTICATED', index=8, number=16, - options=None, + serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='RESOURCE_EXHAUSTED', index=9, number=8, - options=None, + serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='FAILED_PRECONDITION', index=10, number=9, - options=None, + serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='ABORTED', index=11, number=10, - options=None, + serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='OUT_OF_RANGE', index=12, number=11, - options=None, + serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='UNIMPLEMENTED', index=13, number=12, - options=None, + serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='INTERNAL', index=14, number=13, - options=None, + serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='UNAVAILABLE', index=15, number=14, - options=None, + serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='DATA_LOSS', index=16, number=15, - options=None, + serialized_options=None, type=None), ], containing_type=None, - options=None, + serialized_options=None, serialized_start=196, serialized_end=506, ) @@ -114,59 +114,59 @@ values=[ _descriptor.EnumValueDescriptor( name='SUCCESS_LEGACY', index=0, number=0, - options=None, + serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='CANCELLED_LEGACY', index=1, number=1, - options=None, + serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='UNKNOWN_ERROR_LEGACY', index=2, number=2, - options=None, + serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='BAD_INPUT_LEGACY', index=3, number=3, - options=None, + serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='DEADLINE_EXCEEDED_LEGACY', index=4, number=4, - options=None, + serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='INTEGRITY_ERROR_LEGACY', index=5, number=5, - options=None, + serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='PERMISSION_DENIED_LEGACY', index=6, number=6, - options=None, + serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='RESOURCE_EXHAUSTED_LEGACY', index=7, number=7, - options=None, + serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='QUERY_NOT_SERVED_LEGACY', index=8, number=8, - options=None, + serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='NOT_IN_TX_LEGACY', index=9, number=9, - options=None, + serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='INTERNAL_ERROR_LEGACY', index=10, number=10, - options=None, + serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='TRANSIENT_ERROR_LEGACY', index=11, number=11, - options=None, + serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='UNAUTHENTICATED_LEGACY', index=12, number=12, - options=None, + serialized_options=None, type=None), ], containing_type=None, - options=None, + serialized_options=None, serialized_start=509, serialized_end=869, ) @@ -219,28 +219,28 @@ has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='component', full_name='vtrpc.CallerID.component', index=1, number=2, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='subcomponent', full_name='vtrpc.CallerID.subcomponent', index=2, number=3, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], - options=None, + serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], @@ -264,28 +264,28 @@ has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='message', full_name='vtrpc.RPCError.message', index=1, number=2, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='code', full_name='vtrpc.RPCError.code', index=2, number=3, type=14, cpp_type=8, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], - options=None, + serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], @@ -318,6 +318,5 @@ _sym_db.RegisterMessage(RPCError) -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n\017io.vitess.protoZ\"vitess.io/vitess/go/vt/proto/vtrpc')) +DESCRIPTOR._options = None # @@protoc_insertion_point(module_scope) From aea60abc9276aeab9ad679ea6f6d9a99f4261fba Mon Sep 17 00:00:00 2001 From: Rafael Chacon Date: Thu, 10 Oct 2019 11:39:07 -0700 Subject: [PATCH 02/28] Update schema engine to expect mysql.ConnParams Signed-off-by: Rafael Chacon --- go/vt/vttablet/tabletserver/query_engine_test.go | 4 ++-- go/vt/vttablet/tabletserver/schema/engine.go | 10 ++++------ go/vt/vttablet/tabletserver/schema/engine_test.go | 2 +- go/vt/vttablet/tabletserver/tabletserver.go | 4 ++-- go/vt/vttablet/tabletserver/vstreamer/engine.go | 5 ++--- 5 files changed, 11 insertions(+), 14 deletions(-) diff --git a/go/vt/vttablet/tabletserver/query_engine_test.go b/go/vt/vttablet/tabletserver/query_engine_test.go index 739d823982c..57b5e2b509b 100644 --- a/go/vt/vttablet/tabletserver/query_engine_test.go +++ b/go/vt/vttablet/tabletserver/query_engine_test.go @@ -54,7 +54,7 @@ func TestStrictMode(t *testing.T) { // config.EnforceStrictTransTable is true by default. qe := NewQueryEngine(DummyChecker, schema.NewEngine(DummyChecker, config), config) qe.InitDBConfig(dbcfgs) - qe.se.InitDBConfig(dbcfgs) + qe.se.InitDBConfig(dbcfgs.DbaWithDB()) qe.se.Open() if err := qe.Open(); err != nil { t.Error(err) @@ -298,7 +298,7 @@ func newTestQueryEngine(queryPlanCacheSize int, idleTimeout time.Duration, stric config.IdleTimeout = float64(idleTimeout) / 1e9 se := schema.NewEngine(DummyChecker, config) qe := NewQueryEngine(DummyChecker, se, config) - se.InitDBConfig(dbcfgs) + se.InitDBConfig(dbcfgs.DbaWithDB()) qe.InitDBConfig(dbcfgs) return qe } diff --git a/go/vt/vttablet/tabletserver/schema/engine.go b/go/vt/vttablet/tabletserver/schema/engine.go index 2afa16c8f53..36811afb76c 100644 --- a/go/vt/vttablet/tabletserver/schema/engine.go +++ b/go/vt/vttablet/tabletserver/schema/engine.go @@ -31,7 +31,6 @@ import ( "vitess.io/vitess/go/stats" "vitess.io/vitess/go/timer" "vitess.io/vitess/go/vt/concurrency" - "vitess.io/vitess/go/vt/dbconfigs" "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/vterrors" @@ -48,7 +47,7 @@ type notifier func(full map[string]*Table, created, altered, dropped []string) // Engine stores the schema info and performs operations that // keep itself up-to-date. type Engine struct { - dbconfigs *dbconfigs.DBConfigs + cp *mysql.ConnParams // mu protects the following fields. mu sync.Mutex @@ -100,8 +99,8 @@ func NewEngine(checker connpool.MySQLChecker, config tabletenv.TabletConfig) *En } // InitDBConfig must be called before Open. -func (se *Engine) InitDBConfig(dbcfgs *dbconfigs.DBConfigs) { - se.dbconfigs = dbcfgs +func (se *Engine) InitDBConfig(cp *mysql.ConnParams) { + se.cp = cp } // Open initializes the Engine. Calling Open on an already @@ -115,8 +114,7 @@ func (se *Engine) Open() error { start := time.Now() defer log.Infof("Time taken to load the schema: %v", time.Since(start)) ctx := tabletenv.LocalContext() - dbaParams := se.dbconfigs.DbaWithDB() - se.conns.Open(dbaParams, dbaParams, dbaParams) + se.conns.Open(se.cp, se.cp, se.cp) conn, err := se.conns.Get(ctx) if err != nil { diff --git a/go/vt/vttablet/tabletserver/schema/engine_test.go b/go/vt/vttablet/tabletserver/schema/engine_test.go index dcf3c4d8fc8..5e7db5dc99e 100644 --- a/go/vt/vttablet/tabletserver/schema/engine_test.go +++ b/go/vt/vttablet/tabletserver/schema/engine_test.go @@ -412,7 +412,7 @@ func newEngine(queryPlanCacheSize int, reloadTime time.Duration, idleTimeout tim config.SchemaReloadTime = float64(reloadTime) / 1e9 config.IdleTimeout = float64(idleTimeout) / 1e9 se := NewEngine(DummyChecker, config) - se.InitDBConfig(newDBConfigs(db)) + se.InitDBConfig(newDBConfigs(db).DbaWithDB()) return se } diff --git a/go/vt/vttablet/tabletserver/tabletserver.go b/go/vt/vttablet/tabletserver/tabletserver.go index 00c118cae93..657d118de0f 100644 --- a/go/vt/vttablet/tabletserver/tabletserver.go +++ b/go/vt/vttablet/tabletserver/tabletserver.go @@ -389,14 +389,14 @@ func (tsv *TabletServer) InitDBConfig(target querypb.Target, dbcfgs *dbconfigs.D tsv.target = target tsv.dbconfigs = dbcfgs - tsv.se.InitDBConfig(tsv.dbconfigs) + tsv.se.InitDBConfig(tsv.dbconfigs.DbaWithDB()) tsv.qe.InitDBConfig(tsv.dbconfigs) tsv.teCtrl.InitDBConfig(tsv.dbconfigs) tsv.hw.InitDBConfig(tsv.dbconfigs) tsv.hr.InitDBConfig(tsv.dbconfigs) tsv.messager.InitDBConfig(tsv.dbconfigs) tsv.watcher.InitDBConfig(tsv.dbconfigs) - tsv.vstreamer.InitDBConfig(tsv.dbconfigs) + tsv.vstreamer.InitDBConfig(tsv.dbconfigs.DbaWithDB()) return nil } diff --git a/go/vt/vttablet/tabletserver/vstreamer/engine.go b/go/vt/vttablet/tabletserver/vstreamer/engine.go index 7147289fa20..1c393a2b028 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/engine.go +++ b/go/vt/vttablet/tabletserver/vstreamer/engine.go @@ -28,7 +28,6 @@ import ( "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/stats" - "vitess.io/vitess/go/vt/dbconfigs" "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/srvtopo" "vitess.io/vitess/go/vt/topo" @@ -95,8 +94,8 @@ func NewEngine(ts srvtopo.Server, se *schema.Engine) *Engine { } // InitDBConfig performs saves the required info from dbconfigs for future use. -func (vse *Engine) InitDBConfig(dbcfgs *dbconfigs.DBConfigs) { - vse.cp = dbcfgs.DbaWithDB() +func (vse *Engine) InitDBConfig(cp *mysql.ConnParams) { + vse.cp = cp } // Open starts the Engine service. From 11301e3d656b227443a4f76144d975c7254df25f Mon Sep 17 00:00:00 2001 From: Rafael Chacon Date: Thu, 10 Oct 2019 11:41:14 -0700 Subject: [PATCH 03/28] Adds support for file:pos in mysql binlogdump interface Signed-off-by: Rafael Chacon --- go/mysql/flavor.go | 10 ++++++++++ go/mysql/flavor_mariadb.go | 5 +++++ go/mysql/flavor_mysql.go | 5 +++++ 3 files changed, 20 insertions(+) diff --git a/go/mysql/flavor.go b/go/mysql/flavor.go index 2fdbe315722..fd53d84babb 100644 --- a/go/mysql/flavor.go +++ b/go/mysql/flavor.go @@ -62,6 +62,9 @@ type flavor interface { // stopSlave returns the command to stop the slave. stopSlaveCommand() string + // sendBinlogFileDumpCommand sends the packet required to start streaming from file:post + sendBinlogFileDumpCommand(c *Conn, slaveID uint32, binlogFilename string, pos uint32) error + // sendBinlogDumpCommand sends the packet required to start // dumping binlogs from the specified location. sendBinlogDumpCommand(c *Conn, slaveID uint32, startPos Position) error @@ -163,6 +166,13 @@ func (c *Conn) StopSlaveCommand() string { return c.flavor.stopSlaveCommand() } +// SendBinlogFileDumpCommand sends the flavor-specific version of +// the COM_BINLOG_DUMP command to start dumping raw binlog +// events over a slave connection, starting at a given file position. +func (c *Conn) SendBinlogFileDumpCommand(slaveID uint32, binlogFilename string, pos uint32) error { + return c.flavor.sendBinlogFileDumpCommand(c, slaveID, binlogFilename, pos) +} + // SendBinlogDumpCommand sends the flavor-specific version of // the COM_BINLOG_DUMP command to start dumping raw binlog // events over a slave connection, starting at a given GTID. diff --git a/go/mysql/flavor_mariadb.go b/go/mysql/flavor_mariadb.go index 9167dabba5d..20b234730a3 100644 --- a/go/mysql/flavor_mariadb.go +++ b/go/mysql/flavor_mariadb.go @@ -55,6 +55,11 @@ func (mariadbFlavor) stopSlaveCommand() string { return "STOP SLAVE" } +// sendBinlogFileDumpCommand is part of the Flavor interface. +func (mariadbFlavor) sendBinlogFileDumpCommand(c *Conn, slaveID uint32, binlogFilename string, pos uint32) error { + panic("filename binglog not supported for mariadb") +} + // sendBinlogDumpCommand is part of the Flavor interface. func (mariadbFlavor) sendBinlogDumpCommand(c *Conn, slaveID uint32, startPos Position) error { // Tell the server that we understand GTIDs by setting our slave diff --git a/go/mysql/flavor_mysql.go b/go/mysql/flavor_mysql.go index 446e6e95a57..084a34c8219 100644 --- a/go/mysql/flavor_mysql.go +++ b/go/mysql/flavor_mysql.go @@ -53,6 +53,11 @@ func (mysqlFlavor) stopSlaveCommand() string { return "STOP SLAVE" } +// sendBinlogDumpCommand is part of the Flavor interface. +func (mysqlFlavor) sendBinlogFileDumpCommand(c *Conn, slaveID uint32, binlogFilename string, pos uint32) error { + return c.WriteComBinlogDump(slaveID, binlogFilename, pos, 0) +} + // sendBinlogDumpCommand is part of the Flavor interface. func (mysqlFlavor) sendBinlogDumpCommand(c *Conn, slaveID uint32, startPos Position) error { gtidSet, ok := startPos.GTIDSet.(Mysql56GTIDSet) From 2f1d3b1ec8dec21cbceaf65eb55ed5f4a5e59440 Mon Sep 17 00:00:00 2001 From: Rafael Chacon Date: Thu, 10 Oct 2019 11:44:46 -0700 Subject: [PATCH 04/28] Adds BinlogFilePos as a way to encode mysql replication Signed-off-by: Rafael Chacon --- go/mysql/replication_position.go | 29 ++++++++++++++++++ go/mysql/replication_position_test.go | 42 +++++++++++++++++++++++++++ 2 files changed, 71 insertions(+) diff --git a/go/mysql/replication_position.go b/go/mysql/replication_position.go index 8d1ec3f9e08..3ddd7d950ce 100644 --- a/go/mysql/replication_position.go +++ b/go/mysql/replication_position.go @@ -19,6 +19,7 @@ package mysql import ( "encoding/json" "fmt" + "strconv" "strings" "vitess.io/vitess/go/vt/proto/vtrpc" @@ -35,6 +36,12 @@ const ( MaximumPositionSize = 64000 ) +// BinlogFilePos used to encode filename:pos. +type BinlogFilePos struct { + Name string + Pos uint32 +} + // Position represents the information necessary to describe which // transactions a server has seen, so that it can request a replication stream // from a new master that picks up where it left off. @@ -120,6 +127,28 @@ func EncodePosition(rp Position) string { return fmt.Sprintf("%s/%s", rp.GTIDSet.Flavor(), rp.GTIDSet.String()) } +// ParseFilePosition converts a string in the format file:pos +// to BinlogFilePos +func ParseFilePosition(s string) (rp BinlogFilePos, err error) { + if s == "" { + return rp, nil + } + + parts := strings.SplitN(s, ":", 2) + if len(parts) != 2 { + return rp, vterrors.Errorf(vtrpc.Code_INTERNAL, "parse error: unknown file:pos format %#v", s) + } + + pos, err := strconv.Atoi(parts[1]) + if err != nil { + return rp, vterrors.Errorf(vtrpc.Code_INTERNAL, "parse error: pos is not a valid int %#v", s) + } + + rp.Name = parts[0] + rp.Pos = uint32(pos) + return rp, nil +} + // DecodePosition converts a string in the format returned by // EncodePosition back into a Position value with the // correct underlying flavor. diff --git a/go/mysql/replication_position_test.go b/go/mysql/replication_position_test.go index 76dce397d00..2429eeb9fb9 100644 --- a/go/mysql/replication_position_test.go +++ b/go/mysql/replication_position_test.go @@ -208,6 +208,48 @@ func TestPositionAppendToZero(t *testing.T) { } } +func TestParseFilePositionInvalidInput(t *testing.T) { + input := "filenameinvalidpos" + rp, err := ParseFilePosition(input) + if err == nil { + t.Errorf("ParseFilePosition(%#v) expected error, got : %#v", input, rp) + } + + want := `parse error: unknown file:pos format` + got, ok := err.(error) + if !ok || !strings.HasPrefix(got.Error(), want) { + t.Errorf("wrong error, got %#v, want %#v", got, want) + } +} + +func TestParseFilePositionInvalidPos(t *testing.T) { + input := "filename:invalidpos" + rp, err := ParseFilePosition(input) + if err == nil { + t.Errorf("ParseFilePosition(%#v) expected error, got : %#v", input, rp) + } + + want := `parse error: pos is not a valid` + got, ok := err.(error) + if !ok || !strings.HasPrefix(got.Error(), want) { + t.Errorf("wrong error, got %#v, want %#v", got, want) + } +} + +func TestParseFilePosition(t *testing.T) { + input := "filename:2343" + want := BinlogFilePos{Name: "filename", Pos: 2343} + got, err := ParseFilePosition(input) + if err != nil { + t.Errorf("ParseFilePosition(%#v) unexpected error: %#v", input, err) + } + + if got.Name != want.Name || got.Pos != want.Pos { + t.Errorf("ParseFilePosition(%#v) = %#v, want %#v", input, got, want) + } + +} + func TestMustParsePosition(t *testing.T) { flavor := "fake flavor" gtidSetParsers[flavor] = func(s string) (GTIDSet, error) { From a2b0074e89adb5e05c35443e28fa084206bfd552 Mon Sep 17 00:00:00 2001 From: Rafael Chacon Date: Thu, 10 Oct 2019 11:45:45 -0700 Subject: [PATCH 05/28] Adds StartBinlogDumpFromFilePosition to slave connection protocol Signed-off-by: Rafael Chacon --- go/vt/binlog/slave_connection.go | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/go/vt/binlog/slave_connection.go b/go/vt/binlog/slave_connection.go index c44d6e87f3b..0c5d215538e 100644 --- a/go/vt/binlog/slave_connection.go +++ b/go/vt/binlog/slave_connection.go @@ -127,6 +127,26 @@ func (sc *SlaveConnection) StartBinlogDumpFromPosition(ctx context.Context, star return sc.streamEvents(ctx), nil } +// StartBinlogDumpFromFilePosition requests a replication binlog dump from +// the master mysqld at the given binlog filename:pos and then sends binlog +// events to the provided channel. +// The stream will continue in the background, waiting for new events if +// necessary, until the connection is closed, either by the master or +// by canceling the context. +// +// Note the context is valid and used until eventChan is closed. +func (sc *SlaveConnection) StartBinlogDumpFromFilePosition(ctx context.Context, binlogFilename string, pos uint32) (<-chan mysql.BinlogEvent, error) { + ctx, sc.cancel = context.WithCancel(ctx) + + log.Infof("sending binlog file dump command: binlogfilename=%v, pos=%v, slaveID=%v", binlogFilename, pos, sc.slaveID) + if err := sc.SendBinlogFileDumpCommand(sc.slaveID, binlogFilename, pos); err != nil { + log.Errorf("couldn't send binlog dump command: %v", err) + return nil, err + } + + return sc.streamEvents(ctx), nil +} + // streamEvents returns a channel on which events are streamed. func (sc *SlaveConnection) streamEvents(ctx context.Context) chan mysql.BinlogEvent { // FIXME(alainjobart) I think we can use a buffered channel for better performance. From 4210649e52a0080fe91167f315eac38aafccdf76 Mon Sep 17 00:00:00 2001 From: Rafael Chacon Date: Thu, 10 Oct 2019 13:16:09 -0700 Subject: [PATCH 06/28] Enables vreplication to run directly from MySQL * Adds support for VStream to start from filename:pos and not gtid sets. * Adds support for statement based replication streams (this should only be used in the context of mysql streamer, it is not safe for tablet vreplicaiton). * Adds support to run vstream from mysql directly Signed-off-by: Rafael Chacon --- go/vt/binlog/binlogplayer/binlog_player.go | 12 +- .../tabletmanager/vreplication/controller.go | 3 +- .../vreplication/framework_test.go | 2 +- .../tabletmanager/vreplication/vcopier.go | 20 +- .../tabletmanager/vreplication/vplayer.go | 49 ++-- .../tabletmanager/vreplication/vreplicator.go | 30 +-- .../vreplication/vstreamer_client.go | 211 ++++++++++++++++++ .../tabletserver/vstreamer/main_test.go | 2 +- .../tabletserver/vstreamer/testenv/testenv.go | 2 +- .../tabletserver/vstreamer/vstreamer.go | 60 ++++- 10 files changed, 322 insertions(+), 69 deletions(-) create mode 100644 go/vt/vttablet/tabletmanager/vreplication/vstreamer_client.go diff --git a/go/vt/binlog/binlogplayer/binlog_player.go b/go/vt/binlog/binlogplayer/binlog_player.go index b62522a1ce9..088f9d325ca 100644 --- a/go/vt/binlog/binlogplayer/binlog_player.go +++ b/go/vt/binlog/binlogplayer/binlog_player.go @@ -202,7 +202,10 @@ func (blp *BinlogPlayer) applyEvents(ctx context.Context) error { log.Error(err) return err } - blp.position = settings.StartPos + + if settings.GtidStartPos != nil { + blp.position = *settings.GtidStartPos + } blp.stopPosition = settings.StopPos t, err := throttler.NewThrottler( fmt.Sprintf("BinlogPlayer/%d", blp.uid), @@ -517,11 +520,12 @@ func SetVReplicationState(dbClient DBClient, uid uint32, state, message string) // VRSettings contains the settings of a vreplication table. type VRSettings struct { - StartPos mysql.Position + StartPos string StopPos mysql.Position MaxTPS int64 MaxReplicationLag int64 State string + GtidStartPos *mysql.Position } // ReadVRSettings retrieves the throttler settings for @@ -546,7 +550,8 @@ func ReadVRSettings(dbClient DBClient, uid uint32) (VRSettings, error) { if err != nil { return VRSettings{}, fmt.Errorf("failed to parse max_replication_lag column: %v", err) } - startPos, err := mysql.DecodePosition(vrRow[0].ToString()) + startPos := vrRow[0].ToString() + gtidStartPos, err := mysql.DecodePosition(startPos) if err != nil { return VRSettings{}, fmt.Errorf("failed to parse pos column: %v", err) } @@ -557,6 +562,7 @@ func ReadVRSettings(dbClient DBClient, uid uint32) (VRSettings, error) { return VRSettings{ StartPos: startPos, + GtidStartPos: >idStartPos, StopPos: stopPos, MaxTPS: maxTPS, MaxReplicationLag: maxReplicationLag, diff --git a/go/vt/vttablet/tabletmanager/vreplication/controller.go b/go/vt/vttablet/tabletmanager/vreplication/controller.go index 41c333aa9f5..1c4ce0af6ca 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/controller.go +++ b/go/vt/vttablet/tabletmanager/vreplication/controller.go @@ -199,7 +199,8 @@ func (ct *controller) runBlp(ctx context.Context) (err error) { if _, err := dbClient.ExecuteFetch("set names binary", 10000); err != nil { return err } - vreplicator := newVReplicator(ct.id, &ct.source, tablet, ct.blpStats, dbClient, ct.mysqld) + vsClient := NewTabletVStreamerClient(tablet) + vreplicator := newVReplicator(ct.id, &ct.source, vsClient, ct.blpStats, dbClient, ct.mysqld) return vreplicator.Replicate(ctx) } return fmt.Errorf("missing source") diff --git a/go/vt/vttablet/tabletmanager/vreplication/framework_test.go b/go/vt/vttablet/tabletmanager/vreplication/framework_test.go index 825e6e3ea63..bb8e4982094 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/framework_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/framework_test.go @@ -82,7 +82,7 @@ func TestMain(m *testing.M) { // engines cannot be initialized in testenv because it introduces // circular dependencies. streamerEngine = vstreamer.NewEngine(env.SrvTopo, env.SchemaEngine) - streamerEngine.InitDBConfig(env.Dbcfgs) + streamerEngine.InitDBConfig(env.Dbcfgs.DbaWithDB()) streamerEngine.Open(env.KeyspaceName, env.Cells[0]) defer streamerEngine.Close() diff --git a/go/vt/vttablet/tabletmanager/vreplication/vcopier.go b/go/vt/vttablet/tabletmanager/vreplication/vcopier.go index 37152f006e8..2bf53348039 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vcopier.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vcopier.go @@ -30,10 +30,8 @@ import ( "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/binlog/binlogplayer" - "vitess.io/vitess/go/vt/grpcclient" "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/sqlparser" - "vitess.io/vitess/go/vt/vttablet/tabletconn" binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" querypb "vitess.io/vitess/go/vt/proto/query" @@ -124,7 +122,7 @@ func (vc *vcopier) catchup(ctx context.Context, copyState map[string]*sqltypes.R } // If there's no start position, it means we're copying the // first table. So, there's nothing to catch up to. - if settings.StartPos.IsZero() { + if settings.GtidStartPos.IsZero() { return nil } @@ -176,21 +174,15 @@ func (vc *vcopier) copyTable(ctx context.Context, tableName string, copyState ma return fmt.Errorf("plan not found for table: %s, current plans are: %#v", tableName, plan.TargetTables) } - vsClient, err := tabletconn.GetDialer()(vc.vr.sourceTablet, grpcclient.FailFast(false)) + err = vc.vr.sourceVStreamer.Open(ctx) if err != nil { - return fmt.Errorf("error dialing tablet: %v", err) + return fmt.Errorf("error opening vsclient: %v", err) } - defer vsClient.Close(ctx) + defer vc.vr.sourceVStreamer.Close(ctx) ctx, cancel := context.WithTimeout(ctx, copyTimeout) defer cancel() - target := &querypb.Target{ - Keyspace: vc.vr.sourceTablet.Keyspace, - Shard: vc.vr.sourceTablet.Shard, - TabletType: vc.vr.sourceTablet.Type, - } - var lastpkpb *querypb.QueryResult if lastpkqr := copyState[tableName]; lastpkqr != nil { lastpkpb = sqltypes.ResultToProto3(lastpkqr) @@ -198,7 +190,7 @@ func (vc *vcopier) copyTable(ctx context.Context, tableName string, copyState ma var pkfields []*querypb.Field var updateCopyState *sqlparser.ParsedQuery - err = vsClient.VStreamRows(ctx, target, initialPlan.SendRule.Filter, lastpkpb, func(rows *binlogdatapb.VStreamRowsResponse) error { + err = vc.vr.sourceVStreamer.VStreamRows(ctx, initialPlan.SendRule.Filter, lastpkpb, func(rows *binlogdatapb.VStreamRowsResponse) error { select { case <-ctx.Done(): return io.EOF @@ -296,7 +288,7 @@ func (vc *vcopier) fastForward(ctx context.Context, copyState map[string]*sqltyp if err != nil { return err } - if settings.StartPos.IsZero() { + if settings.GtidStartPos.IsZero() { update := binlogplayer.GenerateUpdatePos(vc.vr.id, pos, time.Now().Unix(), 0) _, err := vc.vr.dbClient.Execute(update) return err diff --git a/go/vt/vttablet/tabletmanager/vreplication/vplayer.go b/go/vt/vttablet/tabletmanager/vreplication/vplayer.go index 3a4bfd1fff4..0f37bd5a6d4 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vplayer.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vplayer.go @@ -28,20 +28,20 @@ import ( "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/binlog/binlogplayer" - "vitess.io/vitess/go/vt/grpcclient" "vitess.io/vitess/go/vt/log" - "vitess.io/vitess/go/vt/vttablet/tabletconn" + + // "vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer" binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" - querypb "vitess.io/vitess/go/vt/proto/query" ) type vplayer struct { - vr *vreplicator - startPos mysql.Position - stopPos mysql.Position - saveStop bool - copyState map[string]*sqltypes.Result + vr *vreplicator + startPos mysql.Position + stopPos mysql.Position + startBinlogFilePos *mysql.BinlogFilePos + saveStop bool + copyState map[string]*sqltypes.Result replicatorPlan *ReplicatorPlan tablePlans map[string]*TablePlan @@ -66,8 +66,8 @@ func newVPlayer(vr *vreplicator, settings binlogplayer.VRSettings, copyState map } return &vplayer{ vr: vr, - startPos: settings.StartPos, - pos: settings.StartPos, + startPos: *settings.GtidStartPos, + pos: *settings.GtidStartPos, stopPos: settings.StopPos, saveStop: saveStop, copyState: copyState, @@ -105,28 +105,23 @@ func (vp *vplayer) play(ctx context.Context) error { return nil } -func (vp *vplayer) fetchAndApply(ctx context.Context) error { - log.Infof("Starting VReplication player id: %v, startPos: %v, stop: %v, source: %v, filter: %v", vp.vr.id, vp.startPos, vp.stopPos, vp.vr.sourceTablet, vp.vr.source) +func (vp *vplayer) fetchAndApply(ctx context.Context) (err error) { + log.Infof("Starting VReplication player id: %v, startPos: %v, stop: %v, filter: %v", vp.vr.id, vp.startPos, vp.stopPos, vp.vr.source) - vsClient, err := tabletconn.GetDialer()(vp.vr.sourceTablet, grpcclient.FailFast(false)) + err = vp.vr.sourceVStreamer.Open(ctx) if err != nil { - return fmt.Errorf("error dialing tablet: %v", err) + return fmt.Errorf("error creating vstreamer client: %v", err) } - defer vsClient.Close(ctx) + defer vp.vr.sourceVStreamer.Close(ctx) + ctx, cancel := context.WithCancel(ctx) defer cancel() relay := newRelayLog(ctx, relayLogMaxItems, relayLogMaxSize) - target := &querypb.Target{ - Keyspace: vp.vr.sourceTablet.Keyspace, - Shard: vp.vr.sourceTablet.Shard, - TabletType: vp.vr.sourceTablet.Type, - } - log.Infof("Sending vstream command: %v", vp.replicatorPlan.VStreamFilter) streamErr := make(chan error, 1) go func() { - streamErr <- vsClient.VStream(ctx, target, mysql.EncodePosition(vp.startPos), vp.replicatorPlan.VStreamFilter, func(events []*binlogdatapb.VEvent) error { + streamErr <- vp.vr.sourceVStreamer.VStream(ctx, mysql.EncodePosition(vp.startPos), vp.replicatorPlan.VStreamFilter, func(events []*binlogdatapb.VEvent) error { return relay.Send(events) }) }() @@ -345,7 +340,17 @@ func (vp *vplayer) applyEvent(ctx context.Context, event *binlogdatapb.VEvent, m return err } vp.tablePlans[event.FieldEvent.TableName] = tplan + case binlogdatapb.VEventType_INSERT, binlogdatapb.VEventType_DELETE, binlogdatapb.VEventType_UPDATE: + // This is a player using stament based replication + if err := vp.vr.dbClient.Begin(); err != nil { + return err + } + + if _, err := vp.vr.dbClient.ExecuteWithRetry(ctx, event.Dml); err != nil { + log.Warningf("Fail to run: %v. Got error: %v", event.Dml, err) + } case binlogdatapb.VEventType_ROW: + // This player is configured for row based replicaiton if err := vp.vr.dbClient.Begin(); err != nil { return err } diff --git a/go/vt/vttablet/tabletmanager/vreplication/vreplicator.go b/go/vt/vttablet/tabletmanager/vreplication/vreplicator.go index d067191c279..919651bd537 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vreplicator.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vreplicator.go @@ -30,7 +30,6 @@ import ( "vitess.io/vitess/go/vt/mysqlctl" binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" - topodatapb "vitess.io/vitess/go/vt/proto/topodata" ) var ( @@ -46,25 +45,28 @@ var ( ) type vreplicator struct { - id uint32 - source *binlogdatapb.BinlogSource - sourceTablet *topodatapb.Tablet - stats *binlogplayer.Stats - dbClient *vdbClient + id uint32 + dbClient *vdbClient + // source + source *binlogdatapb.BinlogSource + sourceVStreamer VStreamerClient + + // target + stats *binlogplayer.Stats // mysqld is used to fetch the local schema. mysqld mysqlctl.MysqlDaemon tableKeys map[string][]string } -func newVReplicator(id uint32, source *binlogdatapb.BinlogSource, sourceTablet *topodatapb.Tablet, stats *binlogplayer.Stats, dbClient binlogplayer.DBClient, mysqld mysqlctl.MysqlDaemon) *vreplicator { +func newVReplicator(id uint32, source *binlogdatapb.BinlogSource, sourceVStreamer VStreamerClient, stats *binlogplayer.Stats, dbClient binlogplayer.DBClient, mysqld mysqlctl.MysqlDaemon) *vreplicator { return &vreplicator{ - id: id, - source: source, - sourceTablet: sourceTablet, - stats: stats, - dbClient: newVDBClient(dbClient, stats), - mysqld: mysqld, + id: id, + source: source, + sourceVStreamer: sourceVStreamer, + stats: stats, + dbClient: newVDBClient(dbClient, stats), + mysqld: mysqld, } } @@ -89,7 +91,7 @@ func (vr *vreplicator) Replicate(ctx context.Context) error { if err := newVCopier(vr).copyNext(ctx, settings); err != nil { return err } - case settings.StartPos.IsZero(): + case settings.GtidStartPos.IsZero(): if err := newVCopier(vr).initTablesForCopy(ctx); err != nil { return err } diff --git a/go/vt/vttablet/tabletmanager/vreplication/vstreamer_client.go b/go/vt/vttablet/tabletmanager/vreplication/vstreamer_client.go new file mode 100644 index 00000000000..745a750fbd8 --- /dev/null +++ b/go/vt/vttablet/tabletmanager/vreplication/vstreamer_client.go @@ -0,0 +1,211 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vreplication + +import ( + "errors" + "fmt" + "sync" + + "golang.org/x/net/context" + + "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/vt/grpcclient" + "vitess.io/vitess/go/vt/srvtopo" + "vitess.io/vitess/go/vt/topo/memorytopo" + "vitess.io/vitess/go/vt/vttablet/queryservice" + "vitess.io/vitess/go/vt/vttablet/tabletconn" + "vitess.io/vitess/go/vt/vttablet/tabletserver/connpool" + "vitess.io/vitess/go/vt/vttablet/tabletserver/schema" + "vitess.io/vitess/go/vt/vttablet/tabletserver/tabletenv" + "vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer" + + binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" + querypb "vitess.io/vitess/go/vt/proto/query" + topodatapb "vitess.io/vitess/go/vt/proto/topodata" +) + +var ( + _ VStreamerClient = (*TabletVStreamerClient)(nil) + _ VStreamerClient = (*MySQLVStreamerClient)(nil) +) + +// VStreamerClient exposes the core interface of a vstreamer +type VStreamerClient interface { + // Open sets up all the environment for a vstream + Open(ctx context.Context) error + // Close closes a vstream + Close(ctx context.Context) error + + // VStream streams VReplication events based on the specified filter. + VStream(ctx context.Context, startPos string, filter *binlogdatapb.Filter, send func([]*binlogdatapb.VEvent) error) error + + // VStreamRows streams rows of a table from the specified starting point. + VStreamRows(ctx context.Context, query string, lastpk *querypb.QueryResult, send func(*binlogdatapb.VStreamRowsResponse) error) error +} + +// TabletVStreamerClient a vstream client backed by vttablet +type TabletVStreamerClient struct { + // mu protects isOpen, streamers, streamIdx and kschema. + mu sync.Mutex + + isOpen bool + + tablet *topodatapb.Tablet + target *querypb.Target + tsQueryService queryservice.QueryService +} + +// MySQLVStreamerClient a vstream client backed by MySQL +type MySQLVStreamerClient struct { + // mu protects isOpen, streamers, streamIdx and kschema. + mu sync.Mutex + + isOpen bool + + sourceConnParams *mysql.ConnParams + vsEngine *vstreamer.Engine +} + +// NewTabletVStreamerClient creates a new TabletVStreamerClient +func NewTabletVStreamerClient(tablet *topodatapb.Tablet) *TabletVStreamerClient { + return &TabletVStreamerClient{ + tablet: tablet, + target: &querypb.Target{ + Keyspace: tablet.Keyspace, + Shard: tablet.Shard, + TabletType: tablet.Type, + }, + } +} + +// Open part of the VStreamerClient interface +func (vsClient *TabletVStreamerClient) Open(ctx context.Context) (err error) { + vsClient.mu.Lock() + defer vsClient.mu.Unlock() + if vsClient.isOpen { + return nil + } + vsClient.isOpen = true + + vsClient.tsQueryService, err = tabletconn.GetDialer()(vsClient.tablet, grpcclient.FailFast(false)) + return err +} + +// Close part of the VStreamerClient interface +func (vsClient *TabletVStreamerClient) Close(ctx context.Context) (err error) { + if !vsClient.isOpen { + return nil + } + return vsClient.tsQueryService.Close(ctx) +} + +// VStream part of the VStreamerClient interface +func (vsClient *TabletVStreamerClient) VStream(ctx context.Context, startPos string, filter *binlogdatapb.Filter, send func([]*binlogdatapb.VEvent) error) error { + if !vsClient.isOpen { + return errors.New("Can't VStream without opening client") + } + return vsClient.tsQueryService.VStream(ctx, vsClient.target, startPos, filter, send) +} + +// VStreamRows part of the VStreamerClient interface +func (vsClient *TabletVStreamerClient) VStreamRows(ctx context.Context, query string, lastpk *querypb.QueryResult, send func(*binlogdatapb.VStreamRowsResponse) error) error { + if !vsClient.isOpen { + return errors.New("Can't VStreamRows without opening client") + } + return vsClient.tsQueryService.VStreamRows(ctx, vsClient.target, query, lastpk, send) +} + +// NewMySQLVStreamerClient is a vstream client that allows you to stream directly from MySQL. +// In order to achieve this, the following creates a vstreamer Engine with a dummy in memorytopo. +func NewMySQLVStreamerClient(sourceConnParams *mysql.ConnParams) *MySQLVStreamerClient { + vsClient := &MySQLVStreamerClient{ + sourceConnParams: sourceConnParams, + } + + return vsClient +} + +// Open part of the VStreamerClient interface +func (vsClient *MySQLVStreamerClient) Open(ctx context.Context) (err error) { + vsClient.mu.Lock() + defer vsClient.mu.Unlock() + if vsClient.isOpen { + return nil + } + vsClient.isOpen = true + + // Let's create all the required components by vstreamer.Engine + + sourceSe := schema.NewEngine(checker{}, tabletenv.DefaultQsConfig) + sourceSe.InitDBConfig(vsClient.sourceConnParams) + err = sourceSe.Open() + if err != nil { + return err + } + + topo := memorytopo.NewServer("mysqlstreamer") + srvTopo := srvtopo.NewResilientServer(topo, "TestTopo") + + vsClient.vsEngine = vstreamer.NewEngine(srvTopo, sourceSe) + err = vsClient.Open(ctx) + if err != nil { + return err + } + + return nil +} + +// Close part of the VStreamerClient interface +func (vsClient *MySQLVStreamerClient) Close(ctx context.Context) (err error) { + if !vsClient.isOpen { + return nil + } + vsClient.vsEngine.Close() + return nil +} + +// VStream part of the VStreamerClient interface +func (vsClient *MySQLVStreamerClient) VStream(ctx context.Context, startPos string, filter *binlogdatapb.Filter, send func([]*binlogdatapb.VEvent) error) error { + if !vsClient.isOpen { + return errors.New("Can't VStream without opening client") + } + return vsClient.vsEngine.Stream(ctx, startPos, filter, send) +} + +// VStreamRows part of the VStreamerClient interface +func (vsClient *MySQLVStreamerClient) VStreamRows(ctx context.Context, query string, lastpk *querypb.QueryResult, send func(*binlogdatapb.VStreamRowsResponse) error) error { + if !vsClient.isOpen { + return errors.New("Can't VStream without opening client") + } + var row []sqltypes.Value + if lastpk != nil { + r := sqltypes.Proto3ToResult(lastpk) + if len(r.Rows) != 1 { + return fmt.Errorf("unexpected lastpk input: %v", lastpk) + } + row = r.Rows[0] + } + return vsClient.vsEngine.StreamRows(ctx, query, row, send) +} + +type checker struct{} + +var _ = connpool.MySQLChecker(checker{}) + +func (checker) CheckMySQL() {} diff --git a/go/vt/vttablet/tabletserver/vstreamer/main_test.go b/go/vt/vttablet/tabletserver/vstreamer/main_test.go index 224091078f0..e1b3c476d40 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/main_test.go +++ b/go/vt/vttablet/tabletserver/vstreamer/main_test.go @@ -49,7 +49,7 @@ func TestMain(m *testing.M) { // engine cannot be initialized in testenv because it introduces // circular dependencies. engine = NewEngine(env.SrvTopo, env.SchemaEngine) - engine.InitDBConfig(env.Dbcfgs) + engine.InitDBConfig(env.Dbcfgs.DbaWithDB()) engine.Open(env.KeyspaceName, env.Cells[0]) defer engine.Close() diff --git a/go/vt/vttablet/tabletserver/vstreamer/testenv/testenv.go b/go/vt/vttablet/tabletserver/vstreamer/testenv/testenv.go index be2eb3070fe..b43ac316af8 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/testenv/testenv.go +++ b/go/vt/vttablet/tabletserver/vstreamer/testenv/testenv.go @@ -106,7 +106,7 @@ func Init() (*Env, error) { te.Dbcfgs = dbconfigs.NewTestDBConfigs(te.cluster.MySQLConnParams(), te.cluster.MySQLAppDebugConnParams(), te.cluster.DbName()) te.Mysqld = mysqlctl.NewMysqld(te.Dbcfgs) te.SchemaEngine = schema.NewEngine(checker{}, tabletenv.DefaultQsConfig) - te.SchemaEngine.InitDBConfig(te.Dbcfgs) + te.SchemaEngine.InitDBConfig(te.Dbcfgs.Dba()) // The first vschema should not be empty. Leads to Node not found error. // TODO(sougou): need to fix the bug. diff --git a/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go b/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go index 91e09e8f330..d9d2f29ac72 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go +++ b/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go @@ -103,30 +103,41 @@ func (vs *vstreamer) Cancel() { func (vs *vstreamer) Stream() error { defer vs.cancel() - pos, err := mysql.DecodePosition(vs.startPos) - if err != nil { - return err - } - vs.pos = pos - // Ensure se is Open. If vttablet came up in a non_serving role, // the schema engine may not have been initialized. if err := vs.se.Open(); err != nil { - return wrapError(err, vs.pos) + return wrapError(err, vs.startPos) } conn, err := binlog.NewSlaveConnection(vs.cp) if err != nil { - return wrapError(err, vs.pos) + return wrapError(err, vs.startPos) } defer conn.Close() - events, err := conn.StartBinlogDumpFromPosition(vs.ctx, vs.pos) + pos, err := mysql.DecodePosition(vs.startPos) + if err == nil { + vs.pos = pos + events, err := conn.StartBinlogDumpFromPosition(vs.ctx, vs.pos) + if err != nil { + return wrapError(err, vs.startPos) + } + err = vs.parseEvents(vs.ctx, events) + return wrapError(err, vs.startPos) + } + // Let's try to decode as binlog:file position + filePos, err := mysql.ParseFilePosition(vs.startPos) if err != nil { - return wrapError(err, vs.pos) + return wrapError(err, vs.startPos) + } + + events, err := conn.StartBinlogDumpFromFilePosition(vs.ctx, filePos.Name, filePos.Pos) + if err != nil { + return wrapError(err, vs.startPos) } err = vs.parseEvents(vs.ctx, events) - return wrapError(err, vs.pos) + return wrapError(err, vs.startPos) + } func (vs *vstreamer) parseEvents(ctx context.Context, events <-chan mysql.BinlogEvent) error { @@ -152,6 +163,16 @@ func (vs *vstreamer) parseEvents(ctx context.Context, events <-chan mysql.Binlog bufferedEvents = nil curSize = 0 return vs.send(vevents) + case binlogdatapb.VEventType_INSERT, binlogdatapb.VEventType_DELETE, binlogdatapb.VEventType_UPDATE: + newSize := len(vevent.GetDml()) + if curSize+newSize > *PacketSize { + vevents := bufferedEvents + bufferedEvents = []*binlogdatapb.VEvent{vevent} + curSize = newSize + return vs.send(vevents) + } + curSize += newSize + bufferedEvents = append(bufferedEvents, vevent) case binlogdatapb.VEventType_ROW: // ROW events happen inside transactions. So, we can chunk them. // Buffer everything until packet size is reached, and then send. @@ -295,6 +316,21 @@ func (vs *vstreamer) parseEvent(ev mysql.BinlogEvent) ([]*binlogdatapb.VEvent, e return nil, fmt.Errorf("can't get query from binlog event: %v, event data: %#v", err, ev) } switch cat := sqlparser.Preview(q.SQL); cat { + // case sqlparser.StmtInsert: + // vevents = append(vevents, &binlogdatapb.VEvent{ + // Type: binlogdatapb.VEventType_INSERT, + // Dml: q.SQL, + // }) + // case sqlparser.StmtUpdate: + // vevents = append(vevents, &binlogdatapb.VEvent{ + // Type: binlogdatapb.VEventType_UPDATE, + // Dml: q.SQL, + // }) + // case sqlparser.StmtDelete: + // vevents = append(vevents, &binlogdatapb.VEvent{ + // Type: binlogdatapb.VEventType_DELETE, + // Dml: q.SQL, + // }) case sqlparser.StmtBegin: vevents = append(vevents, &binlogdatapb.VEvent{ Type: binlogdatapb.VEventType_BEGIN, @@ -472,7 +508,7 @@ func (vs *vstreamer) extractRowAndFilter(plan *streamerPlan, data []byte, dataCo return plan.filter(values) } -func wrapError(err error, stopPos mysql.Position) error { +func wrapError(err error, stopPos string) error { if err != nil { err = fmt.Errorf("stream error @ %v: %v", stopPos, err) log.Error(err) From c3c238b90907fbc4eb1a2b756ac0a7b2a788b017 Mon Sep 17 00:00:00 2001 From: Rafael Chacon Date: Fri, 11 Oct 2019 14:50:33 -0700 Subject: [PATCH 07/28] Adds the core of vtshovel program * Adds binary to run vtshovel. * At the moment only working in ephemeral mode (i.e no data is persisted back to vrsettings). * vtshovel only works for statement based replication right now. This is due to now having a good way to have a schema loader. We will itereate on this. Signed-off-by: Rafael Chacon --- go/cmd/vtshovel/vtshovel.go | 251 ++++++++++++++++++ go/mysql/replication_position.go | 2 +- go/vt/binlog/binlogplayer/binlog_player.go | 15 +- .../tabletmanager/vreplication/controller.go | 2 +- .../tabletmanager/vreplication/vplayer.go | 14 +- .../tabletmanager/vreplication/vreplicator.go | 24 +- .../vreplication/vstreamer_client.go | 4 +- .../tabletserver/vstreamer/vstreamer.go | 44 +-- 8 files changed, 310 insertions(+), 46 deletions(-) create mode 100644 go/cmd/vtshovel/vtshovel.go diff --git a/go/cmd/vtshovel/vtshovel.go b/go/cmd/vtshovel/vtshovel.go new file mode 100644 index 00000000000..cf7fb6dfd46 --- /dev/null +++ b/go/cmd/vtshovel/vtshovel.go @@ -0,0 +1,251 @@ +/* +Copyright 2017 Google Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "encoding/json" + "flag" + "io/ioutil" + "math/rand" + "regexp" + "strings" + "time" + + "vitess.io/vitess/go/exit" + "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/vt/binlog/binlogplayer" + "vitess.io/vitess/go/vt/log" + "vitess.io/vitess/go/vt/servenv" + "vitess.io/vitess/go/vt/vterrors" + + "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication" + + binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" + tabletmanagerdatapb "vitess.io/vitess/go/vt/proto/tabletmanagerdata" +) + +var ( + vtShovelConfigFile = flag.String("vtshovel-config-file", "/etc/slack.d/vtshovel.json", "VTShovel Config file") + dryRun = flag.Bool("dry-run", false, "When present, only log DML that are going to be performed in target database") + + autoIncr = regexp.MustCompile(` AUTO_INCREMENT=\d+`) +) + +func init() { + rand.Seed(time.Now().UnixNano()) + servenv.RegisterDefaultFlags() +} + +// VtShovelConfig fields to configure vtshovel client +type VtShovelConfig struct { + // Source MySQL client information + + // MySQLSourceHost ... + MySQLSourceHost string `json:"mysql_source_host"` + // MySQLSourcePort ... + MySQLSourcePort int `json:"mysql_source_port"` + // MySQLSourceUser ... + MySQLSourceUser string `json:"mysql_source_user"` + // MySQLSourcePassword ... + MySQLSourcePassword string `json:"mysql_source_password"` + // MySQLSourceBinlogStartPos ... + MySQLSourceBinlogStartPos string `json:"mysql_source_binlog_start_pos"` + // MySQLSourceDatabase ... + MySQLSourceDBName string `json:"mysql_source_dbname"` + + // Target MySQL client information + + // MySQLTargetHost ... + MySQLTargetHost string `json:"mysql_target_host"` + // MySQLTargetPort ... + MySQLTargetPort int `json:"mysql_target_port"` + // MySQLTargetUser ... + MySQLTargetUser string `json:"mysql_target_user"` + // MySQLTargetPassword ... + MySQLTargetPassword string `json:"mysql_target_password"` + // MySQLTargetDBName ... + MySQLTargetDBName string `json:"mysql_target_dbname"` +} + +func main() { + defer exit.Recover() + + servenv.ParseFlags("vtshovel") + servenv.Init() + + servenv.OnRun(func() { + //vreplication.MySQLAddStatusPart() + // Flags are parsed now. Parse the template using the actual flag value and overwrite the current template. + //addStatusParts(vtg) + }) + + vtShovelConfig, err := loadConfigFromFile(*vtShovelConfigFile) + if err != nil { + log.Fatal(err) + } + + targetConnParams := mysql.ConnParams{ + Host: vtShovelConfig.MySQLTargetHost, + Port: vtShovelConfig.MySQLTargetPort, + Pass: vtShovelConfig.MySQLTargetPassword, + Uname: vtShovelConfig.MySQLTargetUser, + DbName: vtShovelConfig.MySQLTargetDBName, + } + dbTargetClient := newVtShovelDbClient( + binlogplayer.NewDBClient(&targetConnParams), + vtShovelConfig.MySQLSourceBinlogStartPos, + ) + + if err := dbTargetClient.Connect(); err != nil { + log.Fatal(vterrors.Wrap(err, "can't connect to database")) + } + + sourceConnParams := mysql.ConnParams{ + Host: vtShovelConfig.MySQLSourceHost, + Port: vtShovelConfig.MySQLSourcePort, + Pass: vtShovelConfig.MySQLSourcePassword, + Uname: vtShovelConfig.MySQLSourceUser, + } + + servenv.OnClose(dbTargetClient.Close) + + source := binlogdatapb.BinlogSource{ + Filter: &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{ + &binlogdatapb.Rule{ + Match: "/" + vtShovelConfig.MySQLSourceDBName + ".*/", + }, + }, + }, + } + ctx := context.Background() + sourceVstreamClient := vreplication.NewMySQLVStreamerClient(&sourceConnParams) + go func() { + replicator := vreplication.NewVReplicator( + 1, + &source, + sourceVstreamClient, + binlogplayer.NewStats(), + dbTargetClient, + newVtShovelSchemaLoader(), + ) + replicator.Replicate(ctx) + if err != nil { + log.Infof("Error starting stream: %v", err) + + } + return + }() + servenv.RunDefault() +} + +func loadConfigFromFile(file string) (*VtShovelConfig, error) { + data, err := ioutil.ReadFile(file) + if err != nil { + return nil, vterrors.Wrapf(err, "Failed to read %v file", file) + } + vtShovelConfig := &VtShovelConfig{} + err = json.Unmarshal(data, vtShovelConfig) + if err != nil { + return nil, vterrors.Wrap(err, "Error parsing auth server config") + } + return vtShovelConfig, nil +} + +type vtShovelDbClient struct { + dbClient binlogplayer.DBClient + startPos string +} + +type vtShovelSchemaLoader struct{} + +func newVtShovelDbClient(dbClient binlogplayer.DBClient, startPos string) binlogplayer.DBClient { + return &vtShovelDbClient{ + dbClient: dbClient, + startPos: startPos, + } +} + +func newVtShovelSchemaLoader() vreplication.SchemasLoader { + return &vtShovelSchemaLoader{} +} + +func (vdc *vtShovelDbClient) DBName() string { + return vdc.dbClient.DBName() +} + +func (vdc *vtShovelDbClient) Connect() error { + return vdc.dbClient.Connect() +} + +func (vdc *vtShovelDbClient) Begin() error { + return vdc.dbClient.Begin() +} + +func (vdc *vtShovelDbClient) Commit() error { + return vdc.dbClient.Commit() +} + +func (vdc *vtShovelDbClient) Rollback() error { + return vdc.dbClient.Rollback() +} + +func (vdc *vtShovelDbClient) Close() { + vdc.dbClient.Close() +} + +func (vdc *vtShovelDbClient) ExecuteFetch(query string, maxrows int) (*sqltypes.Result, error) { + if strings.Contains(query, "from _vt.copy_state") { + dummyResult := &sqltypes.Result{ + Rows: [][]sqltypes.Value{ + []sqltypes.Value{ + sqltypes.NewInt64(0), + }, + }, + } + return dummyResult, nil + } + + if strings.Contains(query, "from _vt.vreplication") { + dummyResult := &sqltypes.Result{ + Rows: [][]sqltypes.Value{ + []sqltypes.Value{ + sqltypes.NewVarBinary(vdc.startPos), + sqltypes.NewVarBinary(""), // StopPos + sqltypes.NewInt64(10000), // maxTPS + sqltypes.NewInt64(10000), // maxReplicationLag + sqltypes.NewVarBinary("Running"), // state + }, + }, + } + return dummyResult, nil + } + + if strings.Contains(query, "update _vt.vreplication") { + return &sqltypes.Result{}, nil + } + return vdc.dbClient.ExecuteFetch(query, maxrows) +} + +func (vsl *vtShovelSchemaLoader) GetSchema(dbName string, tables, excludeTables []string, includeViews bool) (*tabletmanagerdatapb.SchemaDefinition, error) { + // TODO: This will only work for stament based replication. + return &tabletmanagerdatapb.SchemaDefinition{ + TableDefinitions: []*tabletmanagerdatapb.TableDefinition{}, + }, nil +} diff --git a/go/mysql/replication_position.go b/go/mysql/replication_position.go index 3ddd7d950ce..82db22160fd 100644 --- a/go/mysql/replication_position.go +++ b/go/mysql/replication_position.go @@ -131,7 +131,7 @@ func EncodePosition(rp Position) string { // to BinlogFilePos func ParseFilePosition(s string) (rp BinlogFilePos, err error) { if s == "" { - return rp, nil + return rp, vterrors.Errorf(vtrpc.Code_INTERNAL, "parse error: unknown file:pos format %#v", s) } parts := strings.SplitN(s, ":", 2) diff --git a/go/vt/binlog/binlogplayer/binlog_player.go b/go/vt/binlog/binlogplayer/binlog_player.go index 088f9d325ca..52d119ec3f1 100644 --- a/go/vt/binlog/binlogplayer/binlog_player.go +++ b/go/vt/binlog/binlogplayer/binlog_player.go @@ -203,8 +203,8 @@ func (blp *BinlogPlayer) applyEvents(ctx context.Context) error { return err } - if settings.GtidStartPos != nil { - blp.position = *settings.GtidStartPos + if !settings.GtidStartPos.IsZero() { + blp.position = settings.GtidStartPos } blp.stopPosition = settings.StopPos t, err := throttler.NewThrottler( @@ -525,7 +525,7 @@ type VRSettings struct { MaxTPS int64 MaxReplicationLag int64 State string - GtidStartPos *mysql.Position + GtidStartPos mysql.Position } // ReadVRSettings retrieves the throttler settings for @@ -551,10 +551,9 @@ func ReadVRSettings(dbClient DBClient, uid uint32) (VRSettings, error) { return VRSettings{}, fmt.Errorf("failed to parse max_replication_lag column: %v", err) } startPos := vrRow[0].ToString() - gtidStartPos, err := mysql.DecodePosition(startPos) - if err != nil { - return VRSettings{}, fmt.Errorf("failed to parse pos column: %v", err) - } + // TODO: This will be removed when we start using filename:pos flavor and everythign will by a proper enconded mysql.Position + gtidStartPos, _ := mysql.DecodePosition(startPos) + stopPos, err := mysql.DecodePosition(vrRow[1].ToString()) if err != nil { return VRSettings{}, fmt.Errorf("failed to parse stop_pos column: %v", err) @@ -562,7 +561,7 @@ func ReadVRSettings(dbClient DBClient, uid uint32) (VRSettings, error) { return VRSettings{ StartPos: startPos, - GtidStartPos: >idStartPos, + GtidStartPos: gtidStartPos, StopPos: stopPos, MaxTPS: maxTPS, MaxReplicationLag: maxReplicationLag, diff --git a/go/vt/vttablet/tabletmanager/vreplication/controller.go b/go/vt/vttablet/tabletmanager/vreplication/controller.go index 1c4ce0af6ca..699c66212a6 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/controller.go +++ b/go/vt/vttablet/tabletmanager/vreplication/controller.go @@ -200,7 +200,7 @@ func (ct *controller) runBlp(ctx context.Context) (err error) { return err } vsClient := NewTabletVStreamerClient(tablet) - vreplicator := newVReplicator(ct.id, &ct.source, vsClient, ct.blpStats, dbClient, ct.mysqld) + vreplicator := NewVReplicator(ct.id, &ct.source, vsClient, ct.blpStats, dbClient, ct.mysqld) return vreplicator.Replicate(ctx) } return fmt.Errorf("missing source") diff --git a/go/vt/vttablet/tabletmanager/vreplication/vplayer.go b/go/vt/vttablet/tabletmanager/vreplication/vplayer.go index 0f37bd5a6d4..4deb5cfab08 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vplayer.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vplayer.go @@ -37,7 +37,8 @@ import ( type vplayer struct { vr *vreplicator - startPos mysql.Position + startPos string + gtidStartPos mysql.Position stopPos mysql.Position startBinlogFilePos *mysql.BinlogFilePos saveStop bool @@ -66,8 +67,9 @@ func newVPlayer(vr *vreplicator, settings binlogplayer.VRSettings, copyState map } return &vplayer{ vr: vr, - startPos: *settings.GtidStartPos, - pos: *settings.GtidStartPos, + startPos: settings.StartPos, + gtidStartPos: settings.GtidStartPos, + pos: settings.GtidStartPos, stopPos: settings.StopPos, saveStop: saveStop, copyState: copyState, @@ -78,9 +80,9 @@ func newVPlayer(vr *vreplicator, settings binlogplayer.VRSettings, copyState map // play is not resumable. If pausePos is set, play returns without updating the vreplication state. func (vp *vplayer) play(ctx context.Context) error { - if !vp.stopPos.IsZero() && vp.startPos.AtLeast(vp.stopPos) { + if !vp.stopPos.IsZero() && vp.gtidStartPos.AtLeast(vp.stopPos) { if vp.saveStop { - return vp.vr.setState(binlogplayer.BlpStopped, fmt.Sprintf("Stop position %v already reached: %v", vp.startPos, vp.stopPos)) + return vp.vr.setState(binlogplayer.BlpStopped, fmt.Sprintf("Stop position %v already reached: %v", vp.gtidStartPos, vp.stopPos)) } return nil } @@ -121,7 +123,7 @@ func (vp *vplayer) fetchAndApply(ctx context.Context) (err error) { streamErr := make(chan error, 1) go func() { - streamErr <- vp.vr.sourceVStreamer.VStream(ctx, mysql.EncodePosition(vp.startPos), vp.replicatorPlan.VStreamFilter, func(events []*binlogdatapb.VEvent) error { + streamErr <- vp.vr.sourceVStreamer.VStream(ctx, vp.startPos, vp.replicatorPlan.VStreamFilter, func(events []*binlogdatapb.VEvent) error { return relay.Send(events) }) }() diff --git a/go/vt/vttablet/tabletmanager/vreplication/vreplicator.go b/go/vt/vttablet/tabletmanager/vreplication/vreplicator.go index 919651bd537..97294346722 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vreplicator.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vreplicator.go @@ -27,9 +27,9 @@ import ( "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/binlog/binlogplayer" "vitess.io/vitess/go/vt/log" - "vitess.io/vitess/go/vt/mysqlctl" binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" + tabletmanagerdatapb "vitess.io/vitess/go/vt/proto/tabletmanagerdata" ) var ( @@ -53,20 +53,25 @@ type vreplicator struct { // target stats *binlogplayer.Stats - // mysqld is used to fetch the local schema. - mysqld mysqlctl.MysqlDaemon + // sl is used to fetch the local schema. + sl SchemasLoader tableKeys map[string][]string } -func newVReplicator(id uint32, source *binlogdatapb.BinlogSource, sourceVStreamer VStreamerClient, stats *binlogplayer.Stats, dbClient binlogplayer.DBClient, mysqld mysqlctl.MysqlDaemon) *vreplicator { +type SchemasLoader interface { + GetSchema(dbName string, tables, excludeTables []string, includeViews bool) (*tabletmanagerdatapb.SchemaDefinition, error) +} + +// NewVReplicator creates a new vreplicator +func NewVReplicator(id uint32, source *binlogdatapb.BinlogSource, sourceVStreamer VStreamerClient, stats *binlogplayer.Stats, dbClient binlogplayer.DBClient, sl SchemasLoader) *vreplicator { return &vreplicator{ id: id, source: source, sourceVStreamer: sourceVStreamer, stats: stats, dbClient: newVDBClient(dbClient, stats), - mysqld: mysqld, + sl: sl, } } @@ -86,12 +91,17 @@ func (vr *vreplicator) Replicate(ctx context.Context) error { if settings.State == binlogplayer.BlpStopped { return nil } + + // TODO: This will get remove once we use filename:pos flavor + _, err = mysql.ParseFilePosition(settings.StartPos) + isFilePos := err == nil + switch { case numTablesToCopy != 0: if err := newVCopier(vr).copyNext(ctx, settings); err != nil { return err } - case settings.GtidStartPos.IsZero(): + case settings.GtidStartPos.IsZero() && !isFilePos: if err := newVCopier(vr).initTablesForCopy(ctx); err != nil { return err } @@ -105,7 +115,7 @@ func (vr *vreplicator) Replicate(ctx context.Context) error { } func (vr *vreplicator) buildTableKeys() (map[string][]string, error) { - schema, err := vr.mysqld.GetSchema(vr.dbClient.DBName(), []string{"/.*/"}, nil, false) + schema, err := vr.sl.GetSchema(vr.dbClient.DBName(), []string{"/.*/"}, nil, false) if err != nil { return nil, err } diff --git a/go/vt/vttablet/tabletmanager/vreplication/vstreamer_client.go b/go/vt/vttablet/tabletmanager/vreplication/vstreamer_client.go index 745a750fbd8..b6d3f539e01 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vstreamer_client.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vstreamer_client.go @@ -163,7 +163,9 @@ func (vsClient *MySQLVStreamerClient) Open(ctx context.Context) (err error) { srvTopo := srvtopo.NewResilientServer(topo, "TestTopo") vsClient.vsEngine = vstreamer.NewEngine(srvTopo, sourceSe) - err = vsClient.Open(ctx) + vsClient.vsEngine.InitDBConfig(vsClient.sourceConnParams) + + err = vsClient.vsEngine.Open("mysqlstreamer", "cell1") if err != nil { return err } diff --git a/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go b/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go index 8b0af4ad10b..1e08f77cca9 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go +++ b/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go @@ -115,29 +115,29 @@ func (vs *vstreamer) Stream() error { } defer conn.Close() - pos, err := mysql.DecodePosition(vs.startPos) + // TODO: This case logic depending on startPos will disappear when filename:pos flavor is introduced + filePos, err := mysql.ParseFilePosition(vs.startPos) if err == nil { - vs.pos = pos - events, err := conn.StartBinlogDumpFromPosition(vs.ctx, vs.pos) + events, err := conn.StartBinlogDumpFromFilePosition(vs.ctx, filePos.Name, filePos.Pos) if err != nil { return wrapError(err, vs.startPos) } err = vs.parseEvents(vs.ctx, events) return wrapError(err, vs.startPos) } - // Let's try to decode as binlog:file position - filePos, err := mysql.ParseFilePosition(vs.startPos) + // Let's try to decode as gtidset + pos, err := mysql.DecodePosition(vs.startPos) if err != nil { return wrapError(err, vs.startPos) } - events, err := conn.StartBinlogDumpFromFilePosition(vs.ctx, filePos.Name, filePos.Pos) + vs.pos = pos + events, err := conn.StartBinlogDumpFromPosition(vs.ctx, vs.pos) if err != nil { return wrapError(err, vs.startPos) } err = vs.parseEvents(vs.ctx, events) return wrapError(err, vs.startPos) - } func (vs *vstreamer) parseEvents(ctx context.Context, events <-chan mysql.BinlogEvent) error { @@ -316,21 +316,21 @@ func (vs *vstreamer) parseEvent(ev mysql.BinlogEvent) ([]*binlogdatapb.VEvent, e return nil, fmt.Errorf("can't get query from binlog event: %v, event data: %#v", err, ev) } switch cat := sqlparser.Preview(q.SQL); cat { - // case sqlparser.StmtInsert: - // vevents = append(vevents, &binlogdatapb.VEvent{ - // Type: binlogdatapb.VEventType_INSERT, - // Dml: q.SQL, - // }) - // case sqlparser.StmtUpdate: - // vevents = append(vevents, &binlogdatapb.VEvent{ - // Type: binlogdatapb.VEventType_UPDATE, - // Dml: q.SQL, - // }) - // case sqlparser.StmtDelete: - // vevents = append(vevents, &binlogdatapb.VEvent{ - // Type: binlogdatapb.VEventType_DELETE, - // Dml: q.SQL, - // }) + case sqlparser.StmtInsert: + vevents = append(vevents, &binlogdatapb.VEvent{ + Type: binlogdatapb.VEventType_INSERT, + Dml: q.SQL, + }) + case sqlparser.StmtUpdate: + vevents = append(vevents, &binlogdatapb.VEvent{ + Type: binlogdatapb.VEventType_UPDATE, + Dml: q.SQL, + }) + case sqlparser.StmtDelete: + vevents = append(vevents, &binlogdatapb.VEvent{ + Type: binlogdatapb.VEventType_DELETE, + Dml: q.SQL, + }) case sqlparser.StmtBegin: vevents = append(vevents, &binlogdatapb.VEvent{ Type: binlogdatapb.VEventType_BEGIN, From 8bf38cc699f59dfc211265454603793f49a0b2f9 Mon Sep 17 00:00:00 2001 From: Rafael Chacon Date: Wed, 16 Oct 2019 16:33:20 -0700 Subject: [PATCH 08/28] Simplifies vtshovel logic. It assumes that it runs directly again Signed-off-by: Rafael Chacon --- go/cmd/vtshovel/vtshovel.go | 197 +++++++----------- .../tabletmanager/vreplication/vcopier.go | 4 +- .../tabletmanager/vreplication/vplayer.go | 26 ++- .../tabletmanager/vreplication/vreplicator.go | 19 +- .../tabletserver/vstreamer/planbuilder.go | 7 + .../tabletserver/vstreamer/vstreamer.go | 47 +++-- 6 files changed, 145 insertions(+), 155 deletions(-) diff --git a/go/cmd/vtshovel/vtshovel.go b/go/cmd/vtshovel/vtshovel.go index cf7fb6dfd46..5a70f239a10 100644 --- a/go/cmd/vtshovel/vtshovel.go +++ b/go/cmd/vtshovel/vtshovel.go @@ -22,29 +22,24 @@ import ( "flag" "io/ioutil" "math/rand" - "regexp" - "strings" "time" "vitess.io/vitess/go/exit" "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/vt/binlog/binlogplayer" + "vitess.io/vitess/go/vt/dbconfigs" "vitess.io/vitess/go/vt/log" + "vitess.io/vitess/go/vt/mysqlctl" "vitess.io/vitess/go/vt/servenv" "vitess.io/vitess/go/vt/vterrors" - - "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication" binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" - tabletmanagerdatapb "vitess.io/vitess/go/vt/proto/tabletmanagerdata" ) var ( vtShovelConfigFile = flag.String("vtshovel-config-file", "/etc/slack.d/vtshovel.json", "VTShovel Config file") dryRun = flag.Bool("dry-run", false, "When present, only log DML that are going to be performed in target database") - - autoIncr = regexp.MustCompile(` AUTO_INCREMENT=\d+`) ) func init() { @@ -55,7 +50,6 @@ func init() { // VtShovelConfig fields to configure vtshovel client type VtShovelConfig struct { // Source MySQL client information - // MySQLSourceHost ... MySQLSourceHost string `json:"mysql_source_host"` // MySQLSourcePort ... @@ -68,24 +62,14 @@ type VtShovelConfig struct { MySQLSourceBinlogStartPos string `json:"mysql_source_binlog_start_pos"` // MySQLSourceDatabase ... MySQLSourceDBName string `json:"mysql_source_dbname"` - - // Target MySQL client information - - // MySQLTargetHost ... - MySQLTargetHost string `json:"mysql_target_host"` - // MySQLTargetPort ... - MySQLTargetPort int `json:"mysql_target_port"` - // MySQLTargetUser ... - MySQLTargetUser string `json:"mysql_target_user"` - // MySQLTargetPassword ... - MySQLTargetPassword string `json:"mysql_target_password"` - // MySQLTargetDBName ... - MySQLTargetDBName string `json:"mysql_target_dbname"` } func main() { defer exit.Recover() + dbconfigs.RegisterFlags(dbconfigs.Dba) + mysqlctl.RegisterFlags() + servenv.ParseFlags("vtshovel") servenv.Init() @@ -100,54 +84,92 @@ func main() { log.Fatal(err) } - targetConnParams := mysql.ConnParams{ - Host: vtShovelConfig.MySQLTargetHost, - Port: vtShovelConfig.MySQLTargetPort, - Pass: vtShovelConfig.MySQLTargetPassword, - Uname: vtShovelConfig.MySQLTargetUser, - DbName: vtShovelConfig.MySQLTargetDBName, - } - dbTargetClient := newVtShovelDbClient( - binlogplayer.NewDBClient(&targetConnParams), - vtShovelConfig.MySQLSourceBinlogStartPos, - ) - - if err := dbTargetClient.Connect(); err != nil { - log.Fatal(vterrors.Wrap(err, "can't connect to database")) - } - sourceConnParams := mysql.ConnParams{ - Host: vtShovelConfig.MySQLSourceHost, - Port: vtShovelConfig.MySQLSourcePort, - Pass: vtShovelConfig.MySQLSourcePassword, - Uname: vtShovelConfig.MySQLSourceUser, + Host: vtShovelConfig.MySQLSourceHost, + Port: vtShovelConfig.MySQLSourcePort, + Pass: vtShovelConfig.MySQLSourcePassword, + Uname: vtShovelConfig.MySQLSourceUser, + DbName: vtShovelConfig.MySQLSourceDBName, } - servenv.OnClose(dbTargetClient.Close) - source := binlogdatapb.BinlogSource{ Filter: &binlogdatapb.Filter{ Rules: []*binlogdatapb.Rule{ &binlogdatapb.Rule{ - Match: "/" + vtShovelConfig.MySQLSourceDBName + ".*/", + Match: "/.*", }, }, }, } - ctx := context.Background() + + var mycnf *mysqlctl.Mycnf + var socketFile string + // If no connection parameters were specified, load the mycnf file + // and use the socket from it. If connection parameters were specified, + // we assume that the mysql is not local, and we skip loading mycnf. + // This also means that backup and restore will not be allowed. + if !dbconfigs.HasConnectionParams() { + var err error + if mycnf, err = mysqlctl.NewMycnfFromFlags(123213123); err != nil { + log.Exitf("mycnf read failed: %v", err) + } + socketFile = mycnf.SocketFile + } else { + log.Info("connection parameters were specified. Not loading my.cnf.") + } + + // If connection parameters were specified, socketFile will be empty. + // Otherwise, the socketFile (read from mycnf) will be used to initialize + // dbconfigs. + dbcfgs, err := dbconfigs.Init(socketFile) + if err != nil { + log.Warning(err) + } + + mysqld := mysqlctl.NewMysqld(dbcfgs) + servenv.OnClose(mysqld.Close) + + destConnParams := dbcfgs.Dba() + // Hack to make sure dbname is set correctly given that this is not a tablet + // and SetDBName is not called. + destConnParams.DbName = destConnParams.DeprecatedDBName + + log.Infof("This are the destConnParams:%v", destConnParams) + destDbClient := binlogplayer.NewDBClient(destConnParams) + + if err := destDbClient.Connect(); err != nil { + log.Fatal(vterrors.Wrap(err, "can't connect to database")) + } + servenv.OnClose(destDbClient.Close) + + for _, query := range binlogplayer.CreateVReplicationTable() { + if _, err := destDbClient.ExecuteFetch(query, 0); err != nil { + log.Fatalf("Failed to ensure vreplication table exists: %v", err) + } + } + + newVReplicatorStmt := binlogplayer.CreateVReplication("VTshovel", &source, vtShovelConfig.MySQLSourceBinlogStartPos, int64(1000), int64(100000), time.Now().Unix(), destDbClient.DBName()) + + res, err := destDbClient.ExecuteFetch(newVReplicatorStmt, 0) + if err != nil { + log.Fatalf("Failed to create vreplication stream: %v", err) + } + sourceVstreamClient := vreplication.NewMySQLVStreamerClient(&sourceConnParams) + go func() { + ctx := context.Background() replicator := vreplication.NewVReplicator( - 1, + uint32(res.InsertID), &source, sourceVstreamClient, binlogplayer.NewStats(), - dbTargetClient, - newVtShovelSchemaLoader(), + destDbClient, + mysqld, ) replicator.Replicate(ctx) if err != nil { - log.Infof("Error starting stream: %v", err) + log.Infof("Error with stream: %v", err) } return @@ -172,80 +194,3 @@ type vtShovelDbClient struct { dbClient binlogplayer.DBClient startPos string } - -type vtShovelSchemaLoader struct{} - -func newVtShovelDbClient(dbClient binlogplayer.DBClient, startPos string) binlogplayer.DBClient { - return &vtShovelDbClient{ - dbClient: dbClient, - startPos: startPos, - } -} - -func newVtShovelSchemaLoader() vreplication.SchemasLoader { - return &vtShovelSchemaLoader{} -} - -func (vdc *vtShovelDbClient) DBName() string { - return vdc.dbClient.DBName() -} - -func (vdc *vtShovelDbClient) Connect() error { - return vdc.dbClient.Connect() -} - -func (vdc *vtShovelDbClient) Begin() error { - return vdc.dbClient.Begin() -} - -func (vdc *vtShovelDbClient) Commit() error { - return vdc.dbClient.Commit() -} - -func (vdc *vtShovelDbClient) Rollback() error { - return vdc.dbClient.Rollback() -} - -func (vdc *vtShovelDbClient) Close() { - vdc.dbClient.Close() -} - -func (vdc *vtShovelDbClient) ExecuteFetch(query string, maxrows int) (*sqltypes.Result, error) { - if strings.Contains(query, "from _vt.copy_state") { - dummyResult := &sqltypes.Result{ - Rows: [][]sqltypes.Value{ - []sqltypes.Value{ - sqltypes.NewInt64(0), - }, - }, - } - return dummyResult, nil - } - - if strings.Contains(query, "from _vt.vreplication") { - dummyResult := &sqltypes.Result{ - Rows: [][]sqltypes.Value{ - []sqltypes.Value{ - sqltypes.NewVarBinary(vdc.startPos), - sqltypes.NewVarBinary(""), // StopPos - sqltypes.NewInt64(10000), // maxTPS - sqltypes.NewInt64(10000), // maxReplicationLag - sqltypes.NewVarBinary("Running"), // state - }, - }, - } - return dummyResult, nil - } - - if strings.Contains(query, "update _vt.vreplication") { - return &sqltypes.Result{}, nil - } - return vdc.dbClient.ExecuteFetch(query, maxrows) -} - -func (vsl *vtShovelSchemaLoader) GetSchema(dbName string, tables, excludeTables []string, includeViews bool) (*tabletmanagerdatapb.SchemaDefinition, error) { - // TODO: This will only work for stament based replication. - return &tabletmanagerdatapb.SchemaDefinition{ - TableDefinitions: []*tabletmanagerdatapb.TableDefinition{}, - }, nil -} diff --git a/go/vt/vttablet/tabletmanager/vreplication/vcopier.go b/go/vt/vttablet/tabletmanager/vreplication/vcopier.go index 2bf53348039..1785404d29e 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vcopier.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vcopier.go @@ -38,11 +38,11 @@ import ( ) type vcopier struct { - vr *vreplicator + vr *VReplicator tablePlan *TablePlan } -func newVCopier(vr *vreplicator) *vcopier { +func newVCopier(vr *VReplicator) *vcopier { return &vcopier{ vr: vr, } diff --git a/go/vt/vttablet/tabletmanager/vreplication/vplayer.go b/go/vt/vttablet/tabletmanager/vreplication/vplayer.go index 4deb5cfab08..4e5d6872190 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vplayer.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vplayer.go @@ -36,7 +36,7 @@ import ( ) type vplayer struct { - vr *vreplicator + vr *VReplicator startPos string gtidStartPos mysql.Position stopPos mysql.Position @@ -59,7 +59,7 @@ type vplayer struct { timeOffsetNs int64 } -func newVPlayer(vr *vreplicator, settings binlogplayer.VRSettings, copyState map[string]*sqltypes.Result, pausePos mysql.Position) *vplayer { +func newVPlayer(vr *VReplicator, settings binlogplayer.VRSettings, copyState map[string]*sqltypes.Result, pausePos mysql.Position) *vplayer { saveStop := true if !pausePos.IsZero() { settings.StopPos = pausePos @@ -140,6 +140,9 @@ func (vp *vplayer) fetchAndApply(ctx context.Context) (err error) { cancel() <-streamErr }() + + log.Infof("error applying events: %v", err) + // If the apply thread ends with io.EOF, it means either the Engine // is shutting down and canceled the context, or stop position was reached. // If so, we return nil which will cause the controller to not retry. @@ -168,6 +171,19 @@ func (vp *vplayer) fetchAndApply(ctx context.Context) (err error) { } } +func (vp *vplayer) applyStmtEvent(ctx context.Context, event *binlogdatapb.VEvent) error { + for _, rule := range vp.vr.source.Filter.Rules { + if rule.Filter != "" || rule.Match != "/.*" { + return fmt.Errorf("Filter rules are not supported for SBR replication: %v", rule) + } + + } + if _, err := vp.vr.dbClient.ExecuteWithRetry(ctx, event.Dml); err != nil { + log.Warningf("Fail to run: %v. Got error: %v", event.Dml, err) + } + return nil +} + func (vp *vplayer) applyRowEvent(ctx context.Context, rowEvent *binlogdatapb.RowEvent) error { tplan := vp.tablePlans[rowEvent.TableName] if tplan == nil { @@ -342,14 +358,14 @@ func (vp *vplayer) applyEvent(ctx context.Context, event *binlogdatapb.VEvent, m return err } vp.tablePlans[event.FieldEvent.TableName] = tplan - case binlogdatapb.VEventType_INSERT, binlogdatapb.VEventType_DELETE, binlogdatapb.VEventType_UPDATE: + case binlogdatapb.VEventType_INSERT, binlogdatapb.VEventType_DELETE, binlogdatapb.VEventType_UPDATE, binlogdatapb.VEventType_REPLACE: // This is a player using stament based replication if err := vp.vr.dbClient.Begin(); err != nil { return err } - if _, err := vp.vr.dbClient.ExecuteWithRetry(ctx, event.Dml); err != nil { - log.Warningf("Fail to run: %v. Got error: %v", event.Dml, err) + if err := vp.applyStmtEvent(ctx, event); err != nil { + return err } case binlogdatapb.VEventType_ROW: // This player is configured for row based replicaiton diff --git a/go/vt/vttablet/tabletmanager/vreplication/vreplicator.go b/go/vt/vttablet/tabletmanager/vreplication/vreplicator.go index 97294346722..d64a87f1edd 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vreplicator.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vreplicator.go @@ -44,7 +44,8 @@ var ( replicaLagTolerance = 10 * time.Second ) -type vreplicator struct { +// VReplicator provides the core logic to start vreplication streams +type VReplicator struct { id uint32 dbClient *vdbClient // source @@ -59,13 +60,14 @@ type vreplicator struct { tableKeys map[string][]string } +// SchemasLoader provides a way to load schemas for a vreplicator type SchemasLoader interface { GetSchema(dbName string, tables, excludeTables []string, includeViews bool) (*tabletmanagerdatapb.SchemaDefinition, error) } // NewVReplicator creates a new vreplicator -func NewVReplicator(id uint32, source *binlogdatapb.BinlogSource, sourceVStreamer VStreamerClient, stats *binlogplayer.Stats, dbClient binlogplayer.DBClient, sl SchemasLoader) *vreplicator { - return &vreplicator{ +func NewVReplicator(id uint32, source *binlogdatapb.BinlogSource, sourceVStreamer VStreamerClient, stats *binlogplayer.Stats, dbClient binlogplayer.DBClient, sl SchemasLoader) *VReplicator { + return &VReplicator{ id: id, source: source, sourceVStreamer: sourceVStreamer, @@ -75,7 +77,8 @@ func NewVReplicator(id uint32, source *binlogdatapb.BinlogSource, sourceVStreame } } -func (vr *vreplicator) Replicate(ctx context.Context) error { +// Replicate starts a vreplication stream. +func (vr *VReplicator) Replicate(ctx context.Context) error { tableKeys, err := vr.buildTableKeys() if err != nil { return err @@ -114,7 +117,7 @@ func (vr *vreplicator) Replicate(ctx context.Context) error { } } -func (vr *vreplicator) buildTableKeys() (map[string][]string, error) { +func (vr *VReplicator) buildTableKeys() (map[string][]string, error) { schema, err := vr.sl.GetSchema(vr.dbClient.DBName(), []string{"/.*/"}, nil, false) if err != nil { return nil, err @@ -130,7 +133,7 @@ func (vr *vreplicator) buildTableKeys() (map[string][]string, error) { return tableKeys, nil } -func (vr *vreplicator) readSettings(ctx context.Context) (settings binlogplayer.VRSettings, numTablesToCopy int64, err error) { +func (vr *VReplicator) readSettings(ctx context.Context) (settings binlogplayer.VRSettings, numTablesToCopy int64, err error) { settings, err = binlogplayer.ReadVRSettings(vr.dbClient, vr.id) if err != nil { return settings, numTablesToCopy, fmt.Errorf("error reading VReplication settings: %v", err) @@ -165,7 +168,7 @@ func (vr *vreplicator) readSettings(ctx context.Context) (settings binlogplayer. return settings, numTablesToCopy, nil } -func (vr *vreplicator) setMessage(message string) error { +func (vr *VReplicator) setMessage(message string) error { vr.stats.History.Add(&binlogplayer.StatsHistoryRecord{ Time: time.Now(), Message: message, @@ -177,7 +180,7 @@ func (vr *vreplicator) setMessage(message string) error { return nil } -func (vr *vreplicator) setState(state, message string) error { +func (vr *VReplicator) setState(state, message string) error { return binlogplayer.SetVReplicationState(vr.dbClient, vr.id, state, message) } diff --git a/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go b/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go index ac889024b1c..6c80962a7f2 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go +++ b/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go @@ -99,6 +99,13 @@ func (plan *Plan) filter(values []sqltypes.Value) (bool, []sqltypes.Value, error return true, result, nil } +func mustSendStmt(query mysql.Query, dbname string) bool { + if query.Database != "" && query.Database != dbname { + return false + } + return true +} + func mustSendDDL(query mysql.Query, dbname string, filter *binlogdatapb.Filter) bool { if query.Database != "" && query.Database != dbname { return false diff --git a/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go b/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go index 1e08f77cca9..67d4772d9f7 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go +++ b/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go @@ -137,7 +137,7 @@ func (vs *vstreamer) Stream() error { return wrapError(err, vs.startPos) } err = vs.parseEvents(vs.ctx, events) - return wrapError(err, vs.startPos) + return wrapError(err, vs.pos.String()) } func (vs *vstreamer) parseEvents(ctx context.Context, events <-chan mysql.BinlogEvent) error { @@ -163,7 +163,7 @@ func (vs *vstreamer) parseEvents(ctx context.Context, events <-chan mysql.Binlog bufferedEvents = nil curSize = 0 return vs.send(vevents) - case binlogdatapb.VEventType_INSERT, binlogdatapb.VEventType_DELETE, binlogdatapb.VEventType_UPDATE: + case binlogdatapb.VEventType_INSERT, binlogdatapb.VEventType_DELETE, binlogdatapb.VEventType_UPDATE, binlogdatapb.VEventType_REPLACE: newSize := len(vevent.GetDml()) if curSize+newSize > *PacketSize { vevents := bufferedEvents @@ -315,22 +315,41 @@ func (vs *vstreamer) parseEvent(ev mysql.BinlogEvent) ([]*binlogdatapb.VEvent, e if err != nil { return nil, fmt.Errorf("can't get query from binlog event: %v, event data: %#v", err, ev) } + // Insert/Delete/Update are supported are in here only to have support for vtshovel with + // SBR streams. Vitess itself should never run into cases where it needs to consume non rbr statements. switch cat := sqlparser.Preview(q.SQL); cat { case sqlparser.StmtInsert: - vevents = append(vevents, &binlogdatapb.VEvent{ - Type: binlogdatapb.VEventType_INSERT, - Dml: q.SQL, - }) + mustSend := mustSendStmt(q, vs.cp.DbName) + if mustSend { + vevents = append(vevents, &binlogdatapb.VEvent{ + Type: binlogdatapb.VEventType_INSERT, + Dml: q.SQL, + }) + } case sqlparser.StmtUpdate: - vevents = append(vevents, &binlogdatapb.VEvent{ - Type: binlogdatapb.VEventType_UPDATE, - Dml: q.SQL, - }) + mustSend := mustSendStmt(q, vs.cp.DbName) + if mustSend { + vevents = append(vevents, &binlogdatapb.VEvent{ + Type: binlogdatapb.VEventType_UPDATE, + Dml: q.SQL, + }) + } case sqlparser.StmtDelete: - vevents = append(vevents, &binlogdatapb.VEvent{ - Type: binlogdatapb.VEventType_DELETE, - Dml: q.SQL, - }) + mustSend := mustSendStmt(q, vs.cp.DbName) + if mustSend { + vevents = append(vevents, &binlogdatapb.VEvent{ + Type: binlogdatapb.VEventType_DELETE, + Dml: q.SQL, + }) + } + case sqlparser.StmtReplace: + mustSend := mustSendStmt(q, vs.cp.DbName) + if mustSend { + vevents = append(vevents, &binlogdatapb.VEvent{ + Type: binlogdatapb.VEventType_REPLACE, + Dml: q.SQL, + }) + } case sqlparser.StmtBegin: vevents = append(vevents, &binlogdatapb.VEvent{ Type: binlogdatapb.VEventType_BEGIN, From b56bf67d7ad953540e1369fab53e433a05536a23 Mon Sep 17 00:00:00 2001 From: Rafael Chacon Date: Thu, 17 Oct 2019 16:27:56 -0700 Subject: [PATCH 09/28] Update approach to not require another binary to run vtshovel Signed-off-by: Rafael Chacon --- go/cmd/vtshovel/vtshovel.go | 196 --------------- go/vt/dbconfigs/dbconfigs.go | 25 +- go/vt/proto/binlogdata/binlogdata.pb.go | 226 +++++++++--------- go/vt/vttablet/tabletmanager/action_agent.go | 9 +- .../tabletmanager/vreplication/controller.go | 52 ++-- .../vreplication/controller_test.go | 16 +- .../tabletmanager/vreplication/engine.go | 32 +-- .../tabletmanager/vreplication/engine_test.go | 16 +- .../vreplication/framework_test.go | 2 +- .../tabletserver/vstreamer/vstreamer.go | 4 +- proto/binlogdata.proto | 4 + py/vtproto/binlogdata_pb2.py | 71 +++--- 12 files changed, 254 insertions(+), 399 deletions(-) delete mode 100644 go/cmd/vtshovel/vtshovel.go diff --git a/go/cmd/vtshovel/vtshovel.go b/go/cmd/vtshovel/vtshovel.go deleted file mode 100644 index 5a70f239a10..00000000000 --- a/go/cmd/vtshovel/vtshovel.go +++ /dev/null @@ -1,196 +0,0 @@ -/* -Copyright 2017 Google Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "context" - "encoding/json" - "flag" - "io/ioutil" - "math/rand" - "time" - - "vitess.io/vitess/go/exit" - "vitess.io/vitess/go/mysql" - "vitess.io/vitess/go/vt/binlog/binlogplayer" - "vitess.io/vitess/go/vt/dbconfigs" - "vitess.io/vitess/go/vt/log" - "vitess.io/vitess/go/vt/mysqlctl" - "vitess.io/vitess/go/vt/servenv" - "vitess.io/vitess/go/vt/vterrors" - "vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication" - - binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" -) - -var ( - vtShovelConfigFile = flag.String("vtshovel-config-file", "/etc/slack.d/vtshovel.json", "VTShovel Config file") - dryRun = flag.Bool("dry-run", false, "When present, only log DML that are going to be performed in target database") -) - -func init() { - rand.Seed(time.Now().UnixNano()) - servenv.RegisterDefaultFlags() -} - -// VtShovelConfig fields to configure vtshovel client -type VtShovelConfig struct { - // Source MySQL client information - // MySQLSourceHost ... - MySQLSourceHost string `json:"mysql_source_host"` - // MySQLSourcePort ... - MySQLSourcePort int `json:"mysql_source_port"` - // MySQLSourceUser ... - MySQLSourceUser string `json:"mysql_source_user"` - // MySQLSourcePassword ... - MySQLSourcePassword string `json:"mysql_source_password"` - // MySQLSourceBinlogStartPos ... - MySQLSourceBinlogStartPos string `json:"mysql_source_binlog_start_pos"` - // MySQLSourceDatabase ... - MySQLSourceDBName string `json:"mysql_source_dbname"` -} - -func main() { - defer exit.Recover() - - dbconfigs.RegisterFlags(dbconfigs.Dba) - mysqlctl.RegisterFlags() - - servenv.ParseFlags("vtshovel") - servenv.Init() - - servenv.OnRun(func() { - //vreplication.MySQLAddStatusPart() - // Flags are parsed now. Parse the template using the actual flag value and overwrite the current template. - //addStatusParts(vtg) - }) - - vtShovelConfig, err := loadConfigFromFile(*vtShovelConfigFile) - if err != nil { - log.Fatal(err) - } - - sourceConnParams := mysql.ConnParams{ - Host: vtShovelConfig.MySQLSourceHost, - Port: vtShovelConfig.MySQLSourcePort, - Pass: vtShovelConfig.MySQLSourcePassword, - Uname: vtShovelConfig.MySQLSourceUser, - DbName: vtShovelConfig.MySQLSourceDBName, - } - - source := binlogdatapb.BinlogSource{ - Filter: &binlogdatapb.Filter{ - Rules: []*binlogdatapb.Rule{ - &binlogdatapb.Rule{ - Match: "/.*", - }, - }, - }, - } - - var mycnf *mysqlctl.Mycnf - var socketFile string - // If no connection parameters were specified, load the mycnf file - // and use the socket from it. If connection parameters were specified, - // we assume that the mysql is not local, and we skip loading mycnf. - // This also means that backup and restore will not be allowed. - if !dbconfigs.HasConnectionParams() { - var err error - if mycnf, err = mysqlctl.NewMycnfFromFlags(123213123); err != nil { - log.Exitf("mycnf read failed: %v", err) - } - socketFile = mycnf.SocketFile - } else { - log.Info("connection parameters were specified. Not loading my.cnf.") - } - - // If connection parameters were specified, socketFile will be empty. - // Otherwise, the socketFile (read from mycnf) will be used to initialize - // dbconfigs. - dbcfgs, err := dbconfigs.Init(socketFile) - if err != nil { - log.Warning(err) - } - - mysqld := mysqlctl.NewMysqld(dbcfgs) - servenv.OnClose(mysqld.Close) - - destConnParams := dbcfgs.Dba() - // Hack to make sure dbname is set correctly given that this is not a tablet - // and SetDBName is not called. - destConnParams.DbName = destConnParams.DeprecatedDBName - - log.Infof("This are the destConnParams:%v", destConnParams) - destDbClient := binlogplayer.NewDBClient(destConnParams) - - if err := destDbClient.Connect(); err != nil { - log.Fatal(vterrors.Wrap(err, "can't connect to database")) - } - servenv.OnClose(destDbClient.Close) - - for _, query := range binlogplayer.CreateVReplicationTable() { - if _, err := destDbClient.ExecuteFetch(query, 0); err != nil { - log.Fatalf("Failed to ensure vreplication table exists: %v", err) - } - } - - newVReplicatorStmt := binlogplayer.CreateVReplication("VTshovel", &source, vtShovelConfig.MySQLSourceBinlogStartPos, int64(1000), int64(100000), time.Now().Unix(), destDbClient.DBName()) - - res, err := destDbClient.ExecuteFetch(newVReplicatorStmt, 0) - if err != nil { - log.Fatalf("Failed to create vreplication stream: %v", err) - } - - sourceVstreamClient := vreplication.NewMySQLVStreamerClient(&sourceConnParams) - - go func() { - ctx := context.Background() - replicator := vreplication.NewVReplicator( - uint32(res.InsertID), - &source, - sourceVstreamClient, - binlogplayer.NewStats(), - destDbClient, - mysqld, - ) - replicator.Replicate(ctx) - if err != nil { - log.Infof("Error with stream: %v", err) - - } - return - }() - servenv.RunDefault() -} - -func loadConfigFromFile(file string) (*VtShovelConfig, error) { - data, err := ioutil.ReadFile(file) - if err != nil { - return nil, vterrors.Wrapf(err, "Failed to read %v file", file) - } - vtShovelConfig := &VtShovelConfig{} - err = json.Unmarshal(data, vtShovelConfig) - if err != nil { - return nil, vterrors.Wrap(err, "Error parsing auth server config") - } - return vtShovelConfig, nil -} - -type vtShovelDbClient struct { - dbClient binlogplayer.DBClient - startPos string -} diff --git a/go/vt/dbconfigs/dbconfigs.go b/go/vt/dbconfigs/dbconfigs.go index c15ad1e6415..af675779b18 100644 --- a/go/vt/dbconfigs/dbconfigs.go +++ b/go/vt/dbconfigs/dbconfigs.go @@ -69,14 +69,15 @@ const ( // AllPrivs user should have more privileges than App (should include possibility to do // schema changes and write to internal Vitess tables), but it shouldn't have SUPER // privilege like Dba has. - AllPrivs = "allprivs" - Dba = "dba" - Filtered = "filtered" - Repl = "repl" + AllPrivs = "allprivs" + Dba = "dba" + Filtered = "filtered" + Repl = "repl" + ExternalRepl = "erepl" ) // All can be used to register all flags: RegisterFlags(All...) -var All = []string{App, AppDebug, AllPrivs, Dba, Filtered, Repl} +var All = []string{App, AppDebug, AllPrivs, Dba, Filtered, Repl, ExternalRepl} // RegisterFlags registers the flags for the given DBConfigFlag. // For instance, vttablet will register client, dba and repl. @@ -157,16 +158,26 @@ func (dbcfgs *DBConfigs) DbaWithDB() *mysql.ConnParams { return dbcfgs.makeParams(Dba, true) } -// FilteredWithDB returns connection parameters for appdebug with dbname set. +// FilteredWithDB returns connection parameters for filtered with dbname set. func (dbcfgs *DBConfigs) FilteredWithDB() *mysql.ConnParams { return dbcfgs.makeParams(Filtered, true) } -// Repl returns connection parameters for appdebug with no dbname set. +// Repl returns connection parameters for repl with no dbname set. func (dbcfgs *DBConfigs) Repl() *mysql.ConnParams { return dbcfgs.makeParams(Repl, false) } +// ExternalRepl returns connection parameters for repl with no dbname set. +func (dbcfgs *DBConfigs) ExternalRepl() *mysql.ConnParams { + return dbcfgs.makeParams(Repl, false) +} + +// ExternalReplWithDb returns connection parameters for repl with dbname set. +func (dbcfgs *DBConfigs) ExternalReplWithDb() *mysql.ConnParams { + return dbcfgs.makeParams(Repl, true) +} + // AppWithDB returns connection parameters for app with dbname set. func (dbcfgs *DBConfigs) makeParams(userKey string, withDB bool) *mysql.ConnParams { orig := dbcfgs.userConfigs[userKey] diff --git a/go/vt/proto/binlogdata/binlogdata.pb.go b/go/vt/proto/binlogdata/binlogdata.pb.go index 454658c4f24..e0f0d0f1d23 100644 --- a/go/vt/proto/binlogdata/binlogdata.pb.go +++ b/go/vt/proto/binlogdata/binlogdata.pb.go @@ -715,10 +715,13 @@ type BinlogSource struct { // for the filter. Filter *Filter `protobuf:"bytes,6,opt,name=filter,proto3" json:"filter,omitempty"` // on_ddl specifies the action to be taken when a DDL is encountered. - OnDdl OnDDLAction `protobuf:"varint,7,opt,name=on_ddl,json=onDdl,proto3,enum=binlogdata.OnDDLAction" json:"on_ddl,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + OnDdl OnDDLAction `protobuf:"varint,7,opt,name=on_ddl,json=onDdl,proto3,enum=binlogdata.OnDDLAction" json:"on_ddl,omitempty"` + // Source is an external mysql. This attribute should be set to the username + // to use in the connection + ExternalMysql string `protobuf:"bytes,8,opt,name=external_mysql,json=externalMysql,proto3" json:"external_mysql,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *BinlogSource) Reset() { *m = BinlogSource{} } @@ -795,6 +798,13 @@ func (m *BinlogSource) GetOnDdl() OnDDLAction { return OnDDLAction_IGNORE } +func (m *BinlogSource) GetExternalMysql() string { + if m != nil { + return m.ExternalMysql + } + return "" +} + // RowChange represents one row change type RowChange struct { Before *query.Row `protobuf:"bytes,1,opt,name=before,proto3" json:"before,omitempty"` @@ -1568,107 +1578,109 @@ func init() { func init() { proto.RegisterFile("binlogdata.proto", fileDescriptor_5fd02bcb2e350dad) } var fileDescriptor_5fd02bcb2e350dad = []byte{ - // 1630 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xd4, 0x57, 0xcb, 0x72, 0xf3, 0x48, - 0x15, 0x8e, 0x2d, 0xf9, 0x76, 0x94, 0x38, 0x4a, 0xe7, 0x82, 0x49, 0x31, 0x54, 0x46, 0xc5, 0x90, - 0x90, 0x2a, 0x1c, 0x30, 0xf0, 0xb3, 0x1a, 0x06, 0x5f, 0x94, 0xc4, 0x89, 0x6c, 0xe7, 0x6f, 0x2b, - 0x19, 0x6a, 0x36, 0x2a, 0xc5, 0xea, 0x24, 0x22, 0xb2, 0xe4, 0x5f, 0x6a, 0x3b, 0xe4, 0x01, 0x28, - 0x1e, 0x80, 0x2d, 0x2f, 0xc0, 0x1a, 0xb6, 0x6c, 0xd9, 0xf3, 0x10, 0x3c, 0x00, 0x6f, 0x40, 0xf5, - 0x45, 0xb2, 0x95, 0x0c, 0xf3, 0x67, 0xa8, 0x62, 0xc1, 0x46, 0x75, 0xfa, 0xf4, 0x39, 0xa7, 0xcf, - 0xf9, 0xfa, 0x3b, 0xdd, 0x6a, 0xd0, 0x6f, 0xfd, 0x30, 0x88, 0xee, 0x3d, 0x97, 0xba, 0xcd, 0x59, - 0x1c, 0xd1, 0x08, 0xc1, 0x52, 0xb3, 0xaf, 0x2d, 0x68, 0x3c, 0x9b, 0x88, 0x89, 0x7d, 0xed, 0xc3, - 0x9c, 0xc4, 0xcf, 0x72, 0x50, 0xa7, 0xd1, 0x2c, 0x5a, 0x7a, 0x19, 0x03, 0xa8, 0x74, 0x1f, 0xdc, - 0x38, 0x21, 0x14, 0xed, 0x41, 0x79, 0x12, 0xf8, 0x24, 0xa4, 0x8d, 0xc2, 0x41, 0xe1, 0xa8, 0x84, - 0xe5, 0x08, 0x21, 0x50, 0x27, 0x51, 0x18, 0x36, 0x8a, 0x5c, 0xcb, 0x65, 0x66, 0x9b, 0x90, 0x78, - 0x41, 0xe2, 0x86, 0x22, 0x6c, 0xc5, 0xc8, 0xf8, 0xa7, 0x02, 0x5b, 0x1d, 0x9e, 0x87, 0x1d, 0xbb, - 0x61, 0xe2, 0x4e, 0xa8, 0x1f, 0x85, 0xe8, 0x0c, 0x20, 0xa1, 0x2e, 0x25, 0x53, 0x12, 0xd2, 0xa4, - 0x51, 0x38, 0x50, 0x8e, 0xb4, 0xd6, 0x61, 0x73, 0xa5, 0x82, 0x57, 0x2e, 0xcd, 0x71, 0x6a, 0x8f, - 0x57, 0x5c, 0x51, 0x0b, 0x34, 0xb2, 0x20, 0x21, 0x75, 0x68, 0xf4, 0x48, 0xc2, 0x86, 0x7a, 0x50, - 0x38, 0xd2, 0x5a, 0x5b, 0x4d, 0x51, 0xa0, 0xc9, 0x66, 0x6c, 0x36, 0x81, 0x81, 0x64, 0xf2, 0xfe, - 0xdf, 0x8b, 0x50, 0xcb, 0xa2, 0x21, 0x0b, 0xaa, 0x13, 0x97, 0x92, 0xfb, 0x28, 0x7e, 0xe6, 0x65, - 0xd6, 0x5b, 0x3f, 0x79, 0x63, 0x22, 0xcd, 0xae, 0xf4, 0xc3, 0x59, 0x04, 0xf4, 0x63, 0xa8, 0x4c, - 0x04, 0x7a, 0x1c, 0x1d, 0xad, 0xb5, 0xbd, 0x1a, 0x4c, 0x02, 0x8b, 0x53, 0x1b, 0xa4, 0x83, 0x92, - 0x7c, 0x08, 0x38, 0x64, 0xeb, 0x98, 0x89, 0xc6, 0x9f, 0x0b, 0x50, 0x4d, 0xe3, 0xa2, 0x6d, 0xd8, - 0xec, 0x58, 0xce, 0xf5, 0x10, 0x9b, 0xdd, 0xd1, 0xd9, 0xb0, 0xff, 0x95, 0xd9, 0xd3, 0xd7, 0xd0, - 0x3a, 0x54, 0x3b, 0x96, 0xd3, 0x31, 0xcf, 0xfa, 0x43, 0xbd, 0x80, 0x36, 0xa0, 0xd6, 0xb1, 0x9c, - 0xee, 0x68, 0x30, 0xe8, 0xdb, 0x7a, 0x11, 0x6d, 0x82, 0xd6, 0xb1, 0x1c, 0x3c, 0xb2, 0xac, 0x4e, - 0xbb, 0x7b, 0xa9, 0x2b, 0x68, 0x17, 0xb6, 0x3a, 0x96, 0xd3, 0x1b, 0x58, 0x4e, 0xcf, 0xbc, 0xc2, - 0x66, 0xb7, 0x6d, 0x9b, 0x3d, 0x5d, 0x45, 0x00, 0x65, 0xa6, 0xee, 0x59, 0x7a, 0x49, 0xca, 0x63, - 0xd3, 0xd6, 0xcb, 0x32, 0x5c, 0x7f, 0x38, 0x36, 0xb1, 0xad, 0x57, 0xe4, 0xf0, 0xfa, 0xaa, 0xd7, - 0xb6, 0x4d, 0xbd, 0x2a, 0x87, 0x3d, 0xd3, 0x32, 0x6d, 0x53, 0xaf, 0x5d, 0xa8, 0xd5, 0xa2, 0xae, - 0x5c, 0xa8, 0x55, 0x45, 0x57, 0x8d, 0x3f, 0x16, 0x60, 0x77, 0x4c, 0x63, 0xe2, 0x4e, 0x2f, 0xc9, - 0x33, 0x76, 0xc3, 0x7b, 0x82, 0xc9, 0x87, 0x39, 0x49, 0x28, 0xda, 0x87, 0xea, 0x2c, 0x4a, 0x7c, - 0x86, 0x1d, 0x07, 0xb8, 0x86, 0xb3, 0x31, 0x3a, 0x81, 0xda, 0x23, 0x79, 0x76, 0x62, 0x66, 0x2f, - 0x01, 0x43, 0xcd, 0x8c, 0x90, 0x59, 0xa4, 0xea, 0xa3, 0x94, 0x56, 0xf1, 0x55, 0x3e, 0x8e, 0xaf, - 0x71, 0x07, 0x7b, 0x2f, 0x93, 0x4a, 0x66, 0x51, 0x98, 0x10, 0x64, 0x01, 0x12, 0x8e, 0x0e, 0x5d, - 0xee, 0x2d, 0xcf, 0x4f, 0x6b, 0x7d, 0xf2, 0x8d, 0x04, 0xc0, 0x5b, 0xb7, 0x2f, 0x55, 0xc6, 0xef, - 0x60, 0x5b, 0xac, 0x63, 0xbb, 0xb7, 0x01, 0x49, 0xde, 0x52, 0xfa, 0x1e, 0x94, 0x29, 0x37, 0x6e, - 0x14, 0x0f, 0x94, 0xa3, 0x1a, 0x96, 0xa3, 0x6f, 0x5b, 0xa1, 0x07, 0x3b, 0xf9, 0x95, 0xff, 0x27, - 0xf5, 0xfd, 0x1c, 0x54, 0x3c, 0x0f, 0x08, 0xda, 0x81, 0xd2, 0xd4, 0xa5, 0x93, 0x07, 0x59, 0x8d, - 0x18, 0xb0, 0x52, 0xee, 0xfc, 0x80, 0x92, 0x98, 0x6f, 0x61, 0x0d, 0xcb, 0x91, 0xf1, 0x97, 0x02, - 0x94, 0x4f, 0xb9, 0x88, 0x7e, 0x08, 0xa5, 0x78, 0xce, 0x8a, 0x15, 0xbd, 0xae, 0xaf, 0x66, 0xc0, - 0x22, 0x63, 0x31, 0x8d, 0xfa, 0x50, 0xbf, 0xf3, 0x49, 0xe0, 0xf1, 0xd6, 0x1d, 0x44, 0x9e, 0x60, - 0x45, 0xbd, 0xf5, 0xe9, 0xaa, 0x83, 0x88, 0xd9, 0x3c, 0xcd, 0x19, 0xe2, 0x17, 0x8e, 0xc6, 0x3b, - 0xa8, 0xe7, 0x2d, 0x58, 0x3b, 0x99, 0x18, 0x3b, 0xa3, 0xa1, 0x33, 0xe8, 0x8f, 0x07, 0x6d, 0xbb, - 0x7b, 0xae, 0xaf, 0xf1, 0x8e, 0x31, 0xc7, 0xb6, 0x63, 0x9e, 0x9e, 0x8e, 0xb0, 0xad, 0x17, 0x8c, - 0x3f, 0x15, 0x61, 0x5d, 0x80, 0x32, 0x8e, 0xe6, 0xf1, 0x84, 0xb0, 0x5d, 0x7c, 0x24, 0xcf, 0xc9, - 0xcc, 0x9d, 0x90, 0x74, 0x17, 0xd3, 0x31, 0x03, 0x24, 0x79, 0x70, 0x63, 0x4f, 0x56, 0x2e, 0x06, - 0xe8, 0x17, 0xa0, 0xf1, 0xdd, 0xa4, 0x0e, 0x7d, 0x9e, 0x11, 0xbe, 0x8f, 0xf5, 0xd6, 0xce, 0x92, - 0xd8, 0x7c, 0xaf, 0xa8, 0xfd, 0x3c, 0x23, 0x18, 0x68, 0x26, 0xe7, 0xbb, 0x41, 0x7d, 0x43, 0x37, - 0x2c, 0x39, 0x54, 0xca, 0x71, 0xe8, 0x38, 0xdb, 0x90, 0xb2, 0x8c, 0xf2, 0x0a, 0xbd, 0x74, 0x93, - 0x50, 0x13, 0xca, 0x51, 0xe8, 0x78, 0x5e, 0xd0, 0xa8, 0xf0, 0x34, 0xbf, 0xb3, 0x6a, 0x3b, 0x0a, - 0x7b, 0x3d, 0xab, 0x2d, 0x68, 0x51, 0x8a, 0xc2, 0x9e, 0x17, 0x18, 0xef, 0xa1, 0x86, 0xa3, 0xa7, - 0xee, 0x03, 0x4f, 0xc0, 0x80, 0xf2, 0x2d, 0xb9, 0x8b, 0x62, 0x22, 0x99, 0x05, 0xf2, 0xe4, 0xc5, - 0xd1, 0x13, 0x96, 0x33, 0xe8, 0x00, 0x4a, 0xee, 0x5d, 0x4a, 0x8e, 0xbc, 0x89, 0x98, 0x30, 0x5c, - 0xa8, 0xe2, 0xe8, 0x89, 0xef, 0x13, 0xfa, 0x04, 0x04, 0x22, 0x4e, 0xe8, 0x4e, 0x53, 0xb8, 0x6b, - 0x5c, 0x33, 0x74, 0xa7, 0x04, 0xbd, 0x03, 0x2d, 0x8e, 0x9e, 0x9c, 0x09, 0x5f, 0x5e, 0xb4, 0x8e, - 0xd6, 0xda, 0xcd, 0xb1, 0x29, 0x4d, 0x0e, 0x43, 0x9c, 0x8a, 0x89, 0xf1, 0x1e, 0x60, 0x49, 0x86, - 0x8f, 0x2d, 0xf2, 0x03, 0x06, 0x1f, 0x09, 0xbc, 0x34, 0xfe, 0xba, 0x4c, 0x99, 0x47, 0xc0, 0x72, - 0x8e, 0x01, 0x31, 0x66, 0xbb, 0x7d, 0x46, 0x7d, 0xef, 0xbf, 0xe0, 0x08, 0x02, 0xf5, 0x9e, 0xfa, - 0x1e, 0x27, 0x47, 0x0d, 0x73, 0xd9, 0xf8, 0x02, 0x4a, 0x37, 0x3c, 0xdc, 0x3b, 0xd0, 0xb8, 0x95, - 0xc3, 0xd4, 0x69, 0xd3, 0xe4, 0xca, 0xcc, 0x96, 0xc6, 0x90, 0xa4, 0x62, 0x62, 0xb4, 0x61, 0xe3, - 0x52, 0x2e, 0xcb, 0x0d, 0xbe, 0x7d, 0x5e, 0xc6, 0x5f, 0x8b, 0x50, 0xb9, 0x88, 0xe6, 0x71, 0xe8, - 0x06, 0xa8, 0x0e, 0x45, 0xdf, 0xe3, 0x7e, 0x0a, 0x2e, 0xfa, 0x1e, 0xfa, 0x35, 0xd4, 0xa7, 0xfe, - 0x7d, 0xec, 0x32, 0x3e, 0x08, 0x6a, 0x8b, 0xee, 0xfc, 0xee, 0x6a, 0x66, 0x83, 0xd4, 0x82, 0xf3, - 0x7b, 0x63, 0xba, 0x3a, 0x5c, 0x61, 0xac, 0x92, 0x63, 0xec, 0x67, 0x50, 0x0f, 0xa2, 0x89, 0x1b, - 0x38, 0xd9, 0x79, 0xa9, 0xf2, 0xa4, 0x36, 0xb8, 0xf6, 0x2a, 0x3d, 0x34, 0x5f, 0xe0, 0x52, 0x7a, - 0x23, 0x2e, 0xe8, 0x73, 0x58, 0x9f, 0xb9, 0x31, 0xf5, 0x27, 0xfe, 0xcc, 0x65, 0x7f, 0x1c, 0x65, - 0xee, 0x98, 0x4b, 0x3b, 0x87, 0x1b, 0xce, 0x99, 0xa3, 0x4f, 0x61, 0x3d, 0x26, 0x0b, 0x12, 0x27, - 0xc4, 0x73, 0xd8, 0xba, 0x95, 0x03, 0xe5, 0x48, 0xc1, 0x5a, 0xaa, 0xeb, 0x7b, 0x89, 0xf1, 0xaf, - 0x22, 0x94, 0x6f, 0x04, 0xbb, 0x8e, 0x41, 0xe5, 0xd8, 0x88, 0xbf, 0x89, 0xbd, 0xd5, 0x45, 0x84, - 0x05, 0x07, 0x86, 0xdb, 0xa0, 0xef, 0x41, 0x8d, 0xfa, 0x53, 0x92, 0x50, 0x77, 0x3a, 0xe3, 0x60, - 0x2a, 0x78, 0xa9, 0xf8, 0x3a, 0x8e, 0xb0, 0x5f, 0x06, 0xd6, 0xac, 0x02, 0x1e, 0x26, 0xa2, 0x9f, - 0x42, 0x8d, 0xf5, 0x04, 0xff, 0xc3, 0x69, 0x94, 0x78, 0x93, 0xed, 0xbc, 0xe8, 0x08, 0xbe, 0x2c, - 0xae, 0xc6, 0x69, 0x97, 0xfd, 0x12, 0x34, 0xce, 0x62, 0xe9, 0x24, 0x4e, 0x89, 0xbd, 0xfc, 0x29, - 0x91, 0x76, 0x0b, 0x86, 0xe5, 0xc1, 0x8a, 0x0e, 0xa1, 0xb4, 0xe0, 0x29, 0x55, 0xe4, 0x9f, 0xd6, - 0x6a, 0x71, 0x1c, 0x76, 0x31, 0xcf, 0xae, 0xb1, 0xdf, 0x0a, 0x16, 0x35, 0xaa, 0xaf, 0xaf, 0x31, - 0x49, 0x30, 0x9c, 0xda, 0xf0, 0xaa, 0xa6, 0x41, 0xa3, 0x26, 0xab, 0x9a, 0x06, 0x0c, 0xf3, 0xc9, - 0x3c, 0x8e, 0xf9, 0xbf, 0x9d, 0x3f, 0x25, 0x8d, 0x1d, 0x0e, 0x8e, 0x26, 0x75, 0xb6, 0x3f, 0x25, - 0xc6, 0x1f, 0x8a, 0x50, 0xbf, 0x11, 0xb7, 0x5f, 0x7a, 0xe3, 0x7e, 0x01, 0xdb, 0xe4, 0xee, 0x8e, - 0x4c, 0xa8, 0xbf, 0x20, 0xce, 0xc4, 0x0d, 0x02, 0x12, 0x3b, 0x92, 0xc2, 0x5a, 0x6b, 0xb3, 0x29, - 0xfe, 0x82, 0xbb, 0x5c, 0xdf, 0xef, 0xe1, 0xad, 0xcc, 0x56, 0xaa, 0x3c, 0x64, 0xc2, 0xb6, 0x3f, - 0x9d, 0x12, 0xcf, 0x77, 0xe9, 0x6a, 0x00, 0x71, 0x76, 0xed, 0xca, 0x83, 0xe0, 0xc6, 0x3e, 0x73, - 0x29, 0x59, 0x86, 0xc9, 0x3c, 0xb2, 0x30, 0x9f, 0x31, 0x9e, 0xc7, 0xf7, 0xd9, 0x25, 0xbe, 0x21, - 0x3d, 0x6d, 0xae, 0xc4, 0x72, 0x32, 0xf7, 0x83, 0xa0, 0xbe, 0xf8, 0x41, 0x58, 0x1e, 0xe2, 0xa5, - 0x8f, 0x1d, 0xe2, 0xc6, 0xe7, 0xb0, 0x99, 0x01, 0x21, 0x7f, 0x00, 0x8e, 0xa1, 0xcc, 0x37, 0x37, - 0x3d, 0x3d, 0xd0, 0x6b, 0x1e, 0x62, 0x69, 0x61, 0xfc, 0xbe, 0x08, 0x28, 0xf5, 0x8f, 0x9e, 0x92, - 0xff, 0x53, 0x30, 0x77, 0xa0, 0xc4, 0xf5, 0x12, 0x49, 0x31, 0x60, 0x38, 0x04, 0x6e, 0x42, 0x67, - 0x8f, 0x19, 0x8c, 0xc2, 0xf9, 0x3d, 0xfb, 0x62, 0x92, 0xcc, 0x03, 0x8a, 0xa5, 0x85, 0xf1, 0xb7, - 0x02, 0x6c, 0xe7, 0x70, 0x90, 0x58, 0x2e, 0x2f, 0x84, 0xc2, 0x7f, 0xbe, 0x10, 0xd0, 0x11, 0x54, - 0x67, 0x8f, 0xdf, 0x70, 0x71, 0x64, 0xb3, 0x5f, 0xdb, 0xd7, 0xdf, 0x07, 0x35, 0x8e, 0x9e, 0x92, - 0x86, 0xca, 0x3d, 0x57, 0x6f, 0x49, 0xae, 0x67, 0x57, 0x6d, 0xae, 0x8e, 0xdc, 0x55, 0x2b, 0x66, - 0x8e, 0x7f, 0x05, 0xda, 0xca, 0x8d, 0xcd, 0x7e, 0xec, 0xfb, 0x67, 0xc3, 0x11, 0x36, 0xf5, 0x35, - 0x54, 0x05, 0x75, 0x6c, 0x8f, 0xae, 0xf4, 0x02, 0x93, 0xcc, 0xdf, 0x98, 0x5d, 0xf1, 0x58, 0x60, - 0x92, 0x23, 0x8d, 0x94, 0xe3, 0x7f, 0x14, 0x00, 0x96, 0x47, 0x14, 0xd2, 0xa0, 0x72, 0x3d, 0xbc, - 0x1c, 0x8e, 0xbe, 0x1c, 0x8a, 0x00, 0x67, 0x76, 0xbf, 0xa7, 0x17, 0x50, 0x0d, 0x4a, 0xe2, 0xf5, - 0x51, 0x64, 0x2b, 0xc8, 0xa7, 0x87, 0xc2, 0xde, 0x25, 0xd9, 0xbb, 0x43, 0x45, 0x15, 0x50, 0xb2, - 0xd7, 0x85, 0x7c, 0x4e, 0x94, 0x59, 0x40, 0x6c, 0x5e, 0x59, 0xed, 0xae, 0xa9, 0x57, 0xd8, 0x44, - 0xf6, 0xb0, 0x00, 0x28, 0xa7, 0xaf, 0x0a, 0xe6, 0xc9, 0xde, 0x22, 0xc0, 0xd6, 0x19, 0xd9, 0xe7, - 0x26, 0xd6, 0x35, 0xa6, 0xc3, 0xa3, 0x2f, 0xf5, 0x75, 0xa6, 0x3b, 0xed, 0x9b, 0x56, 0x4f, 0xdf, - 0x60, 0x8f, 0x91, 0x73, 0xb3, 0x8d, 0xed, 0x8e, 0xd9, 0xb6, 0xf5, 0x3a, 0x9b, 0xb9, 0xe1, 0x09, - 0x6e, 0xb2, 0x65, 0x2e, 0x46, 0xd7, 0x78, 0xd8, 0xb6, 0x74, 0xfd, 0xf8, 0x10, 0x36, 0x72, 0x37, - 0x12, 0x5b, 0xcb, 0x6e, 0x77, 0x2c, 0x73, 0xac, 0xaf, 0x31, 0x79, 0x7c, 0xde, 0xc6, 0xbd, 0xb1, - 0x5e, 0xe8, 0xfc, 0xe8, 0xab, 0xc3, 0x85, 0x4f, 0x49, 0x92, 0x34, 0xfd, 0xe8, 0x44, 0x48, 0x27, - 0xf7, 0xd1, 0xc9, 0x82, 0x9e, 0xf0, 0x87, 0xf1, 0xc9, 0xb2, 0x7d, 0x6e, 0xcb, 0x5c, 0xf3, 0xb3, - 0x7f, 0x07, 0x00, 0x00, 0xff, 0xff, 0xc1, 0x08, 0x9a, 0x3a, 0x74, 0x0f, 0x00, 0x00, + // 1652 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xd4, 0x57, 0xdd, 0x72, 0xe3, 0x48, + 0x15, 0x1e, 0x5b, 0xf2, 0xdf, 0x51, 0xe2, 0x28, 0x9d, 0x4c, 0x30, 0x53, 0x2c, 0x95, 0x55, 0x31, + 0x4c, 0x48, 0x15, 0x0e, 0x18, 0x18, 0xae, 0x96, 0xc5, 0x3f, 0x4a, 0xc6, 0x13, 0xd9, 0xce, 0xb4, + 0x95, 0x2c, 0xb5, 0x37, 0x2a, 0xc5, 0xea, 0x24, 0x22, 0xfa, 0xf1, 0x48, 0x6d, 0x67, 0xfd, 0x00, + 0x14, 0x0f, 0xc0, 0x53, 0x70, 0x0d, 0xb7, 0x5c, 0x51, 0xc5, 0x3d, 0x0f, 0xc1, 0x03, 0xf0, 0x06, + 0x54, 0xff, 0x48, 0xb6, 0x32, 0xcb, 0x4e, 0x96, 0x2a, 0x2e, 0xf6, 0x46, 0x75, 0xfa, 0xf4, 0x39, + 0xa7, 0xcf, 0xf9, 0xfa, 0xfc, 0xa8, 0x41, 0xbf, 0xf6, 0xa3, 0x20, 0xbe, 0xf5, 0x5c, 0xea, 0xb6, + 0xe7, 0x49, 0x4c, 0x63, 0x04, 0x6b, 0xce, 0x0b, 0x6d, 0x49, 0x93, 0xf9, 0x4c, 0x6c, 0xbc, 0xd0, + 0xde, 0x2f, 0x48, 0xb2, 0x92, 0x8b, 0x26, 0x8d, 0xe7, 0xf1, 0x5a, 0xcb, 0x18, 0x41, 0xad, 0x7f, + 0xe7, 0x26, 0x29, 0xa1, 0xe8, 0x00, 0xaa, 0xb3, 0xc0, 0x27, 0x11, 0x6d, 0x95, 0x0e, 0x4b, 0x47, + 0x15, 0x2c, 0x57, 0x08, 0x81, 0x3a, 0x8b, 0xa3, 0xa8, 0x55, 0xe6, 0x5c, 0x4e, 0x33, 0xd9, 0x94, + 0x24, 0x4b, 0x92, 0xb4, 0x14, 0x21, 0x2b, 0x56, 0xc6, 0xbf, 0x14, 0xd8, 0xed, 0x71, 0x3f, 0xec, + 0xc4, 0x8d, 0x52, 0x77, 0x46, 0xfd, 0x38, 0x42, 0x67, 0x00, 0x29, 0x75, 0x29, 0x09, 0x49, 0x44, + 0xd3, 0x56, 0xe9, 0x50, 0x39, 0xd2, 0x3a, 0xaf, 0xda, 0x1b, 0x11, 0x7c, 0xa0, 0xd2, 0x9e, 0x66, + 0xf2, 0x78, 0x43, 0x15, 0x75, 0x40, 0x23, 0x4b, 0x12, 0x51, 0x87, 0xc6, 0xf7, 0x24, 0x6a, 0xa9, + 0x87, 0xa5, 0x23, 0xad, 0xb3, 0xdb, 0x16, 0x01, 0x9a, 0x6c, 0xc7, 0x66, 0x1b, 0x18, 0x48, 0x4e, + 0xbf, 0xf8, 0x47, 0x19, 0x1a, 0xb9, 0x35, 0x64, 0x41, 0x7d, 0xe6, 0x52, 0x72, 0x1b, 0x27, 0x2b, + 0x1e, 0x66, 0xb3, 0xf3, 0xb3, 0x27, 0x3a, 0xd2, 0xee, 0x4b, 0x3d, 0x9c, 0x5b, 0x40, 0x3f, 0x85, + 0xda, 0x4c, 0xa0, 0xc7, 0xd1, 0xd1, 0x3a, 0x7b, 0x9b, 0xc6, 0x24, 0xb0, 0x38, 0x93, 0x41, 0x3a, + 0x28, 0xe9, 0xfb, 0x80, 0x43, 0xb6, 0x85, 0x19, 0x69, 0xfc, 0xb9, 0x04, 0xf5, 0xcc, 0x2e, 0xda, + 0x83, 0x9d, 0x9e, 0xe5, 0x5c, 0x8e, 0xb1, 0xd9, 0x9f, 0x9c, 0x8d, 0x87, 0x5f, 0x9a, 0x03, 0xfd, + 0x19, 0xda, 0x82, 0x7a, 0xcf, 0x72, 0x7a, 0xe6, 0xd9, 0x70, 0xac, 0x97, 0xd0, 0x36, 0x34, 0x7a, + 0x96, 0xd3, 0x9f, 0x8c, 0x46, 0x43, 0x5b, 0x2f, 0xa3, 0x1d, 0xd0, 0x7a, 0x96, 0x83, 0x27, 0x96, + 0xd5, 0xeb, 0xf6, 0xcf, 0x75, 0x05, 0x3d, 0x87, 0xdd, 0x9e, 0xe5, 0x0c, 0x46, 0x96, 0x33, 0x30, + 0x2f, 0xb0, 0xd9, 0xef, 0xda, 0xe6, 0x40, 0x57, 0x11, 0x40, 0x95, 0xb1, 0x07, 0x96, 0x5e, 0x91, + 0xf4, 0xd4, 0xb4, 0xf5, 0xaa, 0x34, 0x37, 0x1c, 0x4f, 0x4d, 0x6c, 0xeb, 0x35, 0xb9, 0xbc, 0xbc, + 0x18, 0x74, 0x6d, 0x53, 0xaf, 0xcb, 0xe5, 0xc0, 0xb4, 0x4c, 0xdb, 0xd4, 0x1b, 0x6f, 0xd5, 0x7a, + 0x59, 0x57, 0xde, 0xaa, 0x75, 0x45, 0x57, 0x8d, 0x3f, 0x95, 0xe0, 0xf9, 0x94, 0x26, 0xc4, 0x0d, + 0xcf, 0xc9, 0x0a, 0xbb, 0xd1, 0x2d, 0xc1, 0xe4, 0xfd, 0x82, 0xa4, 0x14, 0xbd, 0x80, 0xfa, 0x3c, + 0x4e, 0x7d, 0x86, 0x1d, 0x07, 0xb8, 0x81, 0xf3, 0x35, 0x3a, 0x81, 0xc6, 0x3d, 0x59, 0x39, 0x09, + 0x93, 0x97, 0x80, 0xa1, 0x76, 0x9e, 0x90, 0xb9, 0xa5, 0xfa, 0xbd, 0xa4, 0x36, 0xf1, 0x55, 0x3e, + 0x8e, 0xaf, 0x71, 0x03, 0x07, 0x8f, 0x9d, 0x4a, 0xe7, 0x71, 0x94, 0x12, 0x64, 0x01, 0x12, 0x8a, + 0x0e, 0x5d, 0xdf, 0x2d, 0xf7, 0x4f, 0xeb, 0x7c, 0xf2, 0x8d, 0x09, 0x80, 0x77, 0xaf, 0x1f, 0xb3, + 0x8c, 0xaf, 0x60, 0x4f, 0x9c, 0x63, 0xbb, 0xd7, 0x01, 0x49, 0x9f, 0x12, 0xfa, 0x01, 0x54, 0x29, + 0x17, 0x6e, 0x95, 0x0f, 0x95, 0xa3, 0x06, 0x96, 0xab, 0x6f, 0x1b, 0xa1, 0x07, 0xfb, 0xc5, 0x93, + 0xff, 0x2f, 0xf1, 0xfd, 0x12, 0x54, 0xbc, 0x08, 0x08, 0xda, 0x87, 0x4a, 0xe8, 0xd2, 0xd9, 0x9d, + 0x8c, 0x46, 0x2c, 0x58, 0x28, 0x37, 0x7e, 0x40, 0x49, 0xc2, 0xaf, 0xb0, 0x81, 0xe5, 0xca, 0xf8, + 0x4b, 0x09, 0xaa, 0xa7, 0x9c, 0x44, 0x3f, 0x86, 0x4a, 0xb2, 0x60, 0xc1, 0x8a, 0x5a, 0xd7, 0x37, + 0x3d, 0x60, 0x96, 0xb1, 0xd8, 0x46, 0x43, 0x68, 0xde, 0xf8, 0x24, 0xf0, 0x78, 0xe9, 0x8e, 0x62, + 0x4f, 0x64, 0x45, 0xb3, 0xf3, 0xe9, 0xa6, 0x82, 0xb0, 0xd9, 0x3e, 0x2d, 0x08, 0xe2, 0x47, 0x8a, + 0xc6, 0x6b, 0x68, 0x16, 0x25, 0x58, 0x39, 0x99, 0x18, 0x3b, 0x93, 0xb1, 0x33, 0x1a, 0x4e, 0x47, + 0x5d, 0xbb, 0xff, 0x46, 0x7f, 0xc6, 0x2b, 0xc6, 0x9c, 0xda, 0x8e, 0x79, 0x7a, 0x3a, 0xc1, 0xb6, + 0x5e, 0x32, 0xfe, 0x5e, 0x86, 0x2d, 0x01, 0xca, 0x34, 0x5e, 0x24, 0x33, 0xc2, 0x6e, 0xf1, 0x9e, + 0xac, 0xd2, 0xb9, 0x3b, 0x23, 0xd9, 0x2d, 0x66, 0x6b, 0x06, 0x48, 0x7a, 0xe7, 0x26, 0x9e, 0x8c, + 0x5c, 0x2c, 0xd0, 0xaf, 0x40, 0xe3, 0xb7, 0x49, 0x1d, 0xba, 0x9a, 0x13, 0x7e, 0x8f, 0xcd, 0xce, + 0xfe, 0x3a, 0xb1, 0xf9, 0x5d, 0x51, 0x7b, 0x35, 0x27, 0x18, 0x68, 0x4e, 0x17, 0xab, 0x41, 0x7d, + 0x42, 0x35, 0xac, 0x73, 0xa8, 0x52, 0xc8, 0xa1, 0xe3, 0xfc, 0x42, 0xaa, 0xd2, 0xca, 0x07, 0xe8, + 0x65, 0x97, 0x84, 0xda, 0x50, 0x8d, 0x23, 0xc7, 0xf3, 0x82, 0x56, 0x8d, 0xbb, 0xf9, 0xbd, 0x4d, + 0xd9, 0x49, 0x34, 0x18, 0x58, 0x5d, 0x91, 0x16, 0x95, 0x38, 0x1a, 0x78, 0x01, 0x7a, 0x09, 0x4d, + 0xf2, 0x15, 0x25, 0x49, 0xe4, 0x06, 0x4e, 0xb8, 0x62, 0xdd, 0xab, 0xce, 0x43, 0xdf, 0xce, 0xb8, + 0x23, 0xc6, 0x34, 0xde, 0x41, 0x03, 0xc7, 0x0f, 0xfd, 0x3b, 0xee, 0xa7, 0x01, 0xd5, 0x6b, 0x72, + 0x13, 0x27, 0x44, 0x26, 0x20, 0xc8, 0x06, 0x8d, 0xe3, 0x07, 0x2c, 0x77, 0xd0, 0x21, 0x54, 0xdc, + 0x9b, 0x2c, 0x87, 0x8a, 0x22, 0x62, 0xc3, 0x70, 0xa1, 0x8e, 0xe3, 0x07, 0x7e, 0x9d, 0xe8, 0x13, + 0x10, 0xc0, 0x39, 0x91, 0x1b, 0x66, 0xb7, 0xd2, 0xe0, 0x9c, 0xb1, 0x1b, 0x12, 0xf4, 0x1a, 0xb4, + 0x24, 0x7e, 0x70, 0x66, 0xfc, 0x78, 0x51, 0x61, 0x5a, 0xe7, 0x79, 0x21, 0xe9, 0x32, 0xe7, 0x30, + 0x24, 0x19, 0x99, 0x1a, 0xef, 0x00, 0xd6, 0x39, 0xf3, 0xb1, 0x43, 0x7e, 0xc4, 0x50, 0x26, 0x81, + 0x97, 0xd9, 0xdf, 0x92, 0x2e, 0x73, 0x0b, 0x58, 0xee, 0x31, 0x20, 0xa6, 0x2c, 0x29, 0xce, 0xa8, + 0xef, 0xfd, 0x0f, 0xa9, 0x84, 0x40, 0xbd, 0xa5, 0xbe, 0xc7, 0x73, 0xa8, 0x81, 0x39, 0x6d, 0x7c, + 0x0e, 0x95, 0x2b, 0x6e, 0xee, 0x35, 0x68, 0x5c, 0xca, 0x61, 0xec, 0xac, 0xb6, 0x0a, 0x61, 0xe6, + 0x47, 0x63, 0x48, 0x33, 0x32, 0x35, 0xba, 0xb0, 0x7d, 0x2e, 0x8f, 0xe5, 0x02, 0xdf, 0xde, 0x2f, + 0xe3, 0xaf, 0x65, 0xa8, 0xbd, 0x8d, 0x17, 0xec, 0xc2, 0x51, 0x13, 0xca, 0xbe, 0xc7, 0xf5, 0x14, + 0x5c, 0xf6, 0x3d, 0xf4, 0x5b, 0x68, 0x86, 0xfe, 0x6d, 0xe2, 0xb2, 0xb4, 0x11, 0x15, 0x20, 0x8a, + 0xf8, 0xfb, 0x9b, 0x9e, 0x8d, 0x32, 0x09, 0x5e, 0x06, 0xdb, 0xe1, 0xe6, 0x72, 0x23, 0xb1, 0x95, + 0x42, 0x62, 0xbf, 0x84, 0x66, 0x10, 0xcf, 0xdc, 0xc0, 0xc9, 0xdb, 0xaa, 0x2a, 0x92, 0x8f, 0x73, + 0x2f, 0xb2, 0xde, 0xfa, 0x08, 0x97, 0xca, 0x13, 0x71, 0x41, 0x9f, 0xc1, 0xd6, 0xdc, 0x4d, 0xa8, + 0x3f, 0xf3, 0xe7, 0x2e, 0xfb, 0x31, 0xa9, 0x72, 0xc5, 0x82, 0xdb, 0x05, 0xdc, 0x70, 0x41, 0x1c, + 0x7d, 0x0a, 0x5b, 0x09, 0x59, 0x92, 0x24, 0x25, 0x9e, 0xc3, 0xce, 0xad, 0x1d, 0x2a, 0x47, 0x0a, + 0xd6, 0x32, 0xde, 0xd0, 0x4b, 0x8d, 0x7f, 0x97, 0xa1, 0x7a, 0x25, 0xb2, 0xeb, 0x18, 0x54, 0x8e, + 0x8d, 0xf8, 0xe9, 0x38, 0xd8, 0x3c, 0x44, 0x48, 0x70, 0x60, 0xb8, 0x0c, 0xfa, 0x01, 0x34, 0xa8, + 0x1f, 0x92, 0x94, 0xba, 0xe1, 0x9c, 0x83, 0xa9, 0xe0, 0x35, 0xe3, 0xeb, 0x72, 0x84, 0xfd, 0x59, + 0xb0, 0x9a, 0x16, 0xf0, 0x30, 0x12, 0xfd, 0x1c, 0x1a, 0xac, 0x26, 0xf8, 0x8f, 0x50, 0xab, 0xc2, + 0x8b, 0x6c, 0xff, 0x51, 0x45, 0xf0, 0x63, 0x71, 0x3d, 0xc9, 0xaa, 0xec, 0xd7, 0xa0, 0xf1, 0x2c, + 0x96, 0x4a, 0xa2, 0x99, 0x1c, 0x14, 0x9b, 0x49, 0x56, 0x2d, 0x18, 0xd6, 0xfd, 0x17, 0xbd, 0x82, + 0xca, 0x92, 0xbb, 0x54, 0x93, 0x3f, 0x64, 0x9b, 0xc1, 0x71, 0xd8, 0xc5, 0x3e, 0x9b, 0x76, 0xbf, + 0x17, 0x59, 0xc4, 0xdb, 0xc8, 0xa3, 0x69, 0x27, 0x13, 0x0c, 0x67, 0x32, 0x3c, 0xaa, 0x30, 0x68, + 0x35, 0x64, 0x54, 0x61, 0xc0, 0x30, 0x9f, 0x2d, 0x92, 0x84, 0xff, 0x02, 0xfa, 0x21, 0x69, 0xed, + 0x73, 0x70, 0x34, 0xc9, 0xb3, 0xfd, 0x90, 0x18, 0x7f, 0x2c, 0x43, 0xf3, 0x4a, 0x0c, 0xc9, 0x6c, + 0x30, 0x7f, 0x0e, 0x7b, 0xe4, 0xe6, 0x86, 0xcc, 0xa8, 0xbf, 0x24, 0xce, 0xcc, 0x0d, 0x02, 0x92, + 0x38, 0x32, 0x85, 0xb5, 0xce, 0x4e, 0x5b, 0xfc, 0x2c, 0xf7, 0x39, 0x7f, 0x38, 0xc0, 0xbb, 0xb9, + 0xac, 0x64, 0x79, 0xc8, 0x84, 0x3d, 0x3f, 0x0c, 0x89, 0xe7, 0xbb, 0x74, 0xd3, 0x80, 0xe8, 0x5d, + 0xcf, 0x65, 0x23, 0xb8, 0xb2, 0xcf, 0x5c, 0x4a, 0xd6, 0x66, 0x72, 0x8d, 0xdc, 0xcc, 0x4b, 0x96, + 0xe7, 0xc9, 0x6d, 0x3e, 0xeb, 0xb7, 0xa5, 0xa6, 0xcd, 0x99, 0x58, 0x6e, 0x16, 0xfe, 0x23, 0xd4, + 0x47, 0xff, 0x11, 0xeb, 0x5e, 0x5f, 0xf9, 0x58, 0xaf, 0x37, 0x3e, 0x83, 0x9d, 0x1c, 0x08, 0xf9, + 0x9f, 0x70, 0x0c, 0x55, 0x7e, 0xb9, 0x59, 0xf7, 0x40, 0x1f, 0xe6, 0x21, 0x96, 0x12, 0xc6, 0x1f, + 0xca, 0x80, 0x32, 0xfd, 0xf8, 0x21, 0xfd, 0x8e, 0x82, 0xb9, 0x0f, 0x15, 0xce, 0x97, 0x48, 0x8a, + 0x05, 0xc3, 0x21, 0x70, 0x53, 0x3a, 0xbf, 0xcf, 0x61, 0x14, 0xca, 0xef, 0xd8, 0x17, 0x93, 0x74, + 0x11, 0x50, 0x2c, 0x25, 0x8c, 0xbf, 0x95, 0x60, 0xaf, 0x80, 0x83, 0xc4, 0x72, 0x3d, 0x10, 0x4a, + 0xff, 0x7d, 0x20, 0xa0, 0x23, 0xa8, 0xcf, 0xef, 0xbf, 0x61, 0x70, 0xe4, 0xbb, 0x5f, 0x5b, 0xd7, + 0x3f, 0x04, 0x35, 0x89, 0x1f, 0xd2, 0x96, 0xca, 0x35, 0x37, 0xa7, 0x24, 0xe7, 0xb3, 0x51, 0x5b, + 0x88, 0xa3, 0x30, 0x6a, 0xc5, 0xce, 0xf1, 0x6f, 0x40, 0xdb, 0x18, 0xec, 0xec, 0xff, 0x7f, 0x78, + 0x36, 0x9e, 0x60, 0x53, 0x7f, 0x86, 0xea, 0xa0, 0x4e, 0xed, 0xc9, 0x85, 0x5e, 0x62, 0x94, 0xf9, + 0x3b, 0xb3, 0x2f, 0xde, 0x14, 0x8c, 0x72, 0xa4, 0x90, 0x72, 0xfc, 0xcf, 0x12, 0xc0, 0xba, 0x45, + 0x21, 0x0d, 0x6a, 0x97, 0xe3, 0xf3, 0xf1, 0xe4, 0x8b, 0xb1, 0x30, 0x70, 0x66, 0x0f, 0x07, 0x7a, + 0x09, 0x35, 0xa0, 0x22, 0x1e, 0x29, 0x65, 0x76, 0x82, 0x7c, 0xa1, 0x28, 0xec, 0xf9, 0x92, 0x3f, + 0x4f, 0x54, 0x54, 0x03, 0x25, 0x7f, 0x84, 0xc8, 0x57, 0x47, 0x95, 0x19, 0xc4, 0xe6, 0x85, 0xd5, + 0xed, 0x9b, 0x7a, 0x8d, 0x6d, 0xe4, 0xef, 0x0f, 0x80, 0x6a, 0xf6, 0xf8, 0x60, 0x9a, 0xec, 0xc9, + 0x02, 0xec, 0x9c, 0x89, 0xfd, 0xc6, 0xc4, 0xba, 0xc6, 0x78, 0x78, 0xf2, 0x85, 0xbe, 0xc5, 0x78, + 0xa7, 0x43, 0xd3, 0x1a, 0xe8, 0xdb, 0xec, 0xcd, 0xf2, 0xc6, 0xec, 0x62, 0xbb, 0x67, 0x76, 0x6d, + 0xbd, 0xc9, 0x76, 0xae, 0xb8, 0x83, 0x3b, 0xec, 0x98, 0xb7, 0x93, 0x4b, 0x3c, 0xee, 0x5a, 0xba, + 0x7e, 0xfc, 0x0a, 0xb6, 0x0b, 0x13, 0x89, 0x9d, 0x65, 0x77, 0x7b, 0x96, 0x39, 0xd5, 0x9f, 0x31, + 0x7a, 0xfa, 0xa6, 0x8b, 0x07, 0x53, 0xbd, 0xd4, 0xfb, 0xc9, 0x97, 0xaf, 0x96, 0x3e, 0x25, 0x69, + 0xda, 0xf6, 0xe3, 0x13, 0x41, 0x9d, 0xdc, 0xc6, 0x27, 0x4b, 0x7a, 0xc2, 0xdf, 0xcf, 0x27, 0xeb, + 0xf2, 0xb9, 0xae, 0x72, 0xce, 0x2f, 0xfe, 0x13, 0x00, 0x00, 0xff, 0xff, 0xc2, 0x82, 0x3a, 0x5d, + 0x9b, 0x0f, 0x00, 0x00, } diff --git a/go/vt/vttablet/tabletmanager/action_agent.go b/go/vt/vttablet/tabletmanager/action_agent.go index 7fe05a7b5b3..fdccaacc715 100644 --- a/go/vt/vttablet/tabletmanager/action_agent.go +++ b/go/vt/vttablet/tabletmanager/action_agent.go @@ -290,7 +290,10 @@ func NewActionAgent( // The db name is set by the Start function called above agent.VREngine = vreplication.NewEngine(ts, tabletAlias.Cell, mysqld, func() binlogplayer.DBClient { return binlogplayer.NewDBClient(agent.DBConfigs.FilteredWithDB()) - }, agent.DBConfigs.FilteredWithDB().DbName) + }, + agent.DBConfigs.ExternalRepl(), + agent.DBConfigs.FilteredWithDB().DbName, + ) servenv.OnTerm(agent.VREngine.Close) // Run a background task to rebuild the SrvKeyspace in our cell/keyspace @@ -357,7 +360,7 @@ func NewTestActionAgent(batchCtx context.Context, ts *topo.Server, tabletAlias * Cnf: nil, MysqlDaemon: mysqlDaemon, DBConfigs: &dbconfigs.DBConfigs{}, - VREngine: vreplication.NewEngine(ts, tabletAlias.Cell, mysqlDaemon, binlogplayer.NewFakeDBClient, ti.DbName()), + VREngine: vreplication.NewEngine(ts, tabletAlias.Cell, mysqlDaemon, binlogplayer.NewFakeDBClient, nil, ti.DbName()), History: history.New(historyLength), _healthy: fmt.Errorf("healthcheck not run yet"), } @@ -396,7 +399,7 @@ func NewComboActionAgent(batchCtx context.Context, ts *topo.Server, tabletAlias Cnf: nil, MysqlDaemon: mysqlDaemon, DBConfigs: dbcfgs, - VREngine: vreplication.NewEngine(nil, "", nil, nil, ""), + VREngine: vreplication.NewEngine(nil, "", nil, nil, nil, ""), gotMysqlPort: true, History: history.New(historyLength), _healthy: fmt.Errorf("healthcheck not run yet"), diff --git a/go/vt/vttablet/tabletmanager/vreplication/controller.go b/go/vt/vttablet/tabletmanager/vreplication/controller.go index 699c66212a6..0f98bc47f77 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/controller.go +++ b/go/vt/vttablet/tabletmanager/vreplication/controller.go @@ -27,6 +27,7 @@ import ( "github.com/golang/protobuf/proto" "golang.org/x/net/context" + "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/sync2" "vitess.io/vitess/go/tb" "vitess.io/vitess/go/vt/binlog/binlogplayer" @@ -43,9 +44,10 @@ var retryDelay = flag.Duration("vreplication_retry_delay", 5*time.Second, "delay // There is no mutex within a controller becaust its members are // either read-only or self-synchronized. type controller struct { - dbClientFactory func() binlogplayer.DBClient - mysqld mysqlctl.MysqlDaemon - blpStats *binlogplayer.Stats + dbClientFactory func() binlogplayer.DBClient + sourceDbConnParams *mysql.ConnParams + mysqld mysqlctl.MysqlDaemon + blpStats *binlogplayer.Stats id uint32 source binlogdatapb.BinlogSource @@ -61,15 +63,16 @@ type controller struct { // newController creates a new controller. Unless a stream is explicitly 'Stopped', // this function launches a goroutine to perform continuous vreplication. -func newController(ctx context.Context, params map[string]string, dbClientFactory func() binlogplayer.DBClient, mysqld mysqlctl.MysqlDaemon, ts *topo.Server, cell, tabletTypesStr string, blpStats *binlogplayer.Stats) (*controller, error) { +func newController(ctx context.Context, params map[string]string, dbClientFactory func() binlogplayer.DBClient, sourceDbConnParams *mysql.ConnParams, mysqld mysqlctl.MysqlDaemon, ts *topo.Server, cell, tabletTypesStr string, blpStats *binlogplayer.Stats) (*controller, error) { if blpStats == nil { blpStats = binlogplayer.NewStats() } ct := &controller{ - dbClientFactory: dbClientFactory, - mysqld: mysqld, - blpStats: blpStats, - done: make(chan struct{}), + dbClientFactory: dbClientFactory, + sourceDbConnParams: sourceDbConnParams, + mysqld: mysqld, + blpStats: blpStats, + done: make(chan struct{}), } // id @@ -92,18 +95,20 @@ func newController(ctx context.Context, params map[string]string, dbClientFactor } ct.stopPos = params["stop_pos"] - // tabletPicker - if v, ok := params["cell"]; ok { - cell = v - } - if v, ok := params["tablet_types"]; ok { - tabletTypesStr = v - } - tp, err := newTabletPicker(ctx, ts, cell, ct.source.Keyspace, ct.source.Shard, tabletTypesStr) - if err != nil { - return nil, err + if ct.source.GetExternalMysql() == "" { + // tabletPicker + if v, ok := params["cell"]; ok { + cell = v + } + if v, ok := params["tablet_types"]; ok { + tabletTypesStr = v + } + tp, err := newTabletPicker(ctx, ts, cell, ct.source.Keyspace, ct.source.Shard, tabletTypesStr) + if err != nil { + return nil, err + } + ct.tabletPicker = tp } - ct.tabletPicker = tp // cancel ctx, ct.cancel = context.WithCancel(ctx) @@ -199,7 +204,14 @@ func (ct *controller) runBlp(ctx context.Context) (err error) { if _, err := dbClient.ExecuteFetch("set names binary", 10000); err != nil { return err } - vsClient := NewTabletVStreamerClient(tablet) + + var vsClient VStreamerClient + if ct.source.GetExternalMysql() == "" { + vsClient = NewTabletVStreamerClient(tablet) + } else { + vsClient = NewMySQLVStreamerClient(ct.sourceDbConnParams) + } + vreplicator := NewVReplicator(ct.id, &ct.source, vsClient, ct.blpStats, dbClient, ct.mysqld) return vreplicator.Replicate(ctx) } diff --git a/go/vt/vttablet/tabletmanager/vreplication/controller_test.go b/go/vt/vttablet/tabletmanager/vreplication/controller_test.go index f330985c4c9..c0644a7f3a6 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/controller_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/controller_test.go @@ -76,7 +76,7 @@ func TestControllerKeyRange(t *testing.T) { dbClientFactory := func() binlogplayer.DBClient { return dbClient } mysqld := &fakemysqldaemon.FakeMysqlDaemon{MysqlPort: 3306} - ct, err := newController(context.Background(), params, dbClientFactory, mysqld, env.TopoServ, env.Cells[0], "replica", nil) + ct, err := newController(context.Background(), params, dbClientFactory, nil, mysqld, env.TopoServ, env.Cells[0], "replica", nil) if err != nil { t.Fatal(err) } @@ -136,7 +136,7 @@ func TestControllerTables(t *testing.T) { }, } - ct, err := newController(context.Background(), params, dbClientFactory, mysqld, env.TopoServ, env.Cells[0], "replica", nil) + ct, err := newController(context.Background(), params, dbClientFactory, nil, mysqld, env.TopoServ, env.Cells[0], "replica", nil) if err != nil { t.Fatal(err) } @@ -153,7 +153,7 @@ func TestControllerBadID(t *testing.T) { params := map[string]string{ "id": "bad", } - _, err := newController(context.Background(), params, nil, nil, nil, "", "", nil) + _, err := newController(context.Background(), params, nil, nil, nil, nil, "", "", nil) want := `strconv.Atoi: parsing "bad": invalid syntax` if err == nil || err.Error() != want { t.Errorf("newController err: %v, want %v", err, want) @@ -166,7 +166,7 @@ func TestControllerStopped(t *testing.T) { "state": binlogplayer.BlpStopped, } - ct, err := newController(context.Background(), params, nil, nil, nil, "", "", nil) + ct, err := newController(context.Background(), params, nil, nil, nil, nil, "", "", nil) if err != nil { t.Fatal(err) } @@ -203,7 +203,7 @@ func TestControllerOverrides(t *testing.T) { dbClientFactory := func() binlogplayer.DBClient { return dbClient } mysqld := &fakemysqldaemon.FakeMysqlDaemon{MysqlPort: 3306} - ct, err := newController(context.Background(), params, dbClientFactory, mysqld, env.TopoServ, env.Cells[0], "rdonly", nil) + ct, err := newController(context.Background(), params, dbClientFactory, nil, mysqld, env.TopoServ, env.Cells[0], "rdonly", nil) if err != nil { t.Fatal(err) } @@ -227,7 +227,7 @@ func TestControllerCanceledContext(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() - ct, err := newController(ctx, params, nil, nil, env.TopoServ, env.Cells[0], "rdonly", nil) + ct, err := newController(ctx, params, nil, nil, nil, env.TopoServ, env.Cells[0], "rdonly", nil) if err != nil { t.Fatal(err) } @@ -269,7 +269,7 @@ func TestControllerRetry(t *testing.T) { dbClientFactory := func() binlogplayer.DBClient { return dbClient } mysqld := &fakemysqldaemon.FakeMysqlDaemon{MysqlPort: 3306} - ct, err := newController(context.Background(), params, dbClientFactory, mysqld, env.TopoServ, env.Cells[0], "rdonly", nil) + ct, err := newController(context.Background(), params, dbClientFactory, nil, mysqld, env.TopoServ, env.Cells[0], "rdonly", nil) if err != nil { t.Fatal(err) } @@ -315,7 +315,7 @@ func TestControllerStopPosition(t *testing.T) { dbClientFactory := func() binlogplayer.DBClient { return dbClient } mysqld := &fakemysqldaemon.FakeMysqlDaemon{MysqlPort: 3306} - ct, err := newController(context.Background(), params, dbClientFactory, mysqld, env.TopoServ, env.Cells[0], "replica", nil) + ct, err := newController(context.Background(), params, dbClientFactory, nil, mysqld, env.TopoServ, env.Cells[0], "replica", nil) if err != nil { t.Fatal(err) } diff --git a/go/vt/vttablet/tabletmanager/vreplication/engine.go b/go/vt/vttablet/tabletmanager/vreplication/engine.go index 4baad2ac62c..2b72a33ea4e 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/engine.go +++ b/go/vt/vttablet/tabletmanager/vreplication/engine.go @@ -77,23 +77,25 @@ type Engine struct { // cancel will cancel the root context, thereby all controllers. cancel context.CancelFunc - ts *topo.Server - cell string - mysqld mysqlctl.MysqlDaemon - dbClientFactory func() binlogplayer.DBClient - dbName string + ts *topo.Server + cell string + mysqld mysqlctl.MysqlDaemon + dbClientFactory func() binlogplayer.DBClient + sourceDbConnParams *mysql.ConnParams + dbName string } // NewEngine creates a new Engine. // A nil ts means that the Engine is disabled. -func NewEngine(ts *topo.Server, cell string, mysqld mysqlctl.MysqlDaemon, dbClientFactory func() binlogplayer.DBClient, dbName string) *Engine { +func NewEngine(ts *topo.Server, cell string, mysqld mysqlctl.MysqlDaemon, dbClientFactory func() binlogplayer.DBClient, sourceDbConnParams *mysql.ConnParams, dbName string) *Engine { vre := &Engine{ - controllers: make(map[int]*controller), - ts: ts, - cell: cell, - mysqld: mysqld, - dbClientFactory: dbClientFactory, - dbName: dbName, + controllers: make(map[int]*controller), + ts: ts, + cell: cell, + mysqld: mysqld, + dbClientFactory: dbClientFactory, + sourceDbConnParams: sourceDbConnParams, + dbName: dbName, } return vre } @@ -187,7 +189,7 @@ func (vre *Engine) initAll() error { return err } for _, row := range rows { - ct, err := newController(vre.ctx, row, vre.dbClientFactory, vre.mysqld, vre.ts, vre.cell, *tabletTypesStr, nil) + ct, err := newController(vre.ctx, row, vre.dbClientFactory, vre.sourceDbConnParams, vre.mysqld, vre.ts, vre.cell, *tabletTypesStr, nil) if err != nil { return err } @@ -280,7 +282,7 @@ func (vre *Engine) Exec(query string) (*sqltypes.Result, error) { if err != nil { return nil, err } - ct, err := newController(vre.ctx, params, vre.dbClientFactory, vre.mysqld, vre.ts, vre.cell, *tabletTypesStr, nil) + ct, err := newController(vre.ctx, params, vre.dbClientFactory, vre.sourceDbConnParams, vre.mysqld, vre.ts, vre.cell, *tabletTypesStr, nil) if err != nil { return nil, err } @@ -318,7 +320,7 @@ func (vre *Engine) Exec(query string) (*sqltypes.Result, error) { } // Create a new controller in place of the old one. // For continuity, the new controller inherits the previous stats. - ct, err := newController(vre.ctx, params, vre.dbClientFactory, vre.mysqld, vre.ts, vre.cell, *tabletTypesStr, blpStats[id]) + ct, err := newController(vre.ctx, params, vre.dbClientFactory, vre.sourceDbConnParams, vre.mysqld, vre.ts, vre.cell, *tabletTypesStr, blpStats[id]) if err != nil { return nil, err } diff --git a/go/vt/vttablet/tabletmanager/vreplication/engine_test.go b/go/vt/vttablet/tabletmanager/vreplication/engine_test.go index d16ca35cc4b..8e7cff1a880 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/engine_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/engine_test.go @@ -41,7 +41,7 @@ func TestEngineOpen(t *testing.T) { // Test Insert - vre := NewEngine(env.TopoServ, env.Cells[0], mysqld, dbClientFactory, dbClient.DBName()) + vre := NewEngine(env.TopoServ, env.Cells[0], mysqld, dbClientFactory, nil, dbClient.DBName()) if vre.IsOpen() { t.Errorf("IsOpen: %v, want false", vre.IsOpen()) } @@ -89,7 +89,7 @@ func TestEngineExec(t *testing.T) { // Test Insert - vre := NewEngine(env.TopoServ, env.Cells[0], mysqld, dbClientFactory, dbClient.DBName()) + vre := NewEngine(env.TopoServ, env.Cells[0], mysqld, dbClientFactory, nil, dbClient.DBName()) dbClient.ExpectRequest("select * from _vt.vreplication where db_name='db'", &sqltypes.Result{}, nil) if err := vre.Open(context.Background()); err != nil { @@ -249,7 +249,7 @@ func TestEngineBadInsert(t *testing.T) { dbClientFactory := func() binlogplayer.DBClient { return dbClient } mysqld := &fakemysqldaemon.FakeMysqlDaemon{MysqlPort: 3306} - vre := NewEngine(env.TopoServ, env.Cells[0], mysqld, dbClientFactory, dbClient.DBName()) + vre := NewEngine(env.TopoServ, env.Cells[0], mysqld, dbClientFactory, nil, dbClient.DBName()) dbClient.ExpectRequest("select * from _vt.vreplication where db_name='db'", &sqltypes.Result{}, nil) if err := vre.Open(context.Background()); err != nil { @@ -279,7 +279,7 @@ func TestEngineSelect(t *testing.T) { dbClientFactory := func() binlogplayer.DBClient { return dbClient } mysqld := &fakemysqldaemon.FakeMysqlDaemon{MysqlPort: 3306} - vre := NewEngine(env.TopoServ, env.Cells[0], mysqld, dbClientFactory, dbClient.DBName()) + vre := NewEngine(env.TopoServ, env.Cells[0], mysqld, dbClientFactory, nil, dbClient.DBName()) dbClient.ExpectRequest("select * from _vt.vreplication where db_name='db'", &sqltypes.Result{}, nil) if err := vre.Open(context.Background()); err != nil { @@ -314,7 +314,7 @@ func TestWaitForPos(t *testing.T) { dbClient := binlogplayer.NewMockDBClient(t) mysqld := &fakemysqldaemon.FakeMysqlDaemon{MysqlPort: 3306} dbClientFactory := func() binlogplayer.DBClient { return dbClient } - vre := NewEngine(env.TopoServ, env.Cells[0], mysqld, dbClientFactory, dbClient.DBName()) + vre := NewEngine(env.TopoServ, env.Cells[0], mysqld, dbClientFactory, nil, dbClient.DBName()) dbClient.ExpectRequest("select * from _vt.vreplication where db_name='db'", &sqltypes.Result{}, nil) if err := vre.Open(context.Background()); err != nil { @@ -344,7 +344,7 @@ func TestWaitForPosError(t *testing.T) { dbClient := binlogplayer.NewMockDBClient(t) mysqld := &fakemysqldaemon.FakeMysqlDaemon{MysqlPort: 3306} dbClientFactory := func() binlogplayer.DBClient { return dbClient } - vre := NewEngine(env.TopoServ, env.Cells[0], mysqld, dbClientFactory, dbClient.DBName()) + vre := NewEngine(env.TopoServ, env.Cells[0], mysqld, dbClientFactory, nil, dbClient.DBName()) err := vre.WaitForPos(context.Background(), 1, "MariaDB/0-1-1084") want := `vreplication engine is closed` @@ -386,7 +386,7 @@ func TestWaitForPosCancel(t *testing.T) { dbClient := binlogplayer.NewMockDBClient(t) mysqld := &fakemysqldaemon.FakeMysqlDaemon{MysqlPort: 3306} dbClientFactory := func() binlogplayer.DBClient { return dbClient } - vre := NewEngine(env.TopoServ, env.Cells[0], mysqld, dbClientFactory, dbClient.DBName()) + vre := NewEngine(env.TopoServ, env.Cells[0], mysqld, dbClientFactory, nil, dbClient.DBName()) dbClient.ExpectRequest("select * from _vt.vreplication where db_name='db'", &sqltypes.Result{}, nil) if err := vre.Open(context.Background()); err != nil { @@ -433,7 +433,7 @@ func TestCreateDBAndTable(t *testing.T) { // Test Insert - vre := NewEngine(env.TopoServ, env.Cells[0], mysqld, dbClientFactory, dbClient.DBName()) + vre := NewEngine(env.TopoServ, env.Cells[0], mysqld, dbClientFactory, nil, dbClient.DBName()) tableNotFound := mysql.SQLError{Num: 1146, Message: "table not found"} dbClient.ExpectRequest("select * from _vt.vreplication where db_name='db'", nil, &tableNotFound) diff --git a/go/vt/vttablet/tabletmanager/vreplication/framework_test.go b/go/vt/vttablet/tabletmanager/vreplication/framework_test.go index bb8e4982094..461ba0ab0df 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/framework_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/framework_test.go @@ -96,7 +96,7 @@ func TestMain(m *testing.M) { return 1 } - playerEngine = NewEngine(env.TopoServ, env.Cells[0], env.Mysqld, realDBClientFactory, vrepldb) + playerEngine = NewEngine(env.TopoServ, env.Cells[0], env.Mysqld, realDBClientFactory, nil, vrepldb) if err := playerEngine.Open(context.Background()); err != nil { fmt.Fprintf(os.Stderr, "%v", err) return 1 diff --git a/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go b/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go index 67d4772d9f7..b9c73a92fba 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go +++ b/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go @@ -315,8 +315,8 @@ func (vs *vstreamer) parseEvent(ev mysql.BinlogEvent) ([]*binlogdatapb.VEvent, e if err != nil { return nil, fmt.Errorf("can't get query from binlog event: %v, event data: %#v", err, ev) } - // Insert/Delete/Update are supported are in here only to have support for vtshovel with - // SBR streams. Vitess itself should never run into cases where it needs to consume non rbr statements. + // Insert/Delete/Update are supported only to be used in the context of vtshovel where source databases + // could be using SBR. Vitess itself should never run into cases where it needs to consume non rbr statements. switch cat := sqlparser.Preview(q.SQL); cat { case sqlparser.StmtInsert: mustSend := mustSendStmt(q, vs.cp.DbName) diff --git a/proto/binlogdata.proto b/proto/binlogdata.proto index 7627c719560..78285253537 100644 --- a/proto/binlogdata.proto +++ b/proto/binlogdata.proto @@ -170,6 +170,10 @@ message BinlogSource { // on_ddl specifies the action to be taken when a DDL is encountered. OnDDLAction on_ddl = 7; + + // Source is an external mysql. This attribute should be set to the username + // to use in the connection + string external_mysql = 8; } // VEventType enumerates the event types. diff --git a/py/vtproto/binlogdata_pb2.py b/py/vtproto/binlogdata_pb2.py index 816b684becd..6b1a69c001b 100644 --- a/py/vtproto/binlogdata_pb2.py +++ b/py/vtproto/binlogdata_pb2.py @@ -23,7 +23,7 @@ package='binlogdata', syntax='proto3', serialized_options=_b('Z\'vitess.io/vitess/go/vt/proto/binlogdata'), - serialized_pb=_b('\n\x10\x62inlogdata.proto\x12\nbinlogdata\x1a\x0bvtrpc.proto\x1a\x0bquery.proto\x1a\x0etopodata.proto\"7\n\x07\x43harset\x12\x0e\n\x06\x63lient\x18\x01 \x01(\x05\x12\x0c\n\x04\x63onn\x18\x02 \x01(\x05\x12\x0e\n\x06server\x18\x03 \x01(\x05\"\xb5\x03\n\x11\x42inlogTransaction\x12;\n\nstatements\x18\x01 \x03(\x0b\x32\'.binlogdata.BinlogTransaction.Statement\x12&\n\x0b\x65vent_token\x18\x04 \x01(\x0b\x32\x11.query.EventToken\x1a\xae\x02\n\tStatement\x12\x42\n\x08\x63\x61tegory\x18\x01 \x01(\x0e\x32\x30.binlogdata.BinlogTransaction.Statement.Category\x12$\n\x07\x63harset\x18\x02 \x01(\x0b\x32\x13.binlogdata.Charset\x12\x0b\n\x03sql\x18\x03 \x01(\x0c\"\xa9\x01\n\x08\x43\x61tegory\x12\x13\n\x0f\x42L_UNRECOGNIZED\x10\x00\x12\x0c\n\x08\x42L_BEGIN\x10\x01\x12\r\n\tBL_COMMIT\x10\x02\x12\x0f\n\x0b\x42L_ROLLBACK\x10\x03\x12\x15\n\x11\x42L_DML_DEPRECATED\x10\x04\x12\n\n\x06\x42L_DDL\x10\x05\x12\n\n\x06\x42L_SET\x10\x06\x12\r\n\tBL_INSERT\x10\x07\x12\r\n\tBL_UPDATE\x10\x08\x12\r\n\tBL_DELETE\x10\tJ\x04\x08\x02\x10\x03J\x04\x08\x03\x10\x04\"v\n\x15StreamKeyRangeRequest\x12\x10\n\x08position\x18\x01 \x01(\t\x12%\n\tkey_range\x18\x02 \x01(\x0b\x32\x12.topodata.KeyRange\x12$\n\x07\x63harset\x18\x03 \x01(\x0b\x32\x13.binlogdata.Charset\"S\n\x16StreamKeyRangeResponse\x12\x39\n\x12\x62inlog_transaction\x18\x01 \x01(\x0b\x32\x1d.binlogdata.BinlogTransaction\"]\n\x13StreamTablesRequest\x12\x10\n\x08position\x18\x01 \x01(\t\x12\x0e\n\x06tables\x18\x02 \x03(\t\x12$\n\x07\x63harset\x18\x03 \x01(\x0b\x32\x13.binlogdata.Charset\"Q\n\x14StreamTablesResponse\x12\x39\n\x12\x62inlog_transaction\x18\x01 \x01(\x0b\x32\x1d.binlogdata.BinlogTransaction\"%\n\x04Rule\x12\r\n\x05match\x18\x01 \x01(\t\x12\x0e\n\x06\x66ilter\x18\x02 \x01(\t\"\x9c\x01\n\x06\x46ilter\x12\x1f\n\x05rules\x18\x01 \x03(\x0b\x32\x10.binlogdata.Rule\x12\x39\n\x0e\x66ieldEventMode\x18\x02 \x01(\x0e\x32!.binlogdata.Filter.FieldEventMode\"6\n\x0e\x46ieldEventMode\x12\x13\n\x0f\x45RR_ON_MISMATCH\x10\x00\x12\x0f\n\x0b\x42\x45ST_EFFORT\x10\x01\"\xde\x01\n\x0c\x42inlogSource\x12\x10\n\x08keyspace\x18\x01 \x01(\t\x12\r\n\x05shard\x18\x02 \x01(\t\x12)\n\x0btablet_type\x18\x03 \x01(\x0e\x32\x14.topodata.TabletType\x12%\n\tkey_range\x18\x04 \x01(\x0b\x32\x12.topodata.KeyRange\x12\x0e\n\x06tables\x18\x05 \x03(\t\x12\"\n\x06\x66ilter\x18\x06 \x01(\x0b\x32\x12.binlogdata.Filter\x12\'\n\x06on_ddl\x18\x07 \x01(\x0e\x32\x17.binlogdata.OnDDLAction\"B\n\tRowChange\x12\x1a\n\x06\x62\x65\x66ore\x18\x01 \x01(\x0b\x32\n.query.Row\x12\x19\n\x05\x61\x66ter\x18\x02 \x01(\x0b\x32\n.query.Row\"J\n\x08RowEvent\x12\x12\n\ntable_name\x18\x01 \x01(\t\x12*\n\x0brow_changes\x18\x02 \x03(\x0b\x32\x15.binlogdata.RowChange\">\n\nFieldEvent\x12\x12\n\ntable_name\x18\x01 \x01(\t\x12\x1c\n\x06\x66ields\x18\x02 \x03(\x0b\x32\x0c.query.Field\":\n\tShardGtid\x12\x10\n\x08keyspace\x18\x01 \x01(\t\x12\r\n\x05shard\x18\x02 \x01(\t\x12\x0c\n\x04gtid\x18\x03 \x01(\t\"3\n\x05VGtid\x12*\n\x0bshard_gtids\x18\x01 \x03(\x0b\x32\x15.binlogdata.ShardGtid\"0\n\rKeyspaceShard\x12\x10\n\x08keyspace\x18\x01 \x01(\t\x12\r\n\x05shard\x18\x02 \x01(\t\"\xe3\x01\n\x07Journal\x12\n\n\x02id\x18\x01 \x01(\x03\x12\x31\n\x0emigration_type\x18\x02 \x01(\x0e\x32\x19.binlogdata.MigrationType\x12\x0e\n\x06tables\x18\x03 \x03(\t\x12\x16\n\x0elocal_position\x18\x04 \x01(\t\x12*\n\x0bshard_gtids\x18\x05 \x03(\x0b\x32\x15.binlogdata.ShardGtid\x12/\n\x0cparticipants\x18\x06 \x03(\x0b\x32\x19.binlogdata.KeyspaceShard\x12\x14\n\x0creversed_ids\x18\x07 \x03(\x03\"\x9d\x02\n\x06VEvent\x12$\n\x04type\x18\x01 \x01(\x0e\x32\x16.binlogdata.VEventType\x12\x11\n\ttimestamp\x18\x02 \x01(\x03\x12\x0c\n\x04gtid\x18\x03 \x01(\t\x12\x0b\n\x03\x64\x64l\x18\x04 \x01(\t\x12\'\n\trow_event\x18\x05 \x01(\x0b\x32\x14.binlogdata.RowEvent\x12+\n\x0b\x66ield_event\x18\x06 \x01(\x0b\x32\x16.binlogdata.FieldEvent\x12 \n\x05vgtid\x18\x07 \x01(\x0b\x32\x11.binlogdata.VGtid\x12$\n\x07journal\x18\x08 \x01(\x0b\x32\x13.binlogdata.Journal\x12\x0b\n\x03\x64ml\x18\t \x01(\t\x12\x14\n\x0c\x63urrent_time\x18\x14 \x01(\x03\"\xc7\x01\n\x0eVStreamRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x1d\n\x06target\x18\x03 \x01(\x0b\x32\r.query.Target\x12\x10\n\x08position\x18\x04 \x01(\t\x12\"\n\x06\x66ilter\x18\x05 \x01(\x0b\x32\x12.binlogdata.Filter\"5\n\x0fVStreamResponse\x12\"\n\x06\x65vents\x18\x01 \x03(\x0b\x32\x12.binlogdata.VEvent\"\xc8\x01\n\x12VStreamRowsRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x1d\n\x06target\x18\x03 \x01(\x0b\x32\r.query.Target\x12\r\n\x05query\x18\x04 \x01(\t\x12\"\n\x06lastpk\x18\x05 \x01(\x0b\x32\x12.query.QueryResult\"\x97\x01\n\x13VStreamRowsResponse\x12\x1c\n\x06\x66ields\x18\x01 \x03(\x0b\x32\x0c.query.Field\x12\x1e\n\x08pkfields\x18\x02 \x03(\x0b\x32\x0c.query.Field\x12\x0c\n\x04gtid\x18\x03 \x01(\t\x12\x18\n\x04rows\x18\x04 \x03(\x0b\x32\n.query.Row\x12\x1a\n\x06lastpk\x18\x05 \x01(\x0b\x32\n.query.Row*>\n\x0bOnDDLAction\x12\n\n\x06IGNORE\x10\x00\x12\x08\n\x04STOP\x10\x01\x12\x08\n\x04\x45XEC\x10\x02\x12\x0f\n\x0b\x45XEC_IGNORE\x10\x03*\xd1\x01\n\nVEventType\x12\x0b\n\x07UNKNOWN\x10\x00\x12\x08\n\x04GTID\x10\x01\x12\t\n\x05\x42\x45GIN\x10\x02\x12\n\n\x06\x43OMMIT\x10\x03\x12\x0c\n\x08ROLLBACK\x10\x04\x12\x07\n\x03\x44\x44L\x10\x05\x12\n\n\x06INSERT\x10\x06\x12\x0b\n\x07REPLACE\x10\x07\x12\n\n\x06UPDATE\x10\x08\x12\n\n\x06\x44\x45LETE\x10\t\x12\x07\n\x03SET\x10\n\x12\t\n\x05OTHER\x10\x0b\x12\x07\n\x03ROW\x10\x0c\x12\t\n\x05\x46IELD\x10\r\x12\r\n\tHEARTBEAT\x10\x0e\x12\t\n\x05VGTID\x10\x0f\x12\x0b\n\x07JOURNAL\x10\x10*\'\n\rMigrationType\x12\n\n\x06TABLES\x10\x00\x12\n\n\x06SHARDS\x10\x01\x42)Z\'vitess.io/vitess/go/vt/proto/binlogdatab\x06proto3') + serialized_pb=_b('\n\x10\x62inlogdata.proto\x12\nbinlogdata\x1a\x0bvtrpc.proto\x1a\x0bquery.proto\x1a\x0etopodata.proto\"7\n\x07\x43harset\x12\x0e\n\x06\x63lient\x18\x01 \x01(\x05\x12\x0c\n\x04\x63onn\x18\x02 \x01(\x05\x12\x0e\n\x06server\x18\x03 \x01(\x05\"\xb5\x03\n\x11\x42inlogTransaction\x12;\n\nstatements\x18\x01 \x03(\x0b\x32\'.binlogdata.BinlogTransaction.Statement\x12&\n\x0b\x65vent_token\x18\x04 \x01(\x0b\x32\x11.query.EventToken\x1a\xae\x02\n\tStatement\x12\x42\n\x08\x63\x61tegory\x18\x01 \x01(\x0e\x32\x30.binlogdata.BinlogTransaction.Statement.Category\x12$\n\x07\x63harset\x18\x02 \x01(\x0b\x32\x13.binlogdata.Charset\x12\x0b\n\x03sql\x18\x03 \x01(\x0c\"\xa9\x01\n\x08\x43\x61tegory\x12\x13\n\x0f\x42L_UNRECOGNIZED\x10\x00\x12\x0c\n\x08\x42L_BEGIN\x10\x01\x12\r\n\tBL_COMMIT\x10\x02\x12\x0f\n\x0b\x42L_ROLLBACK\x10\x03\x12\x15\n\x11\x42L_DML_DEPRECATED\x10\x04\x12\n\n\x06\x42L_DDL\x10\x05\x12\n\n\x06\x42L_SET\x10\x06\x12\r\n\tBL_INSERT\x10\x07\x12\r\n\tBL_UPDATE\x10\x08\x12\r\n\tBL_DELETE\x10\tJ\x04\x08\x02\x10\x03J\x04\x08\x03\x10\x04\"v\n\x15StreamKeyRangeRequest\x12\x10\n\x08position\x18\x01 \x01(\t\x12%\n\tkey_range\x18\x02 \x01(\x0b\x32\x12.topodata.KeyRange\x12$\n\x07\x63harset\x18\x03 \x01(\x0b\x32\x13.binlogdata.Charset\"S\n\x16StreamKeyRangeResponse\x12\x39\n\x12\x62inlog_transaction\x18\x01 \x01(\x0b\x32\x1d.binlogdata.BinlogTransaction\"]\n\x13StreamTablesRequest\x12\x10\n\x08position\x18\x01 \x01(\t\x12\x0e\n\x06tables\x18\x02 \x03(\t\x12$\n\x07\x63harset\x18\x03 \x01(\x0b\x32\x13.binlogdata.Charset\"Q\n\x14StreamTablesResponse\x12\x39\n\x12\x62inlog_transaction\x18\x01 \x01(\x0b\x32\x1d.binlogdata.BinlogTransaction\"%\n\x04Rule\x12\r\n\x05match\x18\x01 \x01(\t\x12\x0e\n\x06\x66ilter\x18\x02 \x01(\t\"\x9c\x01\n\x06\x46ilter\x12\x1f\n\x05rules\x18\x01 \x03(\x0b\x32\x10.binlogdata.Rule\x12\x39\n\x0e\x66ieldEventMode\x18\x02 \x01(\x0e\x32!.binlogdata.Filter.FieldEventMode\"6\n\x0e\x46ieldEventMode\x12\x13\n\x0f\x45RR_ON_MISMATCH\x10\x00\x12\x0f\n\x0b\x42\x45ST_EFFORT\x10\x01\"\xf6\x01\n\x0c\x42inlogSource\x12\x10\n\x08keyspace\x18\x01 \x01(\t\x12\r\n\x05shard\x18\x02 \x01(\t\x12)\n\x0btablet_type\x18\x03 \x01(\x0e\x32\x14.topodata.TabletType\x12%\n\tkey_range\x18\x04 \x01(\x0b\x32\x12.topodata.KeyRange\x12\x0e\n\x06tables\x18\x05 \x03(\t\x12\"\n\x06\x66ilter\x18\x06 \x01(\x0b\x32\x12.binlogdata.Filter\x12\'\n\x06on_ddl\x18\x07 \x01(\x0e\x32\x17.binlogdata.OnDDLAction\x12\x16\n\x0e\x65xternal_mysql\x18\x08 \x01(\t\"B\n\tRowChange\x12\x1a\n\x06\x62\x65\x66ore\x18\x01 \x01(\x0b\x32\n.query.Row\x12\x19\n\x05\x61\x66ter\x18\x02 \x01(\x0b\x32\n.query.Row\"J\n\x08RowEvent\x12\x12\n\ntable_name\x18\x01 \x01(\t\x12*\n\x0brow_changes\x18\x02 \x03(\x0b\x32\x15.binlogdata.RowChange\">\n\nFieldEvent\x12\x12\n\ntable_name\x18\x01 \x01(\t\x12\x1c\n\x06\x66ields\x18\x02 \x03(\x0b\x32\x0c.query.Field\":\n\tShardGtid\x12\x10\n\x08keyspace\x18\x01 \x01(\t\x12\r\n\x05shard\x18\x02 \x01(\t\x12\x0c\n\x04gtid\x18\x03 \x01(\t\"3\n\x05VGtid\x12*\n\x0bshard_gtids\x18\x01 \x03(\x0b\x32\x15.binlogdata.ShardGtid\"0\n\rKeyspaceShard\x12\x10\n\x08keyspace\x18\x01 \x01(\t\x12\r\n\x05shard\x18\x02 \x01(\t\"\xe3\x01\n\x07Journal\x12\n\n\x02id\x18\x01 \x01(\x03\x12\x31\n\x0emigration_type\x18\x02 \x01(\x0e\x32\x19.binlogdata.MigrationType\x12\x0e\n\x06tables\x18\x03 \x03(\t\x12\x16\n\x0elocal_position\x18\x04 \x01(\t\x12*\n\x0bshard_gtids\x18\x05 \x03(\x0b\x32\x15.binlogdata.ShardGtid\x12/\n\x0cparticipants\x18\x06 \x03(\x0b\x32\x19.binlogdata.KeyspaceShard\x12\x14\n\x0creversed_ids\x18\x07 \x03(\x03\"\x9d\x02\n\x06VEvent\x12$\n\x04type\x18\x01 \x01(\x0e\x32\x16.binlogdata.VEventType\x12\x11\n\ttimestamp\x18\x02 \x01(\x03\x12\x0c\n\x04gtid\x18\x03 \x01(\t\x12\x0b\n\x03\x64\x64l\x18\x04 \x01(\t\x12\'\n\trow_event\x18\x05 \x01(\x0b\x32\x14.binlogdata.RowEvent\x12+\n\x0b\x66ield_event\x18\x06 \x01(\x0b\x32\x16.binlogdata.FieldEvent\x12 \n\x05vgtid\x18\x07 \x01(\x0b\x32\x11.binlogdata.VGtid\x12$\n\x07journal\x18\x08 \x01(\x0b\x32\x13.binlogdata.Journal\x12\x0b\n\x03\x64ml\x18\t \x01(\t\x12\x14\n\x0c\x63urrent_time\x18\x14 \x01(\x03\"\xc7\x01\n\x0eVStreamRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x1d\n\x06target\x18\x03 \x01(\x0b\x32\r.query.Target\x12\x10\n\x08position\x18\x04 \x01(\t\x12\"\n\x06\x66ilter\x18\x05 \x01(\x0b\x32\x12.binlogdata.Filter\"5\n\x0fVStreamResponse\x12\"\n\x06\x65vents\x18\x01 \x03(\x0b\x32\x12.binlogdata.VEvent\"\xc8\x01\n\x12VStreamRowsRequest\x12,\n\x13\x65\x66\x66\x65\x63tive_caller_id\x18\x01 \x01(\x0b\x32\x0f.vtrpc.CallerID\x12\x32\n\x13immediate_caller_id\x18\x02 \x01(\x0b\x32\x15.query.VTGateCallerID\x12\x1d\n\x06target\x18\x03 \x01(\x0b\x32\r.query.Target\x12\r\n\x05query\x18\x04 \x01(\t\x12\"\n\x06lastpk\x18\x05 \x01(\x0b\x32\x12.query.QueryResult\"\x97\x01\n\x13VStreamRowsResponse\x12\x1c\n\x06\x66ields\x18\x01 \x03(\x0b\x32\x0c.query.Field\x12\x1e\n\x08pkfields\x18\x02 \x03(\x0b\x32\x0c.query.Field\x12\x0c\n\x04gtid\x18\x03 \x01(\t\x12\x18\n\x04rows\x18\x04 \x03(\x0b\x32\n.query.Row\x12\x1a\n\x06lastpk\x18\x05 \x01(\x0b\x32\n.query.Row*>\n\x0bOnDDLAction\x12\n\n\x06IGNORE\x10\x00\x12\x08\n\x04STOP\x10\x01\x12\x08\n\x04\x45XEC\x10\x02\x12\x0f\n\x0b\x45XEC_IGNORE\x10\x03*\xd1\x01\n\nVEventType\x12\x0b\n\x07UNKNOWN\x10\x00\x12\x08\n\x04GTID\x10\x01\x12\t\n\x05\x42\x45GIN\x10\x02\x12\n\n\x06\x43OMMIT\x10\x03\x12\x0c\n\x08ROLLBACK\x10\x04\x12\x07\n\x03\x44\x44L\x10\x05\x12\n\n\x06INSERT\x10\x06\x12\x0b\n\x07REPLACE\x10\x07\x12\n\n\x06UPDATE\x10\x08\x12\n\n\x06\x44\x45LETE\x10\t\x12\x07\n\x03SET\x10\n\x12\t\n\x05OTHER\x10\x0b\x12\x07\n\x03ROW\x10\x0c\x12\t\n\x05\x46IELD\x10\r\x12\r\n\tHEARTBEAT\x10\x0e\x12\t\n\x05VGTID\x10\x0f\x12\x0b\n\x07JOURNAL\x10\x10*\'\n\rMigrationType\x12\n\n\x06TABLES\x10\x00\x12\n\n\x06SHARDS\x10\x01\x42)Z\'vitess.io/vitess/go/vt/proto/binlogdatab\x06proto3') , dependencies=[vtrpc__pb2.DESCRIPTOR,query__pb2.DESCRIPTOR,topodata__pb2.DESCRIPTOR,]) @@ -52,8 +52,8 @@ ], containing_type=None, serialized_options=None, - serialized_start=2880, - serialized_end=2942, + serialized_start=2904, + serialized_end=2966, ) _sym_db.RegisterEnumDescriptor(_ONDDLACTION) @@ -135,8 +135,8 @@ ], containing_type=None, serialized_options=None, - serialized_start=2945, - serialized_end=3154, + serialized_start=2969, + serialized_end=3178, ) _sym_db.RegisterEnumDescriptor(_VEVENTTYPE) @@ -158,8 +158,8 @@ ], containing_type=None, serialized_options=None, - serialized_start=3156, - serialized_end=3195, + serialized_start=3180, + serialized_end=3219, ) _sym_db.RegisterEnumDescriptor(_MIGRATIONTYPE) @@ -679,6 +679,13 @@ message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='external_mysql', full_name='binlogdata.BinlogSource.external_mysql', index=7, + number=8, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], @@ -692,7 +699,7 @@ oneofs=[ ], serialized_start=1153, - serialized_end=1375, + serialized_end=1399, ) @@ -729,8 +736,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=1377, - serialized_end=1443, + serialized_start=1401, + serialized_end=1467, ) @@ -767,8 +774,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=1445, - serialized_end=1519, + serialized_start=1469, + serialized_end=1543, ) @@ -805,8 +812,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=1521, - serialized_end=1583, + serialized_start=1545, + serialized_end=1607, ) @@ -850,8 +857,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=1585, - serialized_end=1643, + serialized_start=1609, + serialized_end=1667, ) @@ -881,8 +888,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=1645, - serialized_end=1696, + serialized_start=1669, + serialized_end=1720, ) @@ -919,8 +926,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=1698, - serialized_end=1746, + serialized_start=1722, + serialized_end=1770, ) @@ -992,8 +999,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=1749, - serialized_end=1976, + serialized_start=1773, + serialized_end=2000, ) @@ -1086,8 +1093,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=1979, - serialized_end=2264, + serialized_start=2003, + serialized_end=2288, ) @@ -1145,8 +1152,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=2267, - serialized_end=2466, + serialized_start=2291, + serialized_end=2490, ) @@ -1176,8 +1183,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=2468, - serialized_end=2521, + serialized_start=2492, + serialized_end=2545, ) @@ -1235,8 +1242,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=2524, - serialized_end=2724, + serialized_start=2548, + serialized_end=2748, ) @@ -1294,8 +1301,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=2727, - serialized_end=2878, + serialized_start=2751, + serialized_end=2902, ) _BINLOGTRANSACTION_STATEMENT.fields_by_name['category'].enum_type = _BINLOGTRANSACTION_STATEMENT_CATEGORY From c7926ef146cce25f50ea79390ca51ad7e97e9359 Mon Sep 17 00:00:00 2001 From: Rafael Chacon Date: Fri, 18 Oct 2019 15:19:47 -0700 Subject: [PATCH 10/28] Fixes some bugs in dbconfigs and vstreamer client after inital testing Signed-off-by: Rafael Chacon --- go/vt/dbconfigs/dbconfigs.go | 19 +++++++++++++------ go/vt/srvtopo/resilient_server.go | 12 ++++++++++-- go/vt/vttablet/tabletmanager/action_agent.go | 2 +- .../tabletmanager/vreplication/controller.go | 16 +++++++++++----- .../vreplication/vstreamer_client.go | 2 +- go/vt/vttablet/tabletserver/tabletserver.go | 2 +- 6 files changed, 37 insertions(+), 16 deletions(-) diff --git a/go/vt/dbconfigs/dbconfigs.go b/go/vt/dbconfigs/dbconfigs.go index af675779b18..58aedd84f25 100644 --- a/go/vt/dbconfigs/dbconfigs.go +++ b/go/vt/dbconfigs/dbconfigs.go @@ -170,12 +170,14 @@ func (dbcfgs *DBConfigs) Repl() *mysql.ConnParams { // ExternalRepl returns connection parameters for repl with no dbname set. func (dbcfgs *DBConfigs) ExternalRepl() *mysql.ConnParams { - return dbcfgs.makeParams(Repl, false) + return dbcfgs.makeParams(ExternalRepl, false) } -// ExternalReplWithDb returns connection parameters for repl with dbname set. -func (dbcfgs *DBConfigs) ExternalReplWithDb() *mysql.ConnParams { - return dbcfgs.makeParams(Repl, true) +// ExternalReplWithDB returns connection parameters for repl with dbname set. +func (dbcfgs *DBConfigs) ExternalReplWithDB() *mysql.ConnParams { + params := dbcfgs.makeParams(Repl, false) + params.DbName = params.DeprecatedDBName + return params } // AppWithDB returns connection parameters for app with dbname set. @@ -248,8 +250,13 @@ func HasConnectionParams() bool { // is used to initialize the per-user conn params. func Init(defaultSocketFile string) (*DBConfigs, error) { // The new base configs, if set, supersede legacy settings. - for _, uc := range dbConfigs.userConfigs { - if HasConnectionParams() { + for user, uc := range dbConfigs.userConfigs { + // TODO @rafael: For ExternalRepl we need to respect the provided host / port + // At the moment this is an snowflake user connection type that it used by + // vreplication to connect to external mysql hosts that are not part of a vitess + // cluster. In the future we need to refactor all dbconfig to support custom users + // in a more flexible way. + if HasConnectionParams() && user != ExternalRepl { uc.param.Host = baseConfig.Host uc.param.Port = baseConfig.Port uc.param.UnixSocket = baseConfig.UnixSocket diff --git a/go/vt/srvtopo/resilient_server.go b/go/vt/srvtopo/resilient_server.go index 6340bfe6551..169838c7f52 100644 --- a/go/vt/srvtopo/resilient_server.go +++ b/go/vt/srvtopo/resilient_server.go @@ -206,16 +206,24 @@ type srvKeyspaceEntry struct { // NewResilientServer creates a new ResilientServer // based on the provided topo.Server. -func NewResilientServer(base *topo.Server, counterPrefix string) *ResilientServer { +func NewResilientServer(base *topo.Server, counterPrefix string, publishMetrics bool) *ResilientServer { if *srvTopoCacheRefresh > *srvTopoCacheTTL { log.Fatalf("srv_topo_cache_refresh must be less than or equal to srv_topo_cache_ttl") } + var metric string + + if publishMetrics { + metric = counterPrefix + "Counts" + } else { + metric = "" + } + return &ResilientServer{ topoServer: base, cacheTTL: *srvTopoCacheTTL, cacheRefresh: *srvTopoCacheRefresh, - counts: stats.NewCountersWithSingleLabel(counterPrefix+"Counts", "Resilient srvtopo server operations", "type"), + counts: stats.NewCountersWithSingleLabel(metric, "Resilient srvtopo server operations", "type"), srvKeyspaceNamesCache: make(map[string]*srvKeyspaceNamesEntry), srvKeyspaceCache: make(map[string]*srvKeyspaceEntry), diff --git a/go/vt/vttablet/tabletmanager/action_agent.go b/go/vt/vttablet/tabletmanager/action_agent.go index fdccaacc715..a4b6a0536a0 100644 --- a/go/vt/vttablet/tabletmanager/action_agent.go +++ b/go/vt/vttablet/tabletmanager/action_agent.go @@ -291,7 +291,7 @@ func NewActionAgent( agent.VREngine = vreplication.NewEngine(ts, tabletAlias.Cell, mysqld, func() binlogplayer.DBClient { return binlogplayer.NewDBClient(agent.DBConfigs.FilteredWithDB()) }, - agent.DBConfigs.ExternalRepl(), + agent.DBConfigs.ExternalReplWithDB(), agent.DBConfigs.FilteredWithDB().DbName, ) servenv.OnTerm(agent.VREngine.Close) diff --git a/go/vt/vttablet/tabletmanager/vreplication/controller.go b/go/vt/vttablet/tabletmanager/vreplication/controller.go index 0f98bc47f77..246e8e04d95 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/controller.go +++ b/go/vt/vttablet/tabletmanager/vreplication/controller.go @@ -36,6 +36,7 @@ import ( "vitess.io/vitess/go/vt/topo" binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" + topodatapb "vitess.io/vitess/go/vt/proto/topodata" ) var retryDelay = flag.Duration("vreplication_retry_delay", 5*time.Second, "delay before retrying a failed binlog connection") @@ -121,7 +122,9 @@ func newController(ctx context.Context, params map[string]string, dbClientFactor func (ct *controller) run(ctx context.Context) { defer func() { log.Infof("stream %v: stopped", ct.id) - ct.tabletPicker.Close() + if ct.tabletPicker != nil { + ct.tabletPicker.Close() + } close(ct.done) }() @@ -174,11 +177,14 @@ func (ct *controller) runBlp(ctx context.Context) (err error) { } defer dbClient.Close() - tablet, err := ct.tabletPicker.Pick(ctx) - if err != nil { - return err + var tablet *topodatapb.Tablet + if ct.source.GetExternalMysql() == "" { + tablet, err = ct.tabletPicker.Pick(ctx) + if err != nil { + return err + } + ct.sourceTablet.Set(tablet.Alias.String()) } - ct.sourceTablet.Set(tablet.Alias.String()) switch { case len(ct.source.Tables) > 0: diff --git a/go/vt/vttablet/tabletmanager/vreplication/vstreamer_client.go b/go/vt/vttablet/tabletmanager/vreplication/vstreamer_client.go index b6d3f539e01..ce9eee74bfe 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vstreamer_client.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vstreamer_client.go @@ -160,7 +160,7 @@ func (vsClient *MySQLVStreamerClient) Open(ctx context.Context) (err error) { } topo := memorytopo.NewServer("mysqlstreamer") - srvTopo := srvtopo.NewResilientServer(topo, "TestTopo") + srvTopo := srvtopo.NewResilientServer(topo, "streamertopo", false) vsClient.vsEngine = vstreamer.NewEngine(srvTopo, sourceSe) vsClient.vsEngine.InitDBConfig(vsClient.sourceConnParams) diff --git a/go/vt/vttablet/tabletserver/tabletserver.go b/go/vt/vttablet/tabletserver/tabletserver.go index 657d118de0f..ff76ed886fa 100644 --- a/go/vt/vttablet/tabletserver/tabletserver.go +++ b/go/vt/vttablet/tabletserver/tabletserver.go @@ -288,7 +288,7 @@ func NewTabletServer(config tabletenv.TabletConfig, topoServer *topo.Server, ali // So that vtcombo doesn't even call it once, on the first tablet. // And we can remove the tsOnce variable. tsOnce.Do(func() { - srvTopoServer = srvtopo.NewResilientServer(topoServer, "TabletSrvTopo") + srvTopoServer = srvtopo.NewResilientServer(topoServer, "TabletSrvTopo", true) stats.NewGaugeFunc("TabletState", "Tablet server state", func() int64 { tsv.mu.Lock() state := tsv.state From 46c0fe47ddaee07242ed11f58e3a514d9abe2b3d Mon Sep 17 00:00:00 2001 From: Rafael Chacon Date: Mon, 21 Oct 2019 10:07:19 -0700 Subject: [PATCH 11/28] Adds QPS chart to tablet vreplication section Signed-off-by: Rafael Chacon --- .../prometheusbackend/prometheusbackend.go | 2 +- go/stats/rates.go | 29 ++++++ go/vt/binlog/binlogplayer/binlog_player.go | 2 +- go/vt/dbconfigs/dbconfigs.go | 2 +- .../tabletmanager/vreplication/stats.go | 95 +++++++++++++++++++ 5 files changed, 127 insertions(+), 3 deletions(-) diff --git a/go/stats/prometheusbackend/prometheusbackend.go b/go/stats/prometheusbackend/prometheusbackend.go index 961d88fe01a..e13337c2036 100644 --- a/go/stats/prometheusbackend/prometheusbackend.go +++ b/go/stats/prometheusbackend/prometheusbackend.go @@ -66,7 +66,7 @@ func (be PromBackend) publishPrometheusMetric(name string, v expvar.Var) { newMultiTimingsCollector(st, be.buildPromName(name)) case *stats.Histogram: newHistogramCollector(st, be.buildPromName(name)) - case *stats.String, stats.StringFunc, stats.StringMapFunc, *stats.Rates: + case *stats.String, stats.StringFunc, stats.StringMapFunc, *stats.Rates, *stats.RatesFunc: // Silently ignore these types since they don't make sense to // export to Prometheus' data model. default: diff --git a/go/stats/rates.go b/go/stats/rates.go index 7cb7ded97f0..b93691f5f7a 100644 --- a/go/stats/rates.go +++ b/go/stats/rates.go @@ -182,3 +182,32 @@ func (rt *Rates) String() string { } return string(data) } + +type RatesFunc struct { + F func() map[string][]float64 + help string +} + +func NewRateFunc(name string, help string, f func() map[string][]float64) *RatesFunc { + c := &RatesFunc{ + F: f, + help: help, + } + + if name != "" { + publish(name, c) + } + return c +} + +func (rf *RatesFunc) Help() string { + return rf.help +} + +func (rf *RatesFunc) String() string { + data, err := json.Marshal(rf.F()) + if err != nil { + data, _ = json.Marshal(err.Error()) + } + return string(data) +} diff --git a/go/vt/binlog/binlogplayer/binlog_player.go b/go/vt/binlog/binlogplayer/binlog_player.go index 52d119ec3f1..74792c3b871 100644 --- a/go/vt/binlog/binlogplayer/binlog_player.go +++ b/go/vt/binlog/binlogplayer/binlog_player.go @@ -111,7 +111,7 @@ func (bps *Stats) MessageHistory() []string { func NewStats() *Stats { bps := &Stats{} bps.Timings = stats.NewTimings("", "", "") - bps.Rates = stats.NewRates("", bps.Timings, 15, 60e9) + bps.Rates = stats.NewRates("", bps.Timings, 15*60/5, 5*time.Second) bps.History = history.New(3) bps.SecondsBehindMaster.Set(math.MaxInt64) return bps diff --git a/go/vt/dbconfigs/dbconfigs.go b/go/vt/dbconfigs/dbconfigs.go index 58aedd84f25..da9346e433a 100644 --- a/go/vt/dbconfigs/dbconfigs.go +++ b/go/vt/dbconfigs/dbconfigs.go @@ -175,7 +175,7 @@ func (dbcfgs *DBConfigs) ExternalRepl() *mysql.ConnParams { // ExternalReplWithDB returns connection parameters for repl with dbname set. func (dbcfgs *DBConfigs) ExternalReplWithDB() *mysql.ConnParams { - params := dbcfgs.makeParams(Repl, false) + params := dbcfgs.makeParams(ExternalRepl, false) params.DbName = params.DeprecatedDBName return params } diff --git a/go/vt/vttablet/tabletmanager/vreplication/stats.go b/go/vt/vttablet/tabletmanager/vreplication/stats.go index b80e6de4ad5..33672cf3cff 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/stats.go +++ b/go/vt/vttablet/tabletmanager/vreplication/stats.go @@ -73,6 +73,35 @@ func (st *vrStats) register() { } return result }) + + stats.NewCounterFunc( + "VReplicationTotalSecondsBehindMaster", + "vreplication seconds behind master aggregated across all streams", + func() int64 { + st.mu.Lock() + defer st.mu.Unlock() + result := int64(0) + for _, ct := range st.controllers { + result += ct.blpStats.SecondsBehindMaster.Get() + } + return result + }) + + stats.NewRateFunc( + "VReplicationQPS", + "vreplication operations per second aggregated across all streams", + func() map[string][]float64 { + st.mu.Lock() + defer st.mu.Unlock() + result := make(map[string][]float64) + for _, ct := range st.controllers { + for k, v := range ct.blpStats.Rates.Get() { + result[k] = v + } + } + return result + }) + stats.Publish("VReplicationSource", stats.StringMapFunc(func() map[string]string { st.mu.Lock() defer st.mu.Unlock() @@ -193,5 +222,71 @@ var vreplicationTemplate = ` {{range $key, $values := .Rates}}{{$key}}: {{range $values}}{{.}} {{end}}
{{end}} {{range $index, $value := .Messages}}{{$value}}
{{end}} {{end}} +
QPS All Streams
+ + + {{else}}VReplication is closed.{{end}} ` From e73faef5489706551d22f19da9883e91adee339f Mon Sep 17 00:00:00 2001 From: Rafael Chacon Date: Mon, 21 Oct 2019 15:36:27 -0700 Subject: [PATCH 12/28] Adds flag to register metrics. * This will be removed in future PR. Adding while in POC Signed-off-by: Rafael Chacon --- go/cmd/vtcombo/main.go | 2 +- go/cmd/vtgate/vtgate.go | 2 +- go/vt/vttablet/tabletserver/vstreamer/testenv/testenv.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go/cmd/vtcombo/main.go b/go/cmd/vtcombo/main.go index f92181861f4..db4782fab49 100644 --- a/go/cmd/vtcombo/main.go +++ b/go/cmd/vtcombo/main.go @@ -118,7 +118,7 @@ func main() { } // vtgate configuration and init - resilientServer := srvtopo.NewResilientServer(ts, "ResilientSrvTopoServer") + resilientServer := srvtopo.NewResilientServer(ts, "ResilientSrvTopoServer", true) healthCheck := discovery.NewHealthCheck(1*time.Millisecond /*retryDelay*/, 1*time.Hour /*healthCheckTimeout*/) tabletTypesToWait := []topodatapb.TabletType{ topodatapb.TabletType_MASTER, diff --git a/go/cmd/vtgate/vtgate.go b/go/cmd/vtgate/vtgate.go index 2a3aa9445d3..d0ccd2632b4 100644 --- a/go/cmd/vtgate/vtgate.go +++ b/go/cmd/vtgate/vtgate.go @@ -66,7 +66,7 @@ func main() { ts := topo.Open() defer ts.Close() - resilientServer = srvtopo.NewResilientServer(ts, "ResilientSrvTopoServer") + resilientServer = srvtopo.NewResilientServer(ts, "ResilientSrvTopoServer", true) healthCheck = discovery.NewHealthCheck(*healthCheckRetryDelay, *healthCheckTimeout) healthCheck.RegisterStats() diff --git a/go/vt/vttablet/tabletserver/vstreamer/testenv/testenv.go b/go/vt/vttablet/tabletserver/vstreamer/testenv/testenv.go index b43ac316af8..e3b9335b7a3 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/testenv/testenv.go +++ b/go/vt/vttablet/tabletserver/vstreamer/testenv/testenv.go @@ -76,7 +76,7 @@ func Init() (*Env, error) { if err := te.TopoServ.CreateShard(ctx, te.KeyspaceName, te.ShardName); err != nil { panic(err) } - te.SrvTopo = srvtopo.NewResilientServer(te.TopoServ, "TestTopo") + te.SrvTopo = srvtopo.NewResilientServer(te.TopoServ, "TestTopo", true) cfg := vttest.Config{ Topology: &vttestpb.VTTestTopology{ From cf57589a40de9bfa529dda29c3719ac0c8a0153a Mon Sep 17 00:00:00 2001 From: Rafael Chacon Date: Tue, 29 Oct 2019 09:13:58 -0700 Subject: [PATCH 13/28] Addresses comments from review Signed-off-by: Rafael Chacon --- go/cmd/vtcombo/main.go | 2 +- go/cmd/vtgate/vtgate.go | 2 +- go/vt/dbconfigs/dbconfigs.go | 15 ++++----- go/vt/srvtopo/resilient_server.go | 4 +-- .../tabletmanager/vreplication/stats_test.go | 3 +- .../vreplication/vstreamer_client.go | 31 +++++++++++++------ go/vt/vttablet/tabletserver/tabletserver.go | 2 +- .../tabletserver/vstreamer/testenv/testenv.go | 2 +- 8 files changed, 38 insertions(+), 23 deletions(-) diff --git a/go/cmd/vtcombo/main.go b/go/cmd/vtcombo/main.go index db4782fab49..f92181861f4 100644 --- a/go/cmd/vtcombo/main.go +++ b/go/cmd/vtcombo/main.go @@ -118,7 +118,7 @@ func main() { } // vtgate configuration and init - resilientServer := srvtopo.NewResilientServer(ts, "ResilientSrvTopoServer", true) + resilientServer := srvtopo.NewResilientServer(ts, "ResilientSrvTopoServer") healthCheck := discovery.NewHealthCheck(1*time.Millisecond /*retryDelay*/, 1*time.Hour /*healthCheckTimeout*/) tabletTypesToWait := []topodatapb.TabletType{ topodatapb.TabletType_MASTER, diff --git a/go/cmd/vtgate/vtgate.go b/go/cmd/vtgate/vtgate.go index d0ccd2632b4..2a3aa9445d3 100644 --- a/go/cmd/vtgate/vtgate.go +++ b/go/cmd/vtgate/vtgate.go @@ -66,7 +66,7 @@ func main() { ts := topo.Open() defer ts.Close() - resilientServer = srvtopo.NewResilientServer(ts, "ResilientSrvTopoServer", true) + resilientServer = srvtopo.NewResilientServer(ts, "ResilientSrvTopoServer") healthCheck = discovery.NewHealthCheck(*healthCheckRetryDelay, *healthCheckTimeout) healthCheck.RegisterStats() diff --git a/go/vt/dbconfigs/dbconfigs.go b/go/vt/dbconfigs/dbconfigs.go index da9346e433a..6816b7d9665 100644 --- a/go/vt/dbconfigs/dbconfigs.go +++ b/go/vt/dbconfigs/dbconfigs.go @@ -170,7 +170,7 @@ func (dbcfgs *DBConfigs) Repl() *mysql.ConnParams { // ExternalRepl returns connection parameters for repl with no dbname set. func (dbcfgs *DBConfigs) ExternalRepl() *mysql.ConnParams { - return dbcfgs.makeParams(ExternalRepl, false) + return dbcfgs.makeParams(ExternalRepl, true) } // ExternalReplWithDB returns connection parameters for repl with dbname set. @@ -298,12 +298,13 @@ func Init(defaultSocketFile string) (*DBConfigs, error) { func NewTestDBConfigs(genParams, appDebugParams mysql.ConnParams, dbName string) *DBConfigs { dbcfgs := &DBConfigs{ userConfigs: map[string]*userConfig{ - App: {param: genParams}, - AppDebug: {param: appDebugParams}, - AllPrivs: {param: genParams}, - Dba: {param: genParams}, - Filtered: {param: genParams}, - Repl: {param: genParams}, + App: {param: genParams}, + AppDebug: {param: appDebugParams}, + AllPrivs: {param: genParams}, + Dba: {param: genParams}, + Filtered: {param: genParams}, + Repl: {param: genParams}, + ExternalRepl: {param: genParams}, }, } dbcfgs.DBName.Set(dbName) diff --git a/go/vt/srvtopo/resilient_server.go b/go/vt/srvtopo/resilient_server.go index 169838c7f52..28ac127d046 100644 --- a/go/vt/srvtopo/resilient_server.go +++ b/go/vt/srvtopo/resilient_server.go @@ -206,14 +206,14 @@ type srvKeyspaceEntry struct { // NewResilientServer creates a new ResilientServer // based on the provided topo.Server. -func NewResilientServer(base *topo.Server, counterPrefix string, publishMetrics bool) *ResilientServer { +func NewResilientServer(base *topo.Server, counterPrefix string) *ResilientServer { if *srvTopoCacheRefresh > *srvTopoCacheTTL { log.Fatalf("srv_topo_cache_refresh must be less than or equal to srv_topo_cache_ttl") } var metric string - if publishMetrics { + if counterPrefix == "" { metric = counterPrefix + "Counts" } else { metric = "" diff --git a/go/vt/vttablet/tabletmanager/vreplication/stats_test.go b/go/vt/vttablet/tabletmanager/vreplication/stats_test.go index a047ede7955..82cbfd31c53 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/stats_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/stats_test.go @@ -19,6 +19,7 @@ package vreplication import ( "bytes" "html/template" + "strings" "testing" "time" @@ -111,7 +112,7 @@ func TestStatusHtml(t *testing.T) { tpl := template.Must(template.New("test").Parse(vreplicationTemplate)) buf := bytes.NewBuffer(nil) tpl.Execute(buf, testStats.status()) - if buf.String() != wantOut { + if strings.Contains(buf.String(), wantOut) { t.Errorf("output: %v, want %v", buf, wantOut) } } diff --git a/go/vt/vttablet/tabletmanager/vreplication/vstreamer_client.go b/go/vt/vttablet/tabletmanager/vreplication/vstreamer_client.go index ce9eee74bfe..b35129342c8 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vstreamer_client.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vstreamer_client.go @@ -41,8 +41,10 @@ import ( ) var ( - _ VStreamerClient = (*TabletVStreamerClient)(nil) - _ VStreamerClient = (*MySQLVStreamerClient)(nil) + _ VStreamerClient = (*TabletVStreamerClient)(nil) + _ VStreamerClient = (*MySQLVStreamerClient)(nil) + mysqlStreamerClientOnce sync.Once + mysqlSrvTopo *srvtopo.ResilientServer ) // VStreamerClient exposes the core interface of a vstreamer @@ -80,6 +82,7 @@ type MySQLVStreamerClient struct { sourceConnParams *mysql.ConnParams vsEngine *vstreamer.Engine + sourceSe *schema.Engine } // NewTabletVStreamerClient creates a new TabletVStreamerClient @@ -109,9 +112,12 @@ func (vsClient *TabletVStreamerClient) Open(ctx context.Context) (err error) { // Close part of the VStreamerClient interface func (vsClient *TabletVStreamerClient) Close(ctx context.Context) (err error) { + vsClient.mu.Lock() + defer vsClient.mu.Unlock() if !vsClient.isOpen { return nil } + vsClient.isOpen = false return vsClient.tsQueryService.Close(ctx) } @@ -150,19 +156,21 @@ func (vsClient *MySQLVStreamerClient) Open(ctx context.Context) (err error) { } vsClient.isOpen = true + mysqlStreamerClientOnce.Do(func() { + memorytopo := memorytopo.NewServer("mysqlstreamer") + mysqlSrvTopo = srvtopo.NewResilientServer(memorytopo, "") + }) + // Let's create all the required components by vstreamer.Engine - sourceSe := schema.NewEngine(checker{}, tabletenv.DefaultQsConfig) - sourceSe.InitDBConfig(vsClient.sourceConnParams) - err = sourceSe.Open() + vsClient.sourceSe = schema.NewEngine(checker{}, tabletenv.DefaultQsConfig) + vsClient.sourceSe.InitDBConfig(vsClient.sourceConnParams) + err = vsClient.sourceSe.Open() if err != nil { return err } - topo := memorytopo.NewServer("mysqlstreamer") - srvTopo := srvtopo.NewResilientServer(topo, "streamertopo", false) - - vsClient.vsEngine = vstreamer.NewEngine(srvTopo, sourceSe) + vsClient.vsEngine = vstreamer.NewEngine(mysqlSrvTopo, vsClient.sourceSe) vsClient.vsEngine.InitDBConfig(vsClient.sourceConnParams) err = vsClient.vsEngine.Open("mysqlstreamer", "cell1") @@ -175,10 +183,15 @@ func (vsClient *MySQLVStreamerClient) Open(ctx context.Context) (err error) { // Close part of the VStreamerClient interface func (vsClient *MySQLVStreamerClient) Close(ctx context.Context) (err error) { + vsClient.mu.Lock() + defer vsClient.mu.Unlock() if !vsClient.isOpen { return nil } + + vsClient.isOpen = false vsClient.vsEngine.Close() + vsClient.sourceSe.Close() return nil } diff --git a/go/vt/vttablet/tabletserver/tabletserver.go b/go/vt/vttablet/tabletserver/tabletserver.go index ff76ed886fa..657d118de0f 100644 --- a/go/vt/vttablet/tabletserver/tabletserver.go +++ b/go/vt/vttablet/tabletserver/tabletserver.go @@ -288,7 +288,7 @@ func NewTabletServer(config tabletenv.TabletConfig, topoServer *topo.Server, ali // So that vtcombo doesn't even call it once, on the first tablet. // And we can remove the tsOnce variable. tsOnce.Do(func() { - srvTopoServer = srvtopo.NewResilientServer(topoServer, "TabletSrvTopo", true) + srvTopoServer = srvtopo.NewResilientServer(topoServer, "TabletSrvTopo") stats.NewGaugeFunc("TabletState", "Tablet server state", func() int64 { tsv.mu.Lock() state := tsv.state diff --git a/go/vt/vttablet/tabletserver/vstreamer/testenv/testenv.go b/go/vt/vttablet/tabletserver/vstreamer/testenv/testenv.go index e3b9335b7a3..b43ac316af8 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/testenv/testenv.go +++ b/go/vt/vttablet/tabletserver/vstreamer/testenv/testenv.go @@ -76,7 +76,7 @@ func Init() (*Env, error) { if err := te.TopoServ.CreateShard(ctx, te.KeyspaceName, te.ShardName); err != nil { panic(err) } - te.SrvTopo = srvtopo.NewResilientServer(te.TopoServ, "TestTopo", true) + te.SrvTopo = srvtopo.NewResilientServer(te.TopoServ, "TestTopo") cfg := vttest.Config{ Topology: &vttestpb.VTTestTopology{ From 90ee27a7a9741bdc2247a383f2febc982be8a8f2 Mon Sep 17 00:00:00 2001 From: Rafael Chacon Date: Wed, 30 Oct 2019 13:40:34 -0700 Subject: [PATCH 14/28] WIP: Adds test for vstreamer client Signed-off-by: Rafael Chacon --- .../vreplication/framework_test.go | 2 +- .../vreplication/vplayer_test.go | 2882 +++++++++-------- 2 files changed, 1507 insertions(+), 1377 deletions(-) diff --git a/go/vt/vttablet/tabletmanager/vreplication/framework_test.go b/go/vt/vttablet/tabletmanager/vreplication/framework_test.go index 461ba0ab0df..27f78253294 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/framework_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/framework_test.go @@ -96,7 +96,7 @@ func TestMain(m *testing.M) { return 1 } - playerEngine = NewEngine(env.TopoServ, env.Cells[0], env.Mysqld, realDBClientFactory, nil, vrepldb) + playerEngine = NewEngine(env.TopoServ, env.Cells[0], env.Mysqld, realDBClientFactory, env.Dbcfgs.ExternalReplWithDB(), vrepldb) if err := playerEngine.Open(context.Background()); err != nil { fmt.Fprintf(os.Stderr, "%v", err) return 1 diff --git a/go/vt/vttablet/tabletmanager/vreplication/vplayer_test.go b/go/vt/vttablet/tabletmanager/vreplication/vplayer_test.go index 110e5bc8775..d4ee7160d55 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vplayer_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vplayer_test.go @@ -17,9 +17,9 @@ limitations under the License. package vreplication import ( - "flag" + //"flag" "fmt" - "strings" + // "strings" "sync" "testing" "time" @@ -27,14 +27,14 @@ import ( "golang.org/x/net/context" "vitess.io/vitess/go/mysql" - "vitess.io/vitess/go/sqltypes" + // "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/binlog/binlogplayer" binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" topodatapb "vitess.io/vitess/go/vt/proto/topodata" ) -func TestPlayerFilters(t *testing.T) { +func TestMySQLVstreamerClient(t *testing.T) { defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) execStatements(t, []string{ @@ -81,7 +81,14 @@ func TestPlayerFilters(t *testing.T) { Match: "/nopk", }}, } - cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") + + bls := &binlogdatapb.BinlogSource{ + Filter: filter, + OnDdl: binlogdatapb.OnDDLAction_IGNORE, + ExternalMysql: "erepl", + } + + cancel, _ := startVReplicationV2(t, filter, bls, "") defer cancel() testcases := []struct { @@ -115,1401 +122,1524 @@ func TestPlayerFilters(t *testing.T) { data: [][]string{ {"1", "bbb"}, }, - }, { - // delete with insertNormal - input: "delete from src1 where id=1", - output: []string{ - "begin", - "delete from dst1 where id=1", - "/update _vt.vreplication set pos=", - "commit", - }, - table: "dst1", - data: [][]string{}, - }, { - // insert with insertOnDup - input: "insert into src2 values(1, 2, 3)", - output: []string{ - "begin", - "insert into dst2(id,val1,sval2,rcount) values (1,2,ifnull(3, 0),1) on duplicate key update val1=values(val1), sval2=sval2+ifnull(values(sval2), 0), rcount=rcount+1", - "/update _vt.vreplication set pos=", - "commit", - }, - table: "dst2", - data: [][]string{ - {"1", "2", "3", "1"}, - }, - }, { - // update with insertOnDup - input: "update src2 set val1=5, val2=1 where id=1", - output: []string{ - "begin", - "update dst2 set val1=5, sval2=sval2-ifnull(3, 0)+ifnull(1, 0), rcount=rcount where id=1", - "/update _vt.vreplication set pos=", - "commit", - }, - table: "dst2", - data: [][]string{ - {"1", "5", "1", "1"}, - }, - }, { - // delete with insertOnDup - input: "delete from src2 where id=1", - output: []string{ - "begin", - "update dst2 set val1=null, sval2=sval2-ifnull(1, 0), rcount=rcount-1 where id=1", - "/update _vt.vreplication set pos=", - "commit", - }, - table: "dst2", - data: [][]string{ - {"1", "", "0", "0"}, - }, - }, { - // insert with insertIgnore - input: "insert into src3 values(1, 'aaa')", - output: []string{ - "begin", - "insert ignore into dst3(id,val) values (1,'aaa')", - "/update _vt.vreplication set pos=", - "commit", - }, - table: "dst3", - data: [][]string{ - {"1", "aaa"}, - }, - }, { - // update with insertIgnore - input: "update src3 set val='bbb'", - output: []string{ - "begin", - "insert ignore into dst3(id,val) values (1,'bbb')", - "/update _vt.vreplication set pos=", - "commit", - }, - table: "dst3", - data: [][]string{ - {"1", "aaa"}, - }, - }, { - // delete with insertIgnore - input: "delete from src3 where id=1", - output: []string{ - "begin", - "/update _vt.vreplication set pos=", - "commit", - }, - table: "dst3", - data: [][]string{ - {"1", "aaa"}, - }, - }, { - // insert: regular expression filter - input: "insert into yes values(1, 'aaa')", - output: []string{ - "begin", - "insert into yes(id,val) values (1,'aaa')", - "/update _vt.vreplication set pos=", - "commit", - }, - table: "yes", - data: [][]string{ - {"1", "aaa"}, - }, - }, { - // update: regular expression filter - input: "update yes set val='bbb'", - output: []string{ - "begin", - "update yes set val='bbb' where id=1", - "/update _vt.vreplication set pos=", - "commit", - }, - table: "yes", - data: [][]string{ - {"1", "bbb"}, - }, - }, { - // table should not match a rule - input: "insert into no values(1, 'aaa')", - output: []string{}, - }, { - // nopk: insert - input: "insert into nopk values(1, 'aaa')", - output: []string{ - "begin", - "insert into nopk(id,val) values (1,'aaa')", - "/update _vt.vreplication set pos=", - "commit", - }, - table: "nopk", - data: [][]string{ - {"1", "aaa"}, - }, - }, { - // nopk: update - input: "update nopk set val='bbb' where id=1", - output: []string{ - "begin", - "delete from nopk where id=1 and val='aaa'", - "insert into nopk(id,val) values (1,'bbb')", - "/update _vt.vreplication set pos=", - "commit", - }, - table: "nopk", - data: [][]string{ - {"1", "bbb"}, - }, - }, { - // nopk: delete - input: "delete from nopk where id=1", - output: []string{ - "begin", - "delete from nopk where id=1 and val='bbb'", - "/update _vt.vreplication set pos=", - "commit", - }, - table: "nopk", - data: [][]string{}, }} for _, tcases := range testcases { execStatements(t, []string{tcases.input}) + time.Sleep(2 * time.Minute) expectDBClientQueries(t, tcases.output) if tcases.table != "" { expectData(t, tcases.table, tcases.data) } } -} - -func TestPlayerKeywordNames(t *testing.T) { - defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) - - execStatements(t, []string{ - "create table `begin`(`primary` int, `column` varbinary(128), primary key(`primary`))", - fmt.Sprintf("create table %s.`begin`(`primary` int, `column` varbinary(128), primary key(`primary`))", vrepldb), - "create table `rollback`(`primary` int, `column` varbinary(128), primary key(`primary`))", - fmt.Sprintf("create table %s.`rollback`(`primary` int, `column` varbinary(128), primary key(`primary`))", vrepldb), - "create table `commit`(`primary` int, `column` varbinary(128), primary key(`primary`))", - fmt.Sprintf("create table %s.`commit`(`primary` int, `column` varbinary(128), primary key(`primary`))", vrepldb), - }) - defer execStatements(t, []string{ - "drop table `begin`", - fmt.Sprintf("drop table %s.`begin`", vrepldb), - "drop table `rollback`", - fmt.Sprintf("drop table %s.`rollback`", vrepldb), - "drop table `commit`", - fmt.Sprintf("drop table %s.`commit`", vrepldb), - }) - env.SchemaEngine.Reload(context.Background()) - filter := &binlogdatapb.Filter{ - Rules: []*binlogdatapb.Rule{{ - Match: "begin", - Filter: "select * from `begin`", - }, { - Match: "rollback", - Filter: "select `primary`, `column` from `rollback`", - }, { - Match: "commit", - Filter: "select `primary`+1 as `primary`, concat(`column`, 'a') as `column` from `commit`", - }}, - } - cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") - defer cancel() +} - testcases := []struct { - input string - output []string - table string - data [][]string - }{{ - input: "insert into `begin` values(1, 'aaa')", - output: []string{ - "begin", - "insert into `begin`(`primary`,`column`) values (1,'aaa')", - "/update _vt.vreplication set pos=", - "commit", - }, - table: "begin", - data: [][]string{ - {"1", "aaa"}, - }, - }, { - input: "update `begin` set `column`='bbb'", - output: []string{ - "begin", - "update `begin` set `column`='bbb' where `primary`=1", - "/update _vt.vreplication set pos=", - "commit", - }, - table: "begin", - data: [][]string{ - {"1", "bbb"}, - }, - }, { - input: "delete from `begin` where `primary`=1", - output: []string{ - "begin", - "delete from `begin` where `primary`=1", - "/update _vt.vreplication set pos=", - "commit", - }, - table: "begin", - data: [][]string{}, - }, { - input: "insert into `rollback` values(1, 'aaa')", - output: []string{ - "begin", - "insert into `rollback`(`primary`,`column`) values (1,'aaa')", - "/update _vt.vreplication set pos=", - "commit", - }, - table: "rollback", - data: [][]string{ - {"1", "aaa"}, - }, - }, { - input: "update `rollback` set `column`='bbb'", - output: []string{ - "begin", - "update `rollback` set `column`='bbb' where `primary`=1", - "/update _vt.vreplication set pos=", - "commit", - }, - table: "rollback", - data: [][]string{ - {"1", "bbb"}, - }, - }, { - input: "delete from `rollback` where `primary`=1", - output: []string{ - "begin", - "delete from `rollback` where `primary`=1", - "/update _vt.vreplication set pos=", - "commit", - }, - table: "rollback", - data: [][]string{}, - }, { - input: "insert into `commit` values(1, 'aaa')", - output: []string{ - "begin", - "insert into `commit`(`primary`,`column`) values (1 + 1,concat('aaa', 'a'))", - "/update _vt.vreplication set pos=", - "commit", - }, - table: "commit", - data: [][]string{ - {"2", "aaaa"}, - }, - }, { - input: "update `commit` set `column`='bbb' where `primary`=1", - output: []string{ - "begin", - "update `commit` set `column`=concat('bbb', 'a') where `primary`=(1 + 1)", - "/update _vt.vreplication set pos=", - "commit", - }, - table: "commit", - data: [][]string{ - {"2", "bbba"}, - }, - }, { - input: "update `commit` set `primary`=2 where `primary`=1", - output: []string{ - "begin", - "delete from `commit` where `primary`=(1 + 1)", - "insert into `commit`(`primary`,`column`) values (2 + 1,concat('bbb', 'a'))", - "/update _vt.vreplication set pos=", - "commit", - }, - table: "commit", - data: [][]string{ - {"3", "bbba"}, - }, - }, { - input: "delete from `commit` where `primary`=2", - output: []string{ - "begin", - "delete from `commit` where `primary`=(2 + 1)", - "/update _vt.vreplication set pos=", - "commit", - }, - table: "commit", - data: [][]string{}, - }} +// func TestPlayerFilters(t *testing.T) { +// defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) + +// execStatements(t, []string{ +// "create table src1(id int, val varbinary(128), primary key(id))", +// fmt.Sprintf("create table %s.dst1(id int, val varbinary(128), primary key(id))", vrepldb), +// "create table src2(id int, val1 int, val2 int, primary key(id))", +// fmt.Sprintf("create table %s.dst2(id int, val1 int, sval2 int, rcount int, primary key(id))", vrepldb), +// "create table src3(id int, val varbinary(128), primary key(id))", +// fmt.Sprintf("create table %s.dst3(id int, val varbinary(128), primary key(id))", vrepldb), +// "create table yes(id int, val varbinary(128), primary key(id))", +// fmt.Sprintf("create table %s.yes(id int, val varbinary(128), primary key(id))", vrepldb), +// "create table no(id int, val varbinary(128), primary key(id))", +// "create table nopk(id int, val varbinary(128))", +// fmt.Sprintf("create table %s.nopk(id int, val varbinary(128))", vrepldb), +// }) +// defer execStatements(t, []string{ +// "drop table src1", +// fmt.Sprintf("drop table %s.dst1", vrepldb), +// "drop table src2", +// fmt.Sprintf("drop table %s.dst2", vrepldb), +// "drop table src3", +// fmt.Sprintf("drop table %s.dst3", vrepldb), +// "drop table yes", +// fmt.Sprintf("drop table %s.yes", vrepldb), +// "drop table no", +// "drop table nopk", +// fmt.Sprintf("drop table %s.nopk", vrepldb), +// }) +// env.SchemaEngine.Reload(context.Background()) + +// filter := &binlogdatapb.Filter{ +// Rules: []*binlogdatapb.Rule{{ +// Match: "dst1", +// Filter: "select * from src1", +// }, { +// Match: "dst2", +// Filter: "select id, val1, sum(val2) as sval2, count(*) as rcount from src2 group by id", +// }, { +// Match: "dst3", +// Filter: "select id, val from src3 group by id, val", +// }, { +// Match: "/yes", +// }, { +// Match: "/nopk", +// }}, +// } +// cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") +// defer cancel() + +// testcases := []struct { +// input string +// output []string +// table string +// data [][]string +// }{{ +// // insert with insertNormal +// input: "insert into src1 values(1, 'aaa')", +// output: []string{ +// "begin", +// "insert into dst1(id,val) values (1,'aaa')", +// "/update _vt.vreplication set pos=", +// "commit", +// }, +// table: "dst1", +// data: [][]string{ +// {"1", "aaa"}, +// }, +// }, { +// // update with insertNormal +// input: "update src1 set val='bbb'", +// output: []string{ +// "begin", +// "update dst1 set val='bbb' where id=1", +// "/update _vt.vreplication set pos=", +// "commit", +// }, +// table: "dst1", +// data: [][]string{ +// {"1", "bbb"}, +// }, +// }, { +// // delete with insertNormal +// input: "delete from src1 where id=1", +// output: []string{ +// "begin", +// "delete from dst1 where id=1", +// "/update _vt.vreplication set pos=", +// "commit", +// }, +// table: "dst1", +// data: [][]string{}, +// }, { +// // insert with insertOnDup +// input: "insert into src2 values(1, 2, 3)", +// output: []string{ +// "begin", +// "insert into dst2(id,val1,sval2,rcount) values (1,2,ifnull(3, 0),1) on duplicate key update val1=values(val1), sval2=sval2+ifnull(values(sval2), 0), rcount=rcount+1", +// "/update _vt.vreplication set pos=", +// "commit", +// }, +// table: "dst2", +// data: [][]string{ +// {"1", "2", "3", "1"}, +// }, +// }, { +// // update with insertOnDup +// input: "update src2 set val1=5, val2=1 where id=1", +// output: []string{ +// "begin", +// "update dst2 set val1=5, sval2=sval2-ifnull(3, 0)+ifnull(1, 0), rcount=rcount where id=1", +// "/update _vt.vreplication set pos=", +// "commit", +// }, +// table: "dst2", +// data: [][]string{ +// {"1", "5", "1", "1"}, +// }, +// }, { +// // delete with insertOnDup +// input: "delete from src2 where id=1", +// output: []string{ +// "begin", +// "update dst2 set val1=null, sval2=sval2-ifnull(1, 0), rcount=rcount-1 where id=1", +// "/update _vt.vreplication set pos=", +// "commit", +// }, +// table: "dst2", +// data: [][]string{ +// {"1", "", "0", "0"}, +// }, +// }, { +// // insert with insertIgnore +// input: "insert into src3 values(1, 'aaa')", +// output: []string{ +// "begin", +// "insert ignore into dst3(id,val) values (1,'aaa')", +// "/update _vt.vreplication set pos=", +// "commit", +// }, +// table: "dst3", +// data: [][]string{ +// {"1", "aaa"}, +// }, +// }, { +// // update with insertIgnore +// input: "update src3 set val='bbb'", +// output: []string{ +// "begin", +// "insert ignore into dst3(id,val) values (1,'bbb')", +// "/update _vt.vreplication set pos=", +// "commit", +// }, +// table: "dst3", +// data: [][]string{ +// {"1", "aaa"}, +// }, +// }, { +// // delete with insertIgnore +// input: "delete from src3 where id=1", +// output: []string{ +// "begin", +// "/update _vt.vreplication set pos=", +// "commit", +// }, +// table: "dst3", +// data: [][]string{ +// {"1", "aaa"}, +// }, +// }, { +// // insert: regular expression filter +// input: "insert into yes values(1, 'aaa')", +// output: []string{ +// "begin", +// "insert into yes(id,val) values (1,'aaa')", +// "/update _vt.vreplication set pos=", +// "commit", +// }, +// table: "yes", +// data: [][]string{ +// {"1", "aaa"}, +// }, +// }, { +// // update: regular expression filter +// input: "update yes set val='bbb'", +// output: []string{ +// "begin", +// "update yes set val='bbb' where id=1", +// "/update _vt.vreplication set pos=", +// "commit", +// }, +// table: "yes", +// data: [][]string{ +// {"1", "bbb"}, +// }, +// }, { +// // table should not match a rule +// input: "insert into no values(1, 'aaa')", +// output: []string{}, +// }, { +// // nopk: insert +// input: "insert into nopk values(1, 'aaa')", +// output: []string{ +// "begin", +// "insert into nopk(id,val) values (1,'aaa')", +// "/update _vt.vreplication set pos=", +// "commit", +// }, +// table: "nopk", +// data: [][]string{ +// {"1", "aaa"}, +// }, +// }, { +// // nopk: update +// input: "update nopk set val='bbb' where id=1", +// output: []string{ +// "begin", +// "delete from nopk where id=1 and val='aaa'", +// "insert into nopk(id,val) values (1,'bbb')", +// "/update _vt.vreplication set pos=", +// "commit", +// }, +// table: "nopk", +// data: [][]string{ +// {"1", "bbb"}, +// }, +// }, { +// // nopk: delete +// input: "delete from nopk where id=1", +// output: []string{ +// "begin", +// "delete from nopk where id=1 and val='bbb'", +// "/update _vt.vreplication set pos=", +// "commit", +// }, +// table: "nopk", +// data: [][]string{}, +// }} + +// for _, tcases := range testcases { +// execStatements(t, []string{tcases.input}) +// expectDBClientQueries(t, tcases.output) +// if tcases.table != "" { +// expectData(t, tcases.table, tcases.data) +// } +// } +// } + +// func TestPlayerKeywordNames(t *testing.T) { +// defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) + +// execStatements(t, []string{ +// "create table `begin`(`primary` int, `column` varbinary(128), primary key(`primary`))", +// fmt.Sprintf("create table %s.`begin`(`primary` int, `column` varbinary(128), primary key(`primary`))", vrepldb), +// "create table `rollback`(`primary` int, `column` varbinary(128), primary key(`primary`))", +// fmt.Sprintf("create table %s.`rollback`(`primary` int, `column` varbinary(128), primary key(`primary`))", vrepldb), +// "create table `commit`(`primary` int, `column` varbinary(128), primary key(`primary`))", +// fmt.Sprintf("create table %s.`commit`(`primary` int, `column` varbinary(128), primary key(`primary`))", vrepldb), +// }) +// defer execStatements(t, []string{ +// "drop table `begin`", +// fmt.Sprintf("drop table %s.`begin`", vrepldb), +// "drop table `rollback`", +// fmt.Sprintf("drop table %s.`rollback`", vrepldb), +// "drop table `commit`", +// fmt.Sprintf("drop table %s.`commit`", vrepldb), +// }) +// env.SchemaEngine.Reload(context.Background()) + +// filter := &binlogdatapb.Filter{ +// Rules: []*binlogdatapb.Rule{{ +// Match: "begin", +// Filter: "select * from `begin`", +// }, { +// Match: "rollback", +// Filter: "select `primary`, `column` from `rollback`", +// }, { +// Match: "commit", +// Filter: "select `primary`+1 as `primary`, concat(`column`, 'a') as `column` from `commit`", +// }}, +// } +// cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") +// defer cancel() + +// testcases := []struct { +// input string +// output []string +// table string +// data [][]string +// }{{ +// input: "insert into `begin` values(1, 'aaa')", +// output: []string{ +// "begin", +// "insert into `begin`(`primary`,`column`) values (1,'aaa')", +// "/update _vt.vreplication set pos=", +// "commit", +// }, +// table: "begin", +// data: [][]string{ +// {"1", "aaa"}, +// }, +// }, { +// input: "update `begin` set `column`='bbb'", +// output: []string{ +// "begin", +// "update `begin` set `column`='bbb' where `primary`=1", +// "/update _vt.vreplication set pos=", +// "commit", +// }, +// table: "begin", +// data: [][]string{ +// {"1", "bbb"}, +// }, +// }, { +// input: "delete from `begin` where `primary`=1", +// output: []string{ +// "begin", +// "delete from `begin` where `primary`=1", +// "/update _vt.vreplication set pos=", +// "commit", +// }, +// table: "begin", +// data: [][]string{}, +// }, { +// input: "insert into `rollback` values(1, 'aaa')", +// output: []string{ +// "begin", +// "insert into `rollback`(`primary`,`column`) values (1,'aaa')", +// "/update _vt.vreplication set pos=", +// "commit", +// }, +// table: "rollback", +// data: [][]string{ +// {"1", "aaa"}, +// }, +// }, { +// input: "update `rollback` set `column`='bbb'", +// output: []string{ +// "begin", +// "update `rollback` set `column`='bbb' where `primary`=1", +// "/update _vt.vreplication set pos=", +// "commit", +// }, +// table: "rollback", +// data: [][]string{ +// {"1", "bbb"}, +// }, +// }, { +// input: "delete from `rollback` where `primary`=1", +// output: []string{ +// "begin", +// "delete from `rollback` where `primary`=1", +// "/update _vt.vreplication set pos=", +// "commit", +// }, +// table: "rollback", +// data: [][]string{}, +// }, { +// input: "insert into `commit` values(1, 'aaa')", +// output: []string{ +// "begin", +// "insert into `commit`(`primary`,`column`) values (1 + 1,concat('aaa', 'a'))", +// "/update _vt.vreplication set pos=", +// "commit", +// }, +// table: "commit", +// data: [][]string{ +// {"2", "aaaa"}, +// }, +// }, { +// input: "update `commit` set `column`='bbb' where `primary`=1", +// output: []string{ +// "begin", +// "update `commit` set `column`=concat('bbb', 'a') where `primary`=(1 + 1)", +// "/update _vt.vreplication set pos=", +// "commit", +// }, +// table: "commit", +// data: [][]string{ +// {"2", "bbba"}, +// }, +// }, { +// input: "update `commit` set `primary`=2 where `primary`=1", +// output: []string{ +// "begin", +// "delete from `commit` where `primary`=(1 + 1)", +// "insert into `commit`(`primary`,`column`) values (2 + 1,concat('bbb', 'a'))", +// "/update _vt.vreplication set pos=", +// "commit", +// }, +// table: "commit", +// data: [][]string{ +// {"3", "bbba"}, +// }, +// }, { +// input: "delete from `commit` where `primary`=2", +// output: []string{ +// "begin", +// "delete from `commit` where `primary`=(2 + 1)", +// "/update _vt.vreplication set pos=", +// "commit", +// }, +// table: "commit", +// data: [][]string{}, +// }} + +// for _, tcases := range testcases { +// execStatements(t, []string{tcases.input}) +// expectDBClientQueries(t, tcases.output) +// if tcases.table != "" { +// expectData(t, tcases.table, tcases.data) +// } +// } +// } +// func TestUnicode(t *testing.T) { +// defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) + +// execStatements(t, []string{ +// "create table src1(id int, val varchar(128) COLLATE utf8_unicode_ci, primary key(id))", +// fmt.Sprintf("create table %s.dst1(id int, val varchar(128) COLLATE utf8_unicode_ci, primary key(id)) DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci", vrepldb), +// }) +// defer execStatements(t, []string{ +// "drop table src1", +// fmt.Sprintf("drop table %s.dst1", vrepldb), +// }) +// env.SchemaEngine.Reload(context.Background()) + +// filter := &binlogdatapb.Filter{ +// Rules: []*binlogdatapb.Rule{{ +// Match: "dst1", +// Filter: "select * from src1", +// }}, +// } +// cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") +// defer cancel() + +// testcases := []struct { +// input string +// output []string +// table string +// data [][]string +// }{{ +// // insert with insertNormal +// input: "insert into src1 values(1, '👍')", +// output: []string{ +// "begin", +// // We should expect the "Mojibaked" version. +// "insert into dst1(id,val) values (1,'ðŸ‘\u008d')", +// "/update _vt.vreplication set pos=", +// "commit", +// }, +// table: "dst1", +// data: [][]string{ +// {"1", "👍"}, +// }, +// }} + +// // We need a latin1 connection. +// conn, err := env.Mysqld.GetDbaConnection() +// if err != nil { +// t.Fatal(err) +// } +// defer conn.Close() + +// if _, err := conn.ExecuteFetch("set names latin1", 10000, false); err != nil { +// t.Fatal(err) +// } + +// for _, tcases := range testcases { +// if _, err := conn.ExecuteFetch(tcases.input, 10000, false); err != nil { +// t.Error(err) +// } +// expectDBClientQueries(t, tcases.output) +// if tcases.table != "" { +// customExpectData(t, tcases.table, tcases.data, func(ctx context.Context, query string) (*sqltypes.Result, error) { +// return conn.ExecuteFetch(query, 10000, true) +// }) +// } +// } +// } + +// func TestPlayerUpdates(t *testing.T) { +// defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) + +// execStatements(t, []string{ +// "create table t1(id int, grouped int, ungrouped int, summed int, primary key(id))", +// fmt.Sprintf("create table %s.t1(id int, grouped int, ungrouped int, summed int, rcount int, primary key(id))", vrepldb), +// }) +// defer execStatements(t, []string{ +// "drop table t1", +// fmt.Sprintf("drop table %s.t1", vrepldb), +// }) +// env.SchemaEngine.Reload(context.Background()) + +// filter := &binlogdatapb.Filter{ +// Rules: []*binlogdatapb.Rule{{ +// Match: "t1", +// Filter: "select id, grouped, ungrouped, sum(summed) as summed, count(*) as rcount from t1 group by id, grouped", +// }}, +// } +// cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") +// defer cancel() + +// testcases := []struct { +// input string +// output string +// table string +// data [][]string +// }{{ +// // Start with all nulls +// input: "insert into t1 values(1, null, null, null)", +// output: "insert into t1(id,grouped,ungrouped,summed,rcount) values (1,null,null,ifnull(null, 0),1) on duplicate key update ungrouped=values(ungrouped), summed=summed+ifnull(values(summed), 0), rcount=rcount+1", +// table: "t1", +// data: [][]string{ +// {"1", "", "", "0", "1"}, +// }, +// }, { +// // null to null values +// input: "update t1 set grouped=1 where id=1", +// output: "update t1 set ungrouped=null, summed=summed-ifnull(null, 0)+ifnull(null, 0), rcount=rcount where id=1", +// table: "t1", +// data: [][]string{ +// {"1", "", "", "0", "1"}, +// }, +// }, { +// // null to non-null values +// input: "update t1 set ungrouped=1, summed=1 where id=1", +// output: "update t1 set ungrouped=1, summed=summed-ifnull(null, 0)+ifnull(1, 0), rcount=rcount where id=1", +// table: "t1", +// data: [][]string{ +// {"1", "", "1", "1", "1"}, +// }, +// }, { +// // non-null to non-null values +// input: "update t1 set ungrouped=2, summed=2 where id=1", +// output: "update t1 set ungrouped=2, summed=summed-ifnull(1, 0)+ifnull(2, 0), rcount=rcount where id=1", +// table: "t1", +// data: [][]string{ +// {"1", "", "2", "2", "1"}, +// }, +// }, { +// // non-null to null values +// input: "update t1 set ungrouped=null, summed=null where id=1", +// output: "update t1 set ungrouped=null, summed=summed-ifnull(2, 0)+ifnull(null, 0), rcount=rcount where id=1", +// table: "t1", +// data: [][]string{ +// {"1", "", "", "0", "1"}, +// }, +// }, { +// // insert non-null values +// input: "insert into t1 values(2, 2, 3, 4)", +// output: "insert into t1(id,grouped,ungrouped,summed,rcount) values (2,2,3,ifnull(4, 0),1) on duplicate key update ungrouped=values(ungrouped), summed=summed+ifnull(values(summed), 0), rcount=rcount+1", +// table: "t1", +// data: [][]string{ +// {"1", "", "", "0", "1"}, +// {"2", "2", "3", "4", "1"}, +// }, +// }, { +// // delete non-null values +// input: "delete from t1 where id=2", +// output: "update t1 set ungrouped=null, summed=summed-ifnull(4, 0), rcount=rcount-1 where id=2", +// table: "t1", +// data: [][]string{ +// {"1", "", "", "0", "1"}, +// {"2", "2", "", "0", "0"}, +// }, +// }} + +// for _, tcases := range testcases { +// execStatements(t, []string{tcases.input}) +// output := []string{ +// "begin", +// tcases.output, +// "/update _vt.vreplication set pos=", +// "commit", +// } +// if tcases.output == "" { +// output = []string{ +// "begin", +// "/update _vt.vreplication set pos=", +// "commit", +// } +// } +// expectDBClientQueries(t, output) +// if tcases.table != "" { +// expectData(t, tcases.table, tcases.data) +// } +// } +// } + +// func TestPlayerRowMove(t *testing.T) { +// defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) + +// execStatements(t, []string{ +// "create table src(id int, val1 int, val2 int, primary key(id))", +// fmt.Sprintf("create table %s.dst(val1 int, sval2 int, rcount int, primary key(val1))", vrepldb), +// }) +// defer execStatements(t, []string{ +// "drop table src", +// fmt.Sprintf("drop table %s.dst", vrepldb), +// }) +// env.SchemaEngine.Reload(context.Background()) + +// filter := &binlogdatapb.Filter{ +// Rules: []*binlogdatapb.Rule{{ +// Match: "dst", +// Filter: "select val1, sum(val2) as sval2, count(*) as rcount from src group by val1", +// }}, +// } +// cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") +// defer cancel() + +// execStatements(t, []string{ +// "insert into src values(1, 1, 1), (2, 2, 2), (3, 2, 3)", +// }) +// expectDBClientQueries(t, []string{ +// "begin", +// "insert into dst(val1,sval2,rcount) values (1,ifnull(1, 0),1) on duplicate key update sval2=sval2+ifnull(values(sval2), 0), rcount=rcount+1", +// "insert into dst(val1,sval2,rcount) values (2,ifnull(2, 0),1) on duplicate key update sval2=sval2+ifnull(values(sval2), 0), rcount=rcount+1", +// "insert into dst(val1,sval2,rcount) values (2,ifnull(3, 0),1) on duplicate key update sval2=sval2+ifnull(values(sval2), 0), rcount=rcount+1", +// "/update _vt.vreplication set pos=", +// "commit", +// }) +// expectData(t, "dst", [][]string{ +// {"1", "1", "1"}, +// {"2", "5", "2"}, +// }) + +// execStatements(t, []string{ +// "update src set val1=1, val2=4 where id=3", +// }) +// expectDBClientQueries(t, []string{ +// "begin", +// "update dst set sval2=sval2-ifnull(3, 0), rcount=rcount-1 where val1=2", +// "insert into dst(val1,sval2,rcount) values (1,ifnull(4, 0),1) on duplicate key update sval2=sval2+ifnull(values(sval2), 0), rcount=rcount+1", +// "/update _vt.vreplication set pos=", +// "commit", +// }) +// expectData(t, "dst", [][]string{ +// {"1", "5", "2"}, +// {"2", "2", "1"}, +// }) +// } + +// func TestPlayerTypes(t *testing.T) { +// defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) + +// execStatements(t, []string{ +// "create table vitess_ints(tiny tinyint, tinyu tinyint unsigned, small smallint, smallu smallint unsigned, medium mediumint, mediumu mediumint unsigned, normal int, normalu int unsigned, big bigint, bigu bigint unsigned, y year, primary key(tiny))", +// fmt.Sprintf("create table %s.vitess_ints(tiny tinyint, tinyu tinyint unsigned, small smallint, smallu smallint unsigned, medium mediumint, mediumu mediumint unsigned, normal int, normalu int unsigned, big bigint, bigu bigint unsigned, y year, primary key(tiny))", vrepldb), +// "create table vitess_fracts(id int, deci decimal(5,2), num numeric(5,2), f float, d double, primary key(id))", +// fmt.Sprintf("create table %s.vitess_fracts(id int, deci decimal(5,2), num numeric(5,2), f float, d double, primary key(id))", vrepldb), +// "create table vitess_strings(vb varbinary(16), c char(16), vc varchar(16), b binary(4), tb tinyblob, bl blob, ttx tinytext, tx text, en enum('a','b'), s set('a','b'), primary key(vb))", +// fmt.Sprintf("create table %s.vitess_strings(vb varbinary(16), c char(16), vc varchar(16), b binary(4), tb tinyblob, bl blob, ttx tinytext, tx text, en enum('a','b'), s set('a','b'), primary key(vb))", vrepldb), +// "create table vitess_misc(id int, b bit(8), d date, dt datetime, t time, g geometry, primary key(id))", +// fmt.Sprintf("create table %s.vitess_misc(id int, b bit(8), d date, dt datetime, t time, g geometry, primary key(id))", vrepldb), +// "create table vitess_null(id int, val varbinary(128), primary key(id))", +// fmt.Sprintf("create table %s.vitess_null(id int, val varbinary(128), primary key(id))", vrepldb), +// "create table src1(id int, val varbinary(128), primary key(id))", +// fmt.Sprintf("create table %s.src1(id int, val varbinary(128), primary key(id))", vrepldb), +// "create table binary_pk(b binary(4), val varbinary(4), primary key(b))", +// fmt.Sprintf("create table %s.binary_pk(b binary(4), val varbinary(4), primary key(b))", vrepldb), +// }) +// defer execStatements(t, []string{ +// "drop table vitess_ints", +// fmt.Sprintf("drop table %s.vitess_ints", vrepldb), +// "drop table vitess_fracts", +// fmt.Sprintf("drop table %s.vitess_fracts", vrepldb), +// "drop table vitess_strings", +// fmt.Sprintf("drop table %s.vitess_strings", vrepldb), +// "drop table vitess_misc", +// fmt.Sprintf("drop table %s.vitess_misc", vrepldb), +// "drop table vitess_null", +// fmt.Sprintf("drop table %s.vitess_null", vrepldb), +// "drop table binary_pk", +// fmt.Sprintf("drop table %s.binary_pk", vrepldb), +// }) +// env.SchemaEngine.Reload(context.Background()) + +// filter := &binlogdatapb.Filter{ +// Rules: []*binlogdatapb.Rule{{ +// Match: "/.*", +// }}, +// } +// cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") +// defer cancel() +// testcases := []struct { +// input string +// output string +// table string +// data [][]string +// }{{ +// input: "insert into vitess_ints values(-128, 255, -32768, 65535, -8388608, 16777215, -2147483648, 4294967295, -9223372036854775808, 18446744073709551615, 2012)", +// output: "insert into vitess_ints(tiny,tinyu,small,smallu,medium,mediumu,normal,normalu,big,bigu,y) values (-128,255,-32768,65535,-8388608,16777215,-2147483648,4294967295,-9223372036854775808,18446744073709551615,2012)", +// table: "vitess_ints", +// data: [][]string{ +// {"-128", "255", "-32768", "65535", "-8388608", "16777215", "-2147483648", "4294967295", "-9223372036854775808", "18446744073709551615", "2012"}, +// }, +// }, { +// input: "insert into vitess_fracts values(1, 1.99, 2.99, 3.99, 4.99)", +// output: "insert into vitess_fracts(id,deci,num,f,d) values (1,1.99,2.99,3.99E+00,4.99E+00)", +// table: "vitess_fracts", +// data: [][]string{ +// {"1", "1.99", "2.99", "3.99", "4.99"}, +// }, +// }, { +// input: "insert into vitess_strings values('a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'a', 'a,b')", +// output: "insert into vitess_strings(vb,c,vc,b,tb,bl,ttx,tx,en,s) values ('a','b','c','d\\0\\0\\0','e','f','g','h','1','3')", +// table: "vitess_strings", +// data: [][]string{ +// {"a", "b", "c", "d\x00\x00\x00", "e", "f", "g", "h", "a", "a,b"}, +// }, +// }, { +// input: "insert into vitess_misc values(1, '\x01', '2012-01-01', '2012-01-01 15:45:45', '15:45:45', point(1, 2))", +// output: "insert into vitess_misc(id,b,d,dt,t,g) values (1,b'00000001','2012-01-01','2012-01-01 15:45:45','15:45:45','\\0\\0\\0\\0\x01\x01\\0\\0\\0\\0\\0\\0\\0\\0\\0\xf0?\\0\\0\\0\\0\\0\\0\\0@')", +// table: "vitess_misc", +// data: [][]string{ +// {"1", "\x01", "2012-01-01", "2012-01-01 15:45:45", "15:45:45", "\x00\x00\x00\x00\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf0?\x00\x00\x00\x00\x00\x00\x00@"}, +// }, +// }, { +// input: "insert into vitess_null values(1, null)", +// output: "insert into vitess_null(id,val) values (1,null)", +// table: "vitess_null", +// data: [][]string{ +// {"1", ""}, +// }, +// }, { +// input: "insert into binary_pk values('a', 'aaa')", +// output: "insert into binary_pk(b,val) values ('a\\0\\0\\0','aaa')", +// table: "binary_pk", +// data: [][]string{ +// {"a\x00\x00\x00", "aaa"}, +// }, +// }, { +// // Binary pk is a special case: https://github.com/vitessio/vitess/issues/3984 +// input: "update binary_pk set val='bbb' where b='a\\0\\0\\0'", +// output: "update binary_pk set val='bbb' where b='a\\0\\0\\0'", +// table: "binary_pk", +// data: [][]string{ +// {"a\x00\x00\x00", "bbb"}, +// }, +// }} + +// for _, tcases := range testcases { +// execStatements(t, []string{tcases.input}) +// want := []string{ +// "begin", +// tcases.output, +// "/update _vt.vreplication set pos=", +// "commit", +// } +// expectDBClientQueries(t, want) +// if tcases.table != "" { +// expectData(t, tcases.table, tcases.data) +// } +// } +// } + +// func TestPlayerDDL(t *testing.T) { +// defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) +// execStatements(t, []string{ +// "create table t1(id int, primary key(id))", +// fmt.Sprintf("create table %s.t1(id int, primary key(id))", vrepldb), +// }) +// defer execStatements(t, []string{ +// "drop table t1", +// fmt.Sprintf("drop table %s.t1", vrepldb), +// }) +// env.SchemaEngine.Reload(context.Background()) + +// filter := &binlogdatapb.Filter{ +// Rules: []*binlogdatapb.Rule{{ +// Match: "/.*", +// }}, +// } + +// cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") +// // Issue a dummy change to ensure vreplication is initialized. Otherwise there +// // is a race between the DDLs and the schema loader of vstreamer. +// // Root cause seems to be with MySQL where t1 shows up in information_schema before +// // the actual table is created. +// execStatements(t, []string{"insert into t1 values(1)"}) +// expectDBClientQueries(t, []string{ +// "begin", +// "insert into t1(id) values (1)", +// "/update _vt.vreplication set pos=", +// "commit", +// }) + +// execStatements(t, []string{"alter table t1 add column val varchar(128)"}) +// execStatements(t, []string{"alter table t1 drop column val"}) +// expectDBClientQueries(t, []string{ +// "/update _vt.vreplication set pos=", +// "/update _vt.vreplication set pos=", +// }) +// cancel() + +// cancel, id := startVReplication(t, filter, binlogdatapb.OnDDLAction_STOP, "") +// execStatements(t, []string{"alter table t1 add column val varchar(128)"}) +// pos1 := masterPosition(t) +// execStatements(t, []string{"alter table t1 drop column val"}) +// pos2 := masterPosition(t) +// // The stop position must be the GTID of the first DDL +// expectDBClientQueries(t, []string{ +// "begin", +// fmt.Sprintf("/update _vt.vreplication set pos='%s'", pos1), +// "/update _vt.vreplication set state='Stopped'", +// "commit", +// }) +// // Restart vreplication +// if _, err := playerEngine.Exec(fmt.Sprintf(`update _vt.vreplication set state = 'Running', message='' where id=%d`, id)); err != nil { +// t.Fatal(err) +// } +// // It should stop at the next DDL +// expectDBClientQueries(t, []string{ +// "/update.*'Running'", +// // Second update is from vreplicator. +// "/update.*'Running'", +// "begin", +// fmt.Sprintf("/update.*'%s'", pos2), +// "/update _vt.vreplication set state='Stopped'", +// "commit", +// }) +// cancel() + +// execStatements(t, []string{fmt.Sprintf("alter table %s.t1 add column val2 varchar(128)", vrepldb)}) +// cancel, _ = startVReplication(t, filter, binlogdatapb.OnDDLAction_EXEC, "") +// execStatements(t, []string{"alter table t1 add column val1 varchar(128)"}) +// expectDBClientQueries(t, []string{ +// "alter table t1 add column val1 varchar(128)", +// "/update _vt.vreplication set pos=", +// }) +// execStatements(t, []string{"alter table t1 add column val2 varchar(128)"}) +// expectDBClientQueries(t, []string{ +// "alter table t1 add column val2 varchar(128)", +// "/update _vt.vreplication set message='Duplicate", +// }) +// cancel() + +// execStatements(t, []string{ +// "alter table t1 drop column val1", +// "alter table t1 drop column val2", +// fmt.Sprintf("alter table %s.t1 drop column val1", vrepldb), +// }) + +// execStatements(t, []string{fmt.Sprintf("create table %s.t2(id int, primary key(id))", vrepldb)}) +// cancel, _ = startVReplication(t, filter, binlogdatapb.OnDDLAction_EXEC_IGNORE, "") +// execStatements(t, []string{"alter table t1 add column val1 varchar(128)"}) +// expectDBClientQueries(t, []string{ +// "alter table t1 add column val1 varchar(128)", +// "/update _vt.vreplication set pos=", +// }) +// execStatements(t, []string{"alter table t1 add column val2 varchar(128)"}) +// expectDBClientQueries(t, []string{ +// "alter table t1 add column val2 varchar(128)", +// "/update _vt.vreplication set pos=", +// }) +// cancel() +// } + +// func TestPlayerStopPos(t *testing.T) { +// defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) + +// execStatements(t, []string{ +// "create table yes(id int, val varbinary(128), primary key(id))", +// fmt.Sprintf("create table %s.yes(id int, val varbinary(128), primary key(id))", vrepldb), +// "create table no(id int, val varbinary(128), primary key(id))", +// }) +// defer execStatements(t, []string{ +// "drop table yes", +// fmt.Sprintf("drop table %s.yes", vrepldb), +// "drop table no", +// }) +// env.SchemaEngine.Reload(context.Background()) + +// filter := &binlogdatapb.Filter{ +// Rules: []*binlogdatapb.Rule{{ +// Match: "/yes", +// }}, +// } +// bls := &binlogdatapb.BinlogSource{ +// Keyspace: env.KeyspaceName, +// Shard: env.ShardName, +// Filter: filter, +// OnDdl: binlogdatapb.OnDDLAction_IGNORE, +// } +// startPos := masterPosition(t) +// query := binlogplayer.CreateVReplicationState("test", bls, startPos, binlogplayer.BlpStopped, vrepldb) +// qr, err := playerEngine.Exec(query) +// if err != nil { +// t.Fatal(err) +// } +// id := uint32(qr.InsertID) +// for q := range globalDBQueries { +// if strings.HasPrefix(q, "insert into _vt.vreplication") { +// break +// } +// } + +// // Test normal stop. +// execStatements(t, []string{ +// "insert into yes values(1, 'aaa')", +// }) +// stopPos := masterPosition(t) +// query = binlogplayer.StartVReplicationUntil(id, stopPos) +// if _, err := playerEngine.Exec(query); err != nil { +// t.Fatal(err) +// } +// expectDBClientQueries(t, []string{ +// "/update.*'Running'", +// // Second update is from vreplicator. +// "/update.*'Running'", +// "begin", +// "insert into yes(id,val) values (1,'aaa')", +// fmt.Sprintf("/update.*'%s'", stopPos), +// "/update.*'Stopped'", +// "commit", +// }) + +// // Test stopping at empty transaction. +// execStatements(t, []string{ +// "insert into no values(2, 'aaa')", +// "insert into no values(3, 'aaa')", +// }) +// stopPos = masterPosition(t) +// execStatements(t, []string{ +// "insert into no values(4, 'aaa')", +// }) +// query = binlogplayer.StartVReplicationUntil(id, stopPos) +// if _, err := playerEngine.Exec(query); err != nil { +// t.Fatal(err) +// } +// expectDBClientQueries(t, []string{ +// "/update.*'Running'", +// // Second update is from vreplicator. +// "/update.*'Running'", +// "begin", +// // Since 'no' generates empty transactions that are skipped by +// // vplayer, a commit is done only for the stop position event. +// fmt.Sprintf("/update.*'%s'", stopPos), +// "/update.*'Stopped'", +// "commit", +// }) + +// // Test stopping when position is already reached. +// query = binlogplayer.StartVReplicationUntil(id, stopPos) +// if _, err := playerEngine.Exec(query); err != nil { +// t.Fatal(err) +// } +// expectDBClientQueries(t, []string{ +// "/update.*'Running'", +// // Second update is from vreplicator. +// "/update.*'Running'", +// "/update.*'Stopped'.*already reached", +// }) +// } + +// func TestPlayerIdleUpdate(t *testing.T) { +// defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) + +// savedIdleTimeout := idleTimeout +// defer func() { idleTimeout = savedIdleTimeout }() +// idleTimeout = 100 * time.Millisecond + +// execStatements(t, []string{ +// "create table t1(id int, val varbinary(128), primary key(id))", +// fmt.Sprintf("create table %s.t1(id int, val varbinary(128), primary key(id))", vrepldb), +// }) +// defer execStatements(t, []string{ +// "drop table t1", +// fmt.Sprintf("drop table %s.t1", vrepldb), +// }) +// env.SchemaEngine.Reload(context.Background()) + +// filter := &binlogdatapb.Filter{ +// Rules: []*binlogdatapb.Rule{{ +// Match: "/.*", +// }}, +// } +// cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") +// defer cancel() + +// execStatements(t, []string{ +// "insert into t1 values(1, 'aaa')", +// }) +// start := time.Now() +// expectDBClientQueries(t, []string{ +// "begin", +// "insert into t1(id,val) values (1,'aaa')", +// "/update _vt.vreplication set pos=", +// "commit", +// }) +// // The above write will generate a new binlog event, and +// // that event will loopback into player as an empty event. +// // But it must not get saved until idleTimeout has passed. +// // The exact positions are hard to verify because of this +// // loopback mechanism. +// expectDBClientQueries(t, []string{ +// "/update _vt.vreplication set pos=", +// }) +// if duration := time.Since(start); duration < idleTimeout { +// t.Errorf("duration: %v, must be at least %v", duration, idleTimeout) +// } +// } + +// func TestPlayerSplitTransaction(t *testing.T) { +// defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) +// flag.Set("vstream_packet_size", "10") +// defer flag.Set("vstream_packet_size", "10000") + +// execStatements(t, []string{ +// "create table t1(id int, val varbinary(128), primary key(id))", +// fmt.Sprintf("create table %s.t1(id int, val varbinary(128), primary key(id))", vrepldb), +// }) +// defer execStatements(t, []string{ +// "drop table t1", +// fmt.Sprintf("drop table %s.t1", vrepldb), +// }) +// env.SchemaEngine.Reload(context.Background()) + +// filter := &binlogdatapb.Filter{ +// Rules: []*binlogdatapb.Rule{{ +// Match: "/.*", +// }}, +// } +// cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") +// defer cancel() + +// execStatements(t, []string{ +// "begin", +// "insert into t1 values(1, '123456')", +// "insert into t1 values(2, '789012')", +// "commit", +// }) +// // Because the packet size is 10, this is received as two events, +// // but still combined as one transaction. +// expectDBClientQueries(t, []string{ +// "begin", +// "insert into t1(id,val) values (1,'123456')", +// "insert into t1(id,val) values (2,'789012')", +// "/update _vt.vreplication set pos=", +// "commit", +// }) +// } + +// func TestPlayerLockErrors(t *testing.T) { +// defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) + +// execStatements(t, []string{ +// "create table t1(id int, val varbinary(128), primary key(id))", +// fmt.Sprintf("create table %s.t1(id int, val varbinary(128), primary key(id))", vrepldb), +// }) +// defer execStatements(t, []string{ +// "drop table t1", +// fmt.Sprintf("drop table %s.t1", vrepldb), +// }) +// env.SchemaEngine.Reload(context.Background()) + +// filter := &binlogdatapb.Filter{ +// Rules: []*binlogdatapb.Rule{{ +// Match: "/.*", +// }}, +// } +// cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") +// defer cancel() + +// execStatements(t, []string{ +// "begin", +// "insert into t1 values(1, 'aaa')", +// "insert into t1 values(2, 'bbb')", +// "commit", +// }) +// expectDBClientQueries(t, []string{ +// "begin", +// "insert into t1(id,val) values (1,'aaa')", +// "insert into t1(id,val) values (2,'bbb')", +// "/update _vt.vreplication set pos=", +// "commit", +// }) + +// vconn := &realDBClient{nolog: true} +// if err := vconn.Connect(); err != nil { +// t.Error(err) +// } +// defer vconn.Close() + +// // Start a transaction and lock the second row. +// if _, err := vconn.ExecuteFetch("begin", 1); err != nil { +// t.Error(err) +// } +// if _, err := vconn.ExecuteFetch("update t1 set val='bbb' where id=2", 1); err != nil { +// t.Error(err) +// } + +// execStatements(t, []string{ +// "begin", +// "update t1 set val='ccc' where id=1", +// "update t1 set val='ccc' where id=2", +// "commit", +// }) +// // The innodb lock wait timeout is set to 1s. +// expectDBClientQueries(t, []string{ +// "begin", +// "update t1 set val='ccc' where id=1", +// "update t1 set val='ccc' where id=2", +// "rollback", +// }) + +// // Release the lock, and watch the retry go through. +// _, _ = vconn.ExecuteFetch("rollback", 1) +// expectDBClientQueries(t, []string{ +// "begin", +// "update t1 set val='ccc' where id=1", +// "update t1 set val='ccc' where id=2", +// "/update _vt.vreplication set pos=", +// "commit", +// }) +// } + +// func TestPlayerCancelOnLock(t *testing.T) { +// defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) + +// execStatements(t, []string{ +// "create table t1(id int, val varbinary(128), primary key(id))", +// fmt.Sprintf("create table %s.t1(id int, val varbinary(128), primary key(id))", vrepldb), +// }) +// defer execStatements(t, []string{ +// "drop table t1", +// fmt.Sprintf("drop table %s.t1", vrepldb), +// }) +// env.SchemaEngine.Reload(context.Background()) + +// filter := &binlogdatapb.Filter{ +// Rules: []*binlogdatapb.Rule{{ +// Match: "/.*", +// }}, +// } +// cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") +// defer cancel() + +// execStatements(t, []string{ +// "begin", +// "insert into t1 values(1, 'aaa')", +// "commit", +// }) +// expectDBClientQueries(t, []string{ +// "begin", +// "insert into t1(id,val) values (1,'aaa')", +// "/update _vt.vreplication set pos=", +// "commit", +// }) + +// vconn := &realDBClient{nolog: true} +// if err := vconn.Connect(); err != nil { +// t.Error(err) +// } +// defer vconn.Close() + +// // Start a transaction and lock the row. +// if _, err := vconn.ExecuteFetch("begin", 1); err != nil { +// t.Error(err) +// } +// if _, err := vconn.ExecuteFetch("update t1 set val='bbb' where id=1", 1); err != nil { +// t.Error(err) +// } + +// execStatements(t, []string{ +// "begin", +// "update t1 set val='ccc' where id=1", +// "commit", +// }) +// // The innodb lock wait timeout is set to 1s. +// expectDBClientQueries(t, []string{ +// "begin", +// "update t1 set val='ccc' where id=1", +// "rollback", +// }) + +// // VReplication should not get stuck if you cancel now. +// done := make(chan bool) +// go func() { +// cancel() +// close(done) +// }() +// select { +// case <-done: +// case <-time.After(5 * time.Second): +// t.Error("cancel is hung") +// } +// } + +// func TestPlayerBatching(t *testing.T) { +// defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) + +// execStatements(t, []string{ +// "create table t1(id int, val varbinary(128), primary key(id))", +// fmt.Sprintf("create table %s.t1(id int, val varbinary(128), primary key(id))", vrepldb), +// }) +// defer execStatements(t, []string{ +// "drop table t1", +// fmt.Sprintf("drop table %s.t1", vrepldb), +// }) +// env.SchemaEngine.Reload(context.Background()) + +// filter := &binlogdatapb.Filter{ +// Rules: []*binlogdatapb.Rule{{ +// Match: "/.*", +// }}, +// } +// cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_EXEC, "") +// defer cancel() + +// execStatements(t, []string{ +// "insert into t1 values(1, 'aaa')", +// }) +// expectDBClientQueries(t, []string{ +// "begin", +// "insert into t1(id,val) values (1,'aaa')", +// "/update _vt.vreplication set pos=", +// "commit", +// }) + +// vconn := &realDBClient{nolog: true} +// if err := vconn.Connect(); err != nil { +// t.Error(err) +// } +// defer vconn.Close() + +// // Start a transaction and lock the row. +// if _, err := vconn.ExecuteFetch("begin", 1); err != nil { +// t.Error(err) +// } +// if _, err := vconn.ExecuteFetch("update t1 set val='bbb' where id=1", 1); err != nil { +// t.Error(err) +// } + +// // create one transaction +// execStatements(t, []string{ +// "update t1 set val='ccc' where id=1", +// }) +// // Wait for the begin. The update will be blocked. +// expectDBClientQueries(t, []string{ +// "begin", +// }) + +// // Create two more transactions. They will go and wait in the relayLog. +// execStatements(t, []string{ +// "insert into t1 values(2, 'aaa')", +// "insert into t1 values(3, 'aaa')", +// "alter table t1 add column val2 varbinary(128)", +// "alter table t1 drop column val2", +// }) + +// // Release the lock. +// _, _ = vconn.ExecuteFetch("rollback", 1) +// // First transaction will complete. The other two +// // transactions must be batched into one. But the +// // DDLs should be on their own. +// expectDBClientQueries(t, []string{ +// "update t1 set val='ccc' where id=1", +// "/update _vt.vreplication set pos=", +// "commit", +// "begin", +// "insert into t1(id,val) values (2,'aaa')", +// "insert into t1(id,val) values (3,'aaa')", +// "/update _vt.vreplication set pos=", +// "commit", +// "alter table t1 add column val2 varbinary(128)", +// "/update _vt.vreplication set pos=", +// "alter table t1 drop column val2", +// "/update _vt.vreplication set pos=", +// }) +// } + +// func TestPlayerRelayLogMaxSize(t *testing.T) { +// defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) + +// for i := 0; i < 2; i++ { +// // First iteration checks max size, second checks max items +// func() { +// switch i { +// case 0: +// savedSize := relayLogMaxSize +// defer func() { relayLogMaxSize = savedSize }() +// relayLogMaxSize = 10 +// case 1: +// savedLen := relayLogMaxItems +// defer func() { relayLogMaxItems = savedLen }() +// relayLogMaxItems = 2 +// } + +// execStatements(t, []string{ +// "create table t1(id int, val varbinary(128), primary key(id))", +// fmt.Sprintf("create table %s.t1(id int, val varbinary(128), primary key(id))", vrepldb), +// }) +// defer execStatements(t, []string{ +// "drop table t1", +// fmt.Sprintf("drop table %s.t1", vrepldb), +// }) +// env.SchemaEngine.Reload(context.Background()) + +// filter := &binlogdatapb.Filter{ +// Rules: []*binlogdatapb.Rule{{ +// Match: "/.*", +// }}, +// } +// cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") +// defer cancel() + +// execStatements(t, []string{ +// "insert into t1 values(1, '123456')", +// }) +// expectDBClientQueries(t, []string{ +// "begin", +// "insert into t1(id,val) values (1,'123456')", +// "/update _vt.vreplication set pos=", +// "commit", +// }) + +// vconn := &realDBClient{nolog: true} +// if err := vconn.Connect(); err != nil { +// t.Error(err) +// } +// defer vconn.Close() + +// // Start a transaction and lock the row. +// if _, err := vconn.ExecuteFetch("begin", 1); err != nil { +// t.Error(err) +// } +// if _, err := vconn.ExecuteFetch("update t1 set val='bbb' where id=1", 1); err != nil { +// t.Error(err) +// } + +// // create one transaction +// execStatements(t, []string{ +// "update t1 set val='ccc' where id=1", +// }) +// // Wait for the begin. The update will be blocked. +// expectDBClientQueries(t, []string{ +// "begin", +// }) + +// // Create two more transactions. They will go and wait in the relayLog. +// execStatements(t, []string{ +// "insert into t1 values(2, '789012')", +// "insert into t1 values(3, '345678')", +// "insert into t1 values(4, '901234')", +// }) + +// // Release the lock. +// _, _ = vconn.ExecuteFetch("rollback", 1) +// // First transaction will complete. The other two +// // transactions must be batched into one. The last transaction +// // will wait to be sent to the relay until the player fetches +// // them. +// expectDBClientQueries(t, []string{ +// "update t1 set val='ccc' where id=1", +// "/update _vt.vreplication set pos=", +// "commit", +// "begin", +// "insert into t1(id,val) values (2,'789012')", +// "insert into t1(id,val) values (3,'345678')", +// "/update _vt.vreplication set pos=", +// "commit", +// "begin", +// "insert into t1(id,val) values (4,'901234')", +// "/update _vt.vreplication set pos=", +// "commit", +// }) +// }() +// } +// } + +// func TestRestartOnVStreamEnd(t *testing.T) { +// defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) + +// savedDelay := *retryDelay +// defer func() { *retryDelay = savedDelay }() +// *retryDelay = 1 * time.Millisecond + +// execStatements(t, []string{ +// "create table t1(id int, val varbinary(128), primary key(id))", +// fmt.Sprintf("create table %s.t1(id int, val varbinary(128), primary key(id))", vrepldb), +// }) +// defer execStatements(t, []string{ +// "drop table t1", +// fmt.Sprintf("drop table %s.t1", vrepldb), +// }) +// env.SchemaEngine.Reload(context.Background()) + +// filter := &binlogdatapb.Filter{ +// Rules: []*binlogdatapb.Rule{{ +// Match: "/.*", +// }}, +// } +// cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") +// defer cancel() + +// execStatements(t, []string{ +// "insert into t1 values(1, 'aaa')", +// }) +// expectDBClientQueries(t, []string{ +// "begin", +// "insert into t1(id,val) values (1,'aaa')", +// "/update _vt.vreplication set pos=", +// "commit", +// }) + +// streamerEngine.Close() +// expectDBClientQueries(t, []string{ +// "/update _vt.vreplication set message='vstream ended'", +// }) +// if err := streamerEngine.Open(env.KeyspaceName, env.ShardName); err != nil { +// t.Fatal(err) +// } + +// execStatements(t, []string{ +// "insert into t1 values(2, 'aaa')", +// }) +// expectDBClientQueries(t, []string{ +// "/update _vt.vreplication set state='Running'", +// "begin", +// "insert into t1(id,val) values (2,'aaa')", +// "/update _vt.vreplication set pos=", +// "commit", +// }) +// } + +// func TestTimestamp(t *testing.T) { +// defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) + +// execStatements(t, []string{ +// "create table t1(id int, ts timestamp, dt datetime)", +// fmt.Sprintf("create table %s.t1(id int, ts timestamp, dt datetime)", vrepldb), +// }) +// defer execStatements(t, []string{ +// "drop table t1", +// fmt.Sprintf("drop table %s.t1", vrepldb), +// }) +// env.SchemaEngine.Reload(context.Background()) + +// filter := &binlogdatapb.Filter{ +// Rules: []*binlogdatapb.Rule{{ +// Match: "/.*", +// }}, +// } +// cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") +// defer cancel() + +// qr, err := env.Mysqld.FetchSuperQuery(context.Background(), "select now()") +// if err != nil { +// t.Fatal(err) +// } +// want := qr.Rows[0][0].ToString() +// t.Logf("want: %s", want) + +// execStatements(t, []string{ +// fmt.Sprintf("insert into t1 values(1, '%s', '%s')", want, want), +// }) +// expectDBClientQueries(t, []string{ +// "begin", +// // The insert value for ts will be in UTC. +// // We'll check the row instead. +// "/insert into t1", +// "/update _vt.vreplication set pos=", +// "commit", +// }) + +// expectData(t, "t1", [][]string{{"1", want, want}}) +// } - for _, tcases := range testcases { - execStatements(t, []string{tcases.input}) - expectDBClientQueries(t, tcases.output) - if tcases.table != "" { - expectData(t, tcases.table, tcases.data) - } +func execStatements(t *testing.T, queries []string) { + t.Helper() + if err := env.Mysqld.ExecuteSuperQueryList(context.Background(), queries); err != nil { + t.Error(err) } } -func TestUnicode(t *testing.T) { - defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) - execStatements(t, []string{ - "create table src1(id int, val varchar(128) COLLATE utf8_unicode_ci, primary key(id))", - fmt.Sprintf("create table %s.dst1(id int, val varchar(128) COLLATE utf8_unicode_ci, primary key(id)) DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci", vrepldb), - }) - defer execStatements(t, []string{ - "drop table src1", - fmt.Sprintf("drop table %s.dst1", vrepldb), - }) - env.SchemaEngine.Reload(context.Background()) +func startVReplicationV2(t *testing.T, filter *binlogdatapb.Filter, bls *binlogdatapb.BinlogSource, pos string) (cancelFunc func(), id int) { + t.Helper() - filter := &binlogdatapb.Filter{ - Rules: []*binlogdatapb.Rule{{ - Match: "dst1", - Filter: "select * from src1", - }}, + if pos == "" { + pos = masterPosition(t) } - cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") - defer cancel() - - testcases := []struct { - input string - output []string - table string - data [][]string - }{{ - // insert with insertNormal - input: "insert into src1 values(1, '👍')", - output: []string{ - "begin", - // We should expect the "Mojibaked" version. - "insert into dst1(id,val) values (1,'ðŸ‘\u008d')", - "/update _vt.vreplication set pos=", - "commit", - }, - table: "dst1", - data: [][]string{ - {"1", "👍"}, - }, - }} - - // We need a latin1 connection. - conn, err := env.Mysqld.GetDbaConnection() + query := binlogplayer.CreateVReplication("test", bls, pos, 9223372036854775807, 9223372036854775807, 0, vrepldb) + qr, err := playerEngine.Exec(query) if err != nil { t.Fatal(err) } - defer conn.Close() - - if _, err := conn.ExecuteFetch("set names latin1", 10000, false); err != nil { - t.Fatal(err) - } - - for _, tcases := range testcases { - if _, err := conn.ExecuteFetch(tcases.input, 10000, false); err != nil { - t.Error(err) - } - expectDBClientQueries(t, tcases.output) - if tcases.table != "" { - customExpectData(t, tcases.table, tcases.data, func(ctx context.Context, query string) (*sqltypes.Result, error) { - return conn.ExecuteFetch(query, 10000, true) - }) - } - } -} - -func TestPlayerUpdates(t *testing.T) { - defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) - - execStatements(t, []string{ - "create table t1(id int, grouped int, ungrouped int, summed int, primary key(id))", - fmt.Sprintf("create table %s.t1(id int, grouped int, ungrouped int, summed int, rcount int, primary key(id))", vrepldb), - }) - defer execStatements(t, []string{ - "drop table t1", - fmt.Sprintf("drop table %s.t1", vrepldb), - }) - env.SchemaEngine.Reload(context.Background()) - - filter := &binlogdatapb.Filter{ - Rules: []*binlogdatapb.Rule{{ - Match: "t1", - Filter: "select id, grouped, ungrouped, sum(summed) as summed, count(*) as rcount from t1 group by id, grouped", - }}, - } - cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") - defer cancel() - - testcases := []struct { - input string - output string - table string - data [][]string - }{{ - // Start with all nulls - input: "insert into t1 values(1, null, null, null)", - output: "insert into t1(id,grouped,ungrouped,summed,rcount) values (1,null,null,ifnull(null, 0),1) on duplicate key update ungrouped=values(ungrouped), summed=summed+ifnull(values(summed), 0), rcount=rcount+1", - table: "t1", - data: [][]string{ - {"1", "", "", "0", "1"}, - }, - }, { - // null to null values - input: "update t1 set grouped=1 where id=1", - output: "update t1 set ungrouped=null, summed=summed-ifnull(null, 0)+ifnull(null, 0), rcount=rcount where id=1", - table: "t1", - data: [][]string{ - {"1", "", "", "0", "1"}, - }, - }, { - // null to non-null values - input: "update t1 set ungrouped=1, summed=1 where id=1", - output: "update t1 set ungrouped=1, summed=summed-ifnull(null, 0)+ifnull(1, 0), rcount=rcount where id=1", - table: "t1", - data: [][]string{ - {"1", "", "1", "1", "1"}, - }, - }, { - // non-null to non-null values - input: "update t1 set ungrouped=2, summed=2 where id=1", - output: "update t1 set ungrouped=2, summed=summed-ifnull(1, 0)+ifnull(2, 0), rcount=rcount where id=1", - table: "t1", - data: [][]string{ - {"1", "", "2", "2", "1"}, - }, - }, { - // non-null to null values - input: "update t1 set ungrouped=null, summed=null where id=1", - output: "update t1 set ungrouped=null, summed=summed-ifnull(2, 0)+ifnull(null, 0), rcount=rcount where id=1", - table: "t1", - data: [][]string{ - {"1", "", "", "0", "1"}, - }, - }, { - // insert non-null values - input: "insert into t1 values(2, 2, 3, 4)", - output: "insert into t1(id,grouped,ungrouped,summed,rcount) values (2,2,3,ifnull(4, 0),1) on duplicate key update ungrouped=values(ungrouped), summed=summed+ifnull(values(summed), 0), rcount=rcount+1", - table: "t1", - data: [][]string{ - {"1", "", "", "0", "1"}, - {"2", "2", "3", "4", "1"}, - }, - }, { - // delete non-null values - input: "delete from t1 where id=2", - output: "update t1 set ungrouped=null, summed=summed-ifnull(4, 0), rcount=rcount-1 where id=2", - table: "t1", - data: [][]string{ - {"1", "", "", "0", "1"}, - {"2", "2", "", "0", "0"}, - }, - }} - - for _, tcases := range testcases { - execStatements(t, []string{tcases.input}) - output := []string{ - "begin", - tcases.output, - "/update _vt.vreplication set pos=", - "commit", - } - if tcases.output == "" { - output = []string{ - "begin", - "/update _vt.vreplication set pos=", - "commit", - } - } - expectDBClientQueries(t, output) - if tcases.table != "" { - expectData(t, tcases.table, tcases.data) - } - } -} - -func TestPlayerRowMove(t *testing.T) { - defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) - - execStatements(t, []string{ - "create table src(id int, val1 int, val2 int, primary key(id))", - fmt.Sprintf("create table %s.dst(val1 int, sval2 int, rcount int, primary key(val1))", vrepldb), - }) - defer execStatements(t, []string{ - "drop table src", - fmt.Sprintf("drop table %s.dst", vrepldb), - }) - env.SchemaEngine.Reload(context.Background()) - - filter := &binlogdatapb.Filter{ - Rules: []*binlogdatapb.Rule{{ - Match: "dst", - Filter: "select val1, sum(val2) as sval2, count(*) as rcount from src group by val1", - }}, - } - cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") - defer cancel() - - execStatements(t, []string{ - "insert into src values(1, 1, 1), (2, 2, 2), (3, 2, 3)", - }) - expectDBClientQueries(t, []string{ - "begin", - "insert into dst(val1,sval2,rcount) values (1,ifnull(1, 0),1) on duplicate key update sval2=sval2+ifnull(values(sval2), 0), rcount=rcount+1", - "insert into dst(val1,sval2,rcount) values (2,ifnull(2, 0),1) on duplicate key update sval2=sval2+ifnull(values(sval2), 0), rcount=rcount+1", - "insert into dst(val1,sval2,rcount) values (2,ifnull(3, 0),1) on duplicate key update sval2=sval2+ifnull(values(sval2), 0), rcount=rcount+1", - "/update _vt.vreplication set pos=", - "commit", - }) - expectData(t, "dst", [][]string{ - {"1", "1", "1"}, - {"2", "5", "2"}, - }) - - execStatements(t, []string{ - "update src set val1=1, val2=4 where id=3", - }) - expectDBClientQueries(t, []string{ - "begin", - "update dst set sval2=sval2-ifnull(3, 0), rcount=rcount-1 where val1=2", - "insert into dst(val1,sval2,rcount) values (1,ifnull(4, 0),1) on duplicate key update sval2=sval2+ifnull(values(sval2), 0), rcount=rcount+1", - "/update _vt.vreplication set pos=", - "commit", - }) - expectData(t, "dst", [][]string{ - {"1", "5", "2"}, - {"2", "2", "1"}, - }) -} - -func TestPlayerTypes(t *testing.T) { - defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) - - execStatements(t, []string{ - "create table vitess_ints(tiny tinyint, tinyu tinyint unsigned, small smallint, smallu smallint unsigned, medium mediumint, mediumu mediumint unsigned, normal int, normalu int unsigned, big bigint, bigu bigint unsigned, y year, primary key(tiny))", - fmt.Sprintf("create table %s.vitess_ints(tiny tinyint, tinyu tinyint unsigned, small smallint, smallu smallint unsigned, medium mediumint, mediumu mediumint unsigned, normal int, normalu int unsigned, big bigint, bigu bigint unsigned, y year, primary key(tiny))", vrepldb), - "create table vitess_fracts(id int, deci decimal(5,2), num numeric(5,2), f float, d double, primary key(id))", - fmt.Sprintf("create table %s.vitess_fracts(id int, deci decimal(5,2), num numeric(5,2), f float, d double, primary key(id))", vrepldb), - "create table vitess_strings(vb varbinary(16), c char(16), vc varchar(16), b binary(4), tb tinyblob, bl blob, ttx tinytext, tx text, en enum('a','b'), s set('a','b'), primary key(vb))", - fmt.Sprintf("create table %s.vitess_strings(vb varbinary(16), c char(16), vc varchar(16), b binary(4), tb tinyblob, bl blob, ttx tinytext, tx text, en enum('a','b'), s set('a','b'), primary key(vb))", vrepldb), - "create table vitess_misc(id int, b bit(8), d date, dt datetime, t time, g geometry, primary key(id))", - fmt.Sprintf("create table %s.vitess_misc(id int, b bit(8), d date, dt datetime, t time, g geometry, primary key(id))", vrepldb), - "create table vitess_null(id int, val varbinary(128), primary key(id))", - fmt.Sprintf("create table %s.vitess_null(id int, val varbinary(128), primary key(id))", vrepldb), - "create table src1(id int, val varbinary(128), primary key(id))", - fmt.Sprintf("create table %s.src1(id int, val varbinary(128), primary key(id))", vrepldb), - "create table binary_pk(b binary(4), val varbinary(4), primary key(b))", - fmt.Sprintf("create table %s.binary_pk(b binary(4), val varbinary(4), primary key(b))", vrepldb), - }) - defer execStatements(t, []string{ - "drop table vitess_ints", - fmt.Sprintf("drop table %s.vitess_ints", vrepldb), - "drop table vitess_fracts", - fmt.Sprintf("drop table %s.vitess_fracts", vrepldb), - "drop table vitess_strings", - fmt.Sprintf("drop table %s.vitess_strings", vrepldb), - "drop table vitess_misc", - fmt.Sprintf("drop table %s.vitess_misc", vrepldb), - "drop table vitess_null", - fmt.Sprintf("drop table %s.vitess_null", vrepldb), - "drop table binary_pk", - fmt.Sprintf("drop table %s.binary_pk", vrepldb), - }) - env.SchemaEngine.Reload(context.Background()) - - filter := &binlogdatapb.Filter{ - Rules: []*binlogdatapb.Rule{{ - Match: "/.*", - }}, - } - cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") - defer cancel() - testcases := []struct { - input string - output string - table string - data [][]string - }{{ - input: "insert into vitess_ints values(-128, 255, -32768, 65535, -8388608, 16777215, -2147483648, 4294967295, -9223372036854775808, 18446744073709551615, 2012)", - output: "insert into vitess_ints(tiny,tinyu,small,smallu,medium,mediumu,normal,normalu,big,bigu,y) values (-128,255,-32768,65535,-8388608,16777215,-2147483648,4294967295,-9223372036854775808,18446744073709551615,2012)", - table: "vitess_ints", - data: [][]string{ - {"-128", "255", "-32768", "65535", "-8388608", "16777215", "-2147483648", "4294967295", "-9223372036854775808", "18446744073709551615", "2012"}, - }, - }, { - input: "insert into vitess_fracts values(1, 1.99, 2.99, 3.99, 4.99)", - output: "insert into vitess_fracts(id,deci,num,f,d) values (1,1.99,2.99,3.99E+00,4.99E+00)", - table: "vitess_fracts", - data: [][]string{ - {"1", "1.99", "2.99", "3.99", "4.99"}, - }, - }, { - input: "insert into vitess_strings values('a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'a', 'a,b')", - output: "insert into vitess_strings(vb,c,vc,b,tb,bl,ttx,tx,en,s) values ('a','b','c','d\\0\\0\\0','e','f','g','h','1','3')", - table: "vitess_strings", - data: [][]string{ - {"a", "b", "c", "d\x00\x00\x00", "e", "f", "g", "h", "a", "a,b"}, - }, - }, { - input: "insert into vitess_misc values(1, '\x01', '2012-01-01', '2012-01-01 15:45:45', '15:45:45', point(1, 2))", - output: "insert into vitess_misc(id,b,d,dt,t,g) values (1,b'00000001','2012-01-01','2012-01-01 15:45:45','15:45:45','\\0\\0\\0\\0\x01\x01\\0\\0\\0\\0\\0\\0\\0\\0\\0\xf0?\\0\\0\\0\\0\\0\\0\\0@')", - table: "vitess_misc", - data: [][]string{ - {"1", "\x01", "2012-01-01", "2012-01-01 15:45:45", "15:45:45", "\x00\x00\x00\x00\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf0?\x00\x00\x00\x00\x00\x00\x00@"}, - }, - }, { - input: "insert into vitess_null values(1, null)", - output: "insert into vitess_null(id,val) values (1,null)", - table: "vitess_null", - data: [][]string{ - {"1", ""}, - }, - }, { - input: "insert into binary_pk values('a', 'aaa')", - output: "insert into binary_pk(b,val) values ('a\\0\\0\\0','aaa')", - table: "binary_pk", - data: [][]string{ - {"a\x00\x00\x00", "aaa"}, - }, - }, { - // Binary pk is a special case: https://github.com/vitessio/vitess/issues/3984 - input: "update binary_pk set val='bbb' where b='a\\0\\0\\0'", - output: "update binary_pk set val='bbb' where b='a\\0\\0\\0'", - table: "binary_pk", - data: [][]string{ - {"a\x00\x00\x00", "bbb"}, - }, - }} - - for _, tcases := range testcases { - execStatements(t, []string{tcases.input}) - want := []string{ - "begin", - tcases.output, - "/update _vt.vreplication set pos=", - "commit", - } - expectDBClientQueries(t, want) - if tcases.table != "" { - expectData(t, tcases.table, tcases.data) - } - } -} - -func TestPlayerDDL(t *testing.T) { - defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) - execStatements(t, []string{ - "create table t1(id int, primary key(id))", - fmt.Sprintf("create table %s.t1(id int, primary key(id))", vrepldb), - }) - defer execStatements(t, []string{ - "drop table t1", - fmt.Sprintf("drop table %s.t1", vrepldb), - }) - env.SchemaEngine.Reload(context.Background()) - - filter := &binlogdatapb.Filter{ - Rules: []*binlogdatapb.Rule{{ - Match: "/.*", - }}, - } - - cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") - // Issue a dummy change to ensure vreplication is initialized. Otherwise there - // is a race between the DDLs and the schema loader of vstreamer. - // Root cause seems to be with MySQL where t1 shows up in information_schema before - // the actual table is created. - execStatements(t, []string{"insert into t1 values(1)"}) - expectDBClientQueries(t, []string{ - "begin", - "insert into t1(id) values (1)", - "/update _vt.vreplication set pos=", - "commit", - }) - - execStatements(t, []string{"alter table t1 add column val varchar(128)"}) - execStatements(t, []string{"alter table t1 drop column val"}) - expectDBClientQueries(t, []string{ - "/update _vt.vreplication set pos=", - "/update _vt.vreplication set pos=", - }) - cancel() - - cancel, id := startVReplication(t, filter, binlogdatapb.OnDDLAction_STOP, "") - execStatements(t, []string{"alter table t1 add column val varchar(128)"}) - pos1 := masterPosition(t) - execStatements(t, []string{"alter table t1 drop column val"}) - pos2 := masterPosition(t) - // The stop position must be the GTID of the first DDL - expectDBClientQueries(t, []string{ - "begin", - fmt.Sprintf("/update _vt.vreplication set pos='%s'", pos1), - "/update _vt.vreplication set state='Stopped'", - "commit", - }) - // Restart vreplication - if _, err := playerEngine.Exec(fmt.Sprintf(`update _vt.vreplication set state = 'Running', message='' where id=%d`, id)); err != nil { - t.Fatal(err) - } - // It should stop at the next DDL - expectDBClientQueries(t, []string{ - "/update.*'Running'", - // Second update is from vreplicator. - "/update.*'Running'", - "begin", - fmt.Sprintf("/update.*'%s'", pos2), - "/update _vt.vreplication set state='Stopped'", - "commit", - }) - cancel() - - execStatements(t, []string{fmt.Sprintf("alter table %s.t1 add column val2 varchar(128)", vrepldb)}) - cancel, _ = startVReplication(t, filter, binlogdatapb.OnDDLAction_EXEC, "") - execStatements(t, []string{"alter table t1 add column val1 varchar(128)"}) - expectDBClientQueries(t, []string{ - "alter table t1 add column val1 varchar(128)", - "/update _vt.vreplication set pos=", - }) - execStatements(t, []string{"alter table t1 add column val2 varchar(128)"}) - expectDBClientQueries(t, []string{ - "alter table t1 add column val2 varchar(128)", - "/update _vt.vreplication set message='Duplicate", - }) - cancel() - - execStatements(t, []string{ - "alter table t1 drop column val1", - "alter table t1 drop column val2", - fmt.Sprintf("alter table %s.t1 drop column val1", vrepldb), - }) - - execStatements(t, []string{fmt.Sprintf("create table %s.t2(id int, primary key(id))", vrepldb)}) - cancel, _ = startVReplication(t, filter, binlogdatapb.OnDDLAction_EXEC_IGNORE, "") - execStatements(t, []string{"alter table t1 add column val1 varchar(128)"}) - expectDBClientQueries(t, []string{ - "alter table t1 add column val1 varchar(128)", - "/update _vt.vreplication set pos=", - }) - execStatements(t, []string{"alter table t1 add column val2 varchar(128)"}) - expectDBClientQueries(t, []string{ - "alter table t1 add column val2 varchar(128)", - "/update _vt.vreplication set pos=", - }) - cancel() -} - -func TestPlayerStopPos(t *testing.T) { - defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) - - execStatements(t, []string{ - "create table yes(id int, val varbinary(128), primary key(id))", - fmt.Sprintf("create table %s.yes(id int, val varbinary(128), primary key(id))", vrepldb), - "create table no(id int, val varbinary(128), primary key(id))", - }) - defer execStatements(t, []string{ - "drop table yes", - fmt.Sprintf("drop table %s.yes", vrepldb), - "drop table no", - }) - env.SchemaEngine.Reload(context.Background()) - - filter := &binlogdatapb.Filter{ - Rules: []*binlogdatapb.Rule{{ - Match: "/yes", - }}, - } - bls := &binlogdatapb.BinlogSource{ - Keyspace: env.KeyspaceName, - Shard: env.ShardName, - Filter: filter, - OnDdl: binlogdatapb.OnDDLAction_IGNORE, - } - startPos := masterPosition(t) - query := binlogplayer.CreateVReplicationState("test", bls, startPos, binlogplayer.BlpStopped, vrepldb) - qr, err := playerEngine.Exec(query) - if err != nil { - t.Fatal(err) - } - id := uint32(qr.InsertID) - for q := range globalDBQueries { - if strings.HasPrefix(q, "insert into _vt.vreplication") { - break - } - } - - // Test normal stop. - execStatements(t, []string{ - "insert into yes values(1, 'aaa')", - }) - stopPos := masterPosition(t) - query = binlogplayer.StartVReplicationUntil(id, stopPos) - if _, err := playerEngine.Exec(query); err != nil { - t.Fatal(err) - } - expectDBClientQueries(t, []string{ - "/update.*'Running'", - // Second update is from vreplicator. - "/update.*'Running'", - "begin", - "insert into yes(id,val) values (1,'aaa')", - fmt.Sprintf("/update.*'%s'", stopPos), - "/update.*'Stopped'", - "commit", - }) - - // Test stopping at empty transaction. - execStatements(t, []string{ - "insert into no values(2, 'aaa')", - "insert into no values(3, 'aaa')", - }) - stopPos = masterPosition(t) - execStatements(t, []string{ - "insert into no values(4, 'aaa')", - }) - query = binlogplayer.StartVReplicationUntil(id, stopPos) - if _, err := playerEngine.Exec(query); err != nil { - t.Fatal(err) - } - expectDBClientQueries(t, []string{ - "/update.*'Running'", - // Second update is from vreplicator. - "/update.*'Running'", - "begin", - // Since 'no' generates empty transactions that are skipped by - // vplayer, a commit is done only for the stop position event. - fmt.Sprintf("/update.*'%s'", stopPos), - "/update.*'Stopped'", - "commit", - }) - - // Test stopping when position is already reached. - query = binlogplayer.StartVReplicationUntil(id, stopPos) - if _, err := playerEngine.Exec(query); err != nil { - t.Fatal(err) - } - expectDBClientQueries(t, []string{ - "/update.*'Running'", - // Second update is from vreplicator. - "/update.*'Running'", - "/update.*'Stopped'.*already reached", - }) -} - -func TestPlayerIdleUpdate(t *testing.T) { - defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) - - savedIdleTimeout := idleTimeout - defer func() { idleTimeout = savedIdleTimeout }() - idleTimeout = 100 * time.Millisecond - - execStatements(t, []string{ - "create table t1(id int, val varbinary(128), primary key(id))", - fmt.Sprintf("create table %s.t1(id int, val varbinary(128), primary key(id))", vrepldb), - }) - defer execStatements(t, []string{ - "drop table t1", - fmt.Sprintf("drop table %s.t1", vrepldb), - }) - env.SchemaEngine.Reload(context.Background()) - - filter := &binlogdatapb.Filter{ - Rules: []*binlogdatapb.Rule{{ - Match: "/.*", - }}, - } - cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") - defer cancel() - - execStatements(t, []string{ - "insert into t1 values(1, 'aaa')", - }) - start := time.Now() - expectDBClientQueries(t, []string{ - "begin", - "insert into t1(id,val) values (1,'aaa')", - "/update _vt.vreplication set pos=", - "commit", - }) - // The above write will generate a new binlog event, and - // that event will loopback into player as an empty event. - // But it must not get saved until idleTimeout has passed. - // The exact positions are hard to verify because of this - // loopback mechanism. - expectDBClientQueries(t, []string{ - "/update _vt.vreplication set pos=", - }) - if duration := time.Since(start); duration < idleTimeout { - t.Errorf("duration: %v, must be at least %v", duration, idleTimeout) - } -} - -func TestPlayerSplitTransaction(t *testing.T) { - defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) - flag.Set("vstream_packet_size", "10") - defer flag.Set("vstream_packet_size", "10000") - - execStatements(t, []string{ - "create table t1(id int, val varbinary(128), primary key(id))", - fmt.Sprintf("create table %s.t1(id int, val varbinary(128), primary key(id))", vrepldb), - }) - defer execStatements(t, []string{ - "drop table t1", - fmt.Sprintf("drop table %s.t1", vrepldb), - }) - env.SchemaEngine.Reload(context.Background()) - - filter := &binlogdatapb.Filter{ - Rules: []*binlogdatapb.Rule{{ - Match: "/.*", - }}, - } - cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") - defer cancel() - - execStatements(t, []string{ - "begin", - "insert into t1 values(1, '123456')", - "insert into t1 values(2, '789012')", - "commit", - }) - // Because the packet size is 10, this is received as two events, - // but still combined as one transaction. - expectDBClientQueries(t, []string{ - "begin", - "insert into t1(id,val) values (1,'123456')", - "insert into t1(id,val) values (2,'789012')", - "/update _vt.vreplication set pos=", - "commit", - }) -} - -func TestPlayerLockErrors(t *testing.T) { - defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) - - execStatements(t, []string{ - "create table t1(id int, val varbinary(128), primary key(id))", - fmt.Sprintf("create table %s.t1(id int, val varbinary(128), primary key(id))", vrepldb), - }) - defer execStatements(t, []string{ - "drop table t1", - fmt.Sprintf("drop table %s.t1", vrepldb), - }) - env.SchemaEngine.Reload(context.Background()) - - filter := &binlogdatapb.Filter{ - Rules: []*binlogdatapb.Rule{{ - Match: "/.*", - }}, - } - cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") - defer cancel() - - execStatements(t, []string{ - "begin", - "insert into t1 values(1, 'aaa')", - "insert into t1 values(2, 'bbb')", - "commit", - }) - expectDBClientQueries(t, []string{ - "begin", - "insert into t1(id,val) values (1,'aaa')", - "insert into t1(id,val) values (2,'bbb')", - "/update _vt.vreplication set pos=", - "commit", - }) - - vconn := &realDBClient{nolog: true} - if err := vconn.Connect(); err != nil { - t.Error(err) - } - defer vconn.Close() - - // Start a transaction and lock the second row. - if _, err := vconn.ExecuteFetch("begin", 1); err != nil { - t.Error(err) - } - if _, err := vconn.ExecuteFetch("update t1 set val='bbb' where id=2", 1); err != nil { - t.Error(err) - } - - execStatements(t, []string{ - "begin", - "update t1 set val='ccc' where id=1", - "update t1 set val='ccc' where id=2", - "commit", - }) - // The innodb lock wait timeout is set to 1s. - expectDBClientQueries(t, []string{ - "begin", - "update t1 set val='ccc' where id=1", - "update t1 set val='ccc' where id=2", - "rollback", - }) - - // Release the lock, and watch the retry go through. - _, _ = vconn.ExecuteFetch("rollback", 1) - expectDBClientQueries(t, []string{ - "begin", - "update t1 set val='ccc' where id=1", - "update t1 set val='ccc' where id=2", - "/update _vt.vreplication set pos=", - "commit", - }) -} - -func TestPlayerCancelOnLock(t *testing.T) { - defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) - - execStatements(t, []string{ - "create table t1(id int, val varbinary(128), primary key(id))", - fmt.Sprintf("create table %s.t1(id int, val varbinary(128), primary key(id))", vrepldb), - }) - defer execStatements(t, []string{ - "drop table t1", - fmt.Sprintf("drop table %s.t1", vrepldb), - }) - env.SchemaEngine.Reload(context.Background()) - - filter := &binlogdatapb.Filter{ - Rules: []*binlogdatapb.Rule{{ - Match: "/.*", - }}, - } - cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") - defer cancel() - - execStatements(t, []string{ - "begin", - "insert into t1 values(1, 'aaa')", - "commit", - }) - expectDBClientQueries(t, []string{ - "begin", - "insert into t1(id,val) values (1,'aaa')", - "/update _vt.vreplication set pos=", - "commit", - }) - - vconn := &realDBClient{nolog: true} - if err := vconn.Connect(); err != nil { - t.Error(err) - } - defer vconn.Close() - - // Start a transaction and lock the row. - if _, err := vconn.ExecuteFetch("begin", 1); err != nil { - t.Error(err) - } - if _, err := vconn.ExecuteFetch("update t1 set val='bbb' where id=1", 1); err != nil { - t.Error(err) - } - - execStatements(t, []string{ - "begin", - "update t1 set val='ccc' where id=1", - "commit", - }) - // The innodb lock wait timeout is set to 1s. - expectDBClientQueries(t, []string{ - "begin", - "update t1 set val='ccc' where id=1", - "rollback", - }) - - // VReplication should not get stuck if you cancel now. - done := make(chan bool) - go func() { - cancel() - close(done) - }() - select { - case <-done: - case <-time.After(5 * time.Second): - t.Error("cancel is hung") - } -} - -func TestPlayerBatching(t *testing.T) { - defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) - - execStatements(t, []string{ - "create table t1(id int, val varbinary(128), primary key(id))", - fmt.Sprintf("create table %s.t1(id int, val varbinary(128), primary key(id))", vrepldb), - }) - defer execStatements(t, []string{ - "drop table t1", - fmt.Sprintf("drop table %s.t1", vrepldb), - }) - env.SchemaEngine.Reload(context.Background()) - - filter := &binlogdatapb.Filter{ - Rules: []*binlogdatapb.Rule{{ - Match: "/.*", - }}, - } - cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_EXEC, "") - defer cancel() - - execStatements(t, []string{ - "insert into t1 values(1, 'aaa')", - }) - expectDBClientQueries(t, []string{ - "begin", - "insert into t1(id,val) values (1,'aaa')", - "/update _vt.vreplication set pos=", - "commit", - }) - - vconn := &realDBClient{nolog: true} - if err := vconn.Connect(); err != nil { - t.Error(err) - } - defer vconn.Close() - - // Start a transaction and lock the row. - if _, err := vconn.ExecuteFetch("begin", 1); err != nil { - t.Error(err) - } - if _, err := vconn.ExecuteFetch("update t1 set val='bbb' where id=1", 1); err != nil { - t.Error(err) - } - - // create one transaction - execStatements(t, []string{ - "update t1 set val='ccc' where id=1", - }) - // Wait for the begin. The update will be blocked. - expectDBClientQueries(t, []string{ - "begin", - }) - - // Create two more transactions. They will go and wait in the relayLog. - execStatements(t, []string{ - "insert into t1 values(2, 'aaa')", - "insert into t1 values(3, 'aaa')", - "alter table t1 add column val2 varbinary(128)", - "alter table t1 drop column val2", - }) - - // Release the lock. - _, _ = vconn.ExecuteFetch("rollback", 1) - // First transaction will complete. The other two - // transactions must be batched into one. But the - // DDLs should be on their own. - expectDBClientQueries(t, []string{ - "update t1 set val='ccc' where id=1", - "/update _vt.vreplication set pos=", - "commit", - "begin", - "insert into t1(id,val) values (2,'aaa')", - "insert into t1(id,val) values (3,'aaa')", - "/update _vt.vreplication set pos=", - "commit", - "alter table t1 add column val2 varbinary(128)", - "/update _vt.vreplication set pos=", - "alter table t1 drop column val2", - "/update _vt.vreplication set pos=", - }) -} - -func TestPlayerRelayLogMaxSize(t *testing.T) { - defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) - - for i := 0; i < 2; i++ { - // First iteration checks max size, second checks max items - func() { - switch i { - case 0: - savedSize := relayLogMaxSize - defer func() { relayLogMaxSize = savedSize }() - relayLogMaxSize = 10 - case 1: - savedLen := relayLogMaxItems - defer func() { relayLogMaxItems = savedLen }() - relayLogMaxItems = 2 - } - - execStatements(t, []string{ - "create table t1(id int, val varbinary(128), primary key(id))", - fmt.Sprintf("create table %s.t1(id int, val varbinary(128), primary key(id))", vrepldb), - }) - defer execStatements(t, []string{ - "drop table t1", - fmt.Sprintf("drop table %s.t1", vrepldb), - }) - env.SchemaEngine.Reload(context.Background()) - - filter := &binlogdatapb.Filter{ - Rules: []*binlogdatapb.Rule{{ - Match: "/.*", - }}, - } - cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") - defer cancel() - - execStatements(t, []string{ - "insert into t1 values(1, '123456')", - }) - expectDBClientQueries(t, []string{ - "begin", - "insert into t1(id,val) values (1,'123456')", - "/update _vt.vreplication set pos=", - "commit", - }) - - vconn := &realDBClient{nolog: true} - if err := vconn.Connect(); err != nil { - t.Error(err) - } - defer vconn.Close() - - // Start a transaction and lock the row. - if _, err := vconn.ExecuteFetch("begin", 1); err != nil { - t.Error(err) - } - if _, err := vconn.ExecuteFetch("update t1 set val='bbb' where id=1", 1); err != nil { - t.Error(err) - } - - // create one transaction - execStatements(t, []string{ - "update t1 set val='ccc' where id=1", - }) - // Wait for the begin. The update will be blocked. - expectDBClientQueries(t, []string{ - "begin", - }) - - // Create two more transactions. They will go and wait in the relayLog. - execStatements(t, []string{ - "insert into t1 values(2, '789012')", - "insert into t1 values(3, '345678')", - "insert into t1 values(4, '901234')", - }) - - // Release the lock. - _, _ = vconn.ExecuteFetch("rollback", 1) - // First transaction will complete. The other two - // transactions must be batched into one. The last transaction - // will wait to be sent to the relay until the player fetches - // them. - expectDBClientQueries(t, []string{ - "update t1 set val='ccc' where id=1", - "/update _vt.vreplication set pos=", - "commit", - "begin", - "insert into t1(id,val) values (2,'789012')", - "insert into t1(id,val) values (3,'345678')", - "/update _vt.vreplication set pos=", - "commit", - "begin", - "insert into t1(id,val) values (4,'901234')", - "/update _vt.vreplication set pos=", - "commit", - }) - }() - } -} - -func TestRestartOnVStreamEnd(t *testing.T) { - defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) - - savedDelay := *retryDelay - defer func() { *retryDelay = savedDelay }() - *retryDelay = 1 * time.Millisecond - - execStatements(t, []string{ - "create table t1(id int, val varbinary(128), primary key(id))", - fmt.Sprintf("create table %s.t1(id int, val varbinary(128), primary key(id))", vrepldb), - }) - defer execStatements(t, []string{ - "drop table t1", - fmt.Sprintf("drop table %s.t1", vrepldb), - }) - env.SchemaEngine.Reload(context.Background()) - - filter := &binlogdatapb.Filter{ - Rules: []*binlogdatapb.Rule{{ - Match: "/.*", - }}, - } - cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") - defer cancel() - - execStatements(t, []string{ - "insert into t1 values(1, 'aaa')", - }) - expectDBClientQueries(t, []string{ - "begin", - "insert into t1(id,val) values (1,'aaa')", - "/update _vt.vreplication set pos=", - "commit", - }) - - streamerEngine.Close() - expectDBClientQueries(t, []string{ - "/update _vt.vreplication set message='vstream ended'", - }) - if err := streamerEngine.Open(env.KeyspaceName, env.ShardName); err != nil { - t.Fatal(err) - } - - execStatements(t, []string{ - "insert into t1 values(2, 'aaa')", - }) expectDBClientQueries(t, []string{ + "/insert into _vt.vreplication", "/update _vt.vreplication set state='Running'", - "begin", - "insert into t1(id,val) values (2,'aaa')", - "/update _vt.vreplication set pos=", - "commit", }) -} - -func TestTimestamp(t *testing.T) { - defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) - execStatements(t, []string{ - "create table t1(id int, ts timestamp, dt datetime)", - fmt.Sprintf("create table %s.t1(id int, ts timestamp, dt datetime)", vrepldb), - }) - defer execStatements(t, []string{ - "drop table t1", - fmt.Sprintf("drop table %s.t1", vrepldb), - }) - env.SchemaEngine.Reload(context.Background()) - - filter := &binlogdatapb.Filter{ - Rules: []*binlogdatapb.Rule{{ - Match: "/.*", - }}, - } - cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") - defer cancel() - - qr, err := env.Mysqld.FetchSuperQuery(context.Background(), "select now()") - if err != nil { - t.Fatal(err) - } - want := qr.Rows[0][0].ToString() - t.Logf("want: %s", want) - - execStatements(t, []string{ - fmt.Sprintf("insert into t1 values(1, '%s', '%s')", want, want), - }) - expectDBClientQueries(t, []string{ - "begin", - // The insert value for ts will be in UTC. - // We'll check the row instead. - "/insert into t1", - "/update _vt.vreplication set pos=", - "commit", - }) - - expectData(t, "t1", [][]string{{"1", want, want}}) -} - -func execStatements(t *testing.T, queries []string) { - t.Helper() - if err := env.Mysqld.ExecuteSuperQueryList(context.Background(), queries); err != nil { - t.Error(err) - } + var once sync.Once + return func() { + t.Helper() + once.Do(func() { + query := fmt.Sprintf("delete from _vt.vreplication where id = %d", qr.InsertID) + if _, err := playerEngine.Exec(query); err != nil { + t.Fatal(err) + } + expectDeleteQueries(t) + }) + }, int(qr.InsertID) } func startVReplication(t *testing.T, filter *binlogdatapb.Filter, onddl binlogdatapb.OnDDLAction, pos string) (cancelFunc func(), id int) { From 3fc6b08e331706b85698e90f8664b3b649367059 Mon Sep 17 00:00:00 2001 From: Rafael Chacon Date: Wed, 30 Oct 2019 14:51:24 -0700 Subject: [PATCH 15/28] Fixes bug in dbconfigs that was causing vstreamer to not work correctly Signed-off-by: Rafael Chacon --- go/vt/dbconfigs/dbconfigs.go | 9 +- .../vreplication/vplayer_test.go | 2949 ++++++++--------- 2 files changed, 1480 insertions(+), 1478 deletions(-) diff --git a/go/vt/dbconfigs/dbconfigs.go b/go/vt/dbconfigs/dbconfigs.go index 6816b7d9665..f17bc9afed6 100644 --- a/go/vt/dbconfigs/dbconfigs.go +++ b/go/vt/dbconfigs/dbconfigs.go @@ -175,8 +175,13 @@ func (dbcfgs *DBConfigs) ExternalRepl() *mysql.ConnParams { // ExternalReplWithDB returns connection parameters for repl with dbname set. func (dbcfgs *DBConfigs) ExternalReplWithDB() *mysql.ConnParams { - params := dbcfgs.makeParams(ExternalRepl, false) - params.DbName = params.DeprecatedDBName + params := dbcfgs.makeParams(ExternalRepl, true) + // TODO @rafael: This is a hack to allows to configure external databases by providing + // db-config-erepl-dbname. + if params.DeprecatedDBName != "" { + params.DbName = params.DeprecatedDBName + return params + } return params } diff --git a/go/vt/vttablet/tabletmanager/vreplication/vplayer_test.go b/go/vt/vttablet/tabletmanager/vreplication/vplayer_test.go index d4ee7160d55..8cf648cb0db 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vplayer_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vplayer_test.go @@ -17,9 +17,9 @@ limitations under the License. package vreplication import ( - //"flag" + "flag" "fmt" - // "strings" + "strings" "sync" "testing" "time" @@ -27,7 +27,7 @@ import ( "golang.org/x/net/context" "vitess.io/vitess/go/mysql" - // "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/binlog/binlogplayer" binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" @@ -35,8 +35,6 @@ import ( ) func TestMySQLVstreamerClient(t *testing.T) { - defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) - execStatements(t, []string{ "create table src1(id int, val varbinary(128), primary key(id))", fmt.Sprintf("create table %s.dst1(id int, val varbinary(128), primary key(id))", vrepldb), @@ -126,7 +124,6 @@ func TestMySQLVstreamerClient(t *testing.T) { for _, tcases := range testcases { execStatements(t, []string{tcases.input}) - time.Sleep(2 * time.Minute) expectDBClientQueries(t, tcases.output) if tcases.table != "" { expectData(t, tcases.table, tcases.data) @@ -135,1476 +132,1476 @@ func TestMySQLVstreamerClient(t *testing.T) { } -// func TestPlayerFilters(t *testing.T) { -// defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) - -// execStatements(t, []string{ -// "create table src1(id int, val varbinary(128), primary key(id))", -// fmt.Sprintf("create table %s.dst1(id int, val varbinary(128), primary key(id))", vrepldb), -// "create table src2(id int, val1 int, val2 int, primary key(id))", -// fmt.Sprintf("create table %s.dst2(id int, val1 int, sval2 int, rcount int, primary key(id))", vrepldb), -// "create table src3(id int, val varbinary(128), primary key(id))", -// fmt.Sprintf("create table %s.dst3(id int, val varbinary(128), primary key(id))", vrepldb), -// "create table yes(id int, val varbinary(128), primary key(id))", -// fmt.Sprintf("create table %s.yes(id int, val varbinary(128), primary key(id))", vrepldb), -// "create table no(id int, val varbinary(128), primary key(id))", -// "create table nopk(id int, val varbinary(128))", -// fmt.Sprintf("create table %s.nopk(id int, val varbinary(128))", vrepldb), -// }) -// defer execStatements(t, []string{ -// "drop table src1", -// fmt.Sprintf("drop table %s.dst1", vrepldb), -// "drop table src2", -// fmt.Sprintf("drop table %s.dst2", vrepldb), -// "drop table src3", -// fmt.Sprintf("drop table %s.dst3", vrepldb), -// "drop table yes", -// fmt.Sprintf("drop table %s.yes", vrepldb), -// "drop table no", -// "drop table nopk", -// fmt.Sprintf("drop table %s.nopk", vrepldb), -// }) -// env.SchemaEngine.Reload(context.Background()) - -// filter := &binlogdatapb.Filter{ -// Rules: []*binlogdatapb.Rule{{ -// Match: "dst1", -// Filter: "select * from src1", -// }, { -// Match: "dst2", -// Filter: "select id, val1, sum(val2) as sval2, count(*) as rcount from src2 group by id", -// }, { -// Match: "dst3", -// Filter: "select id, val from src3 group by id, val", -// }, { -// Match: "/yes", -// }, { -// Match: "/nopk", -// }}, -// } -// cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") -// defer cancel() - -// testcases := []struct { -// input string -// output []string -// table string -// data [][]string -// }{{ -// // insert with insertNormal -// input: "insert into src1 values(1, 'aaa')", -// output: []string{ -// "begin", -// "insert into dst1(id,val) values (1,'aaa')", -// "/update _vt.vreplication set pos=", -// "commit", -// }, -// table: "dst1", -// data: [][]string{ -// {"1", "aaa"}, -// }, -// }, { -// // update with insertNormal -// input: "update src1 set val='bbb'", -// output: []string{ -// "begin", -// "update dst1 set val='bbb' where id=1", -// "/update _vt.vreplication set pos=", -// "commit", -// }, -// table: "dst1", -// data: [][]string{ -// {"1", "bbb"}, -// }, -// }, { -// // delete with insertNormal -// input: "delete from src1 where id=1", -// output: []string{ -// "begin", -// "delete from dst1 where id=1", -// "/update _vt.vreplication set pos=", -// "commit", -// }, -// table: "dst1", -// data: [][]string{}, -// }, { -// // insert with insertOnDup -// input: "insert into src2 values(1, 2, 3)", -// output: []string{ -// "begin", -// "insert into dst2(id,val1,sval2,rcount) values (1,2,ifnull(3, 0),1) on duplicate key update val1=values(val1), sval2=sval2+ifnull(values(sval2), 0), rcount=rcount+1", -// "/update _vt.vreplication set pos=", -// "commit", -// }, -// table: "dst2", -// data: [][]string{ -// {"1", "2", "3", "1"}, -// }, -// }, { -// // update with insertOnDup -// input: "update src2 set val1=5, val2=1 where id=1", -// output: []string{ -// "begin", -// "update dst2 set val1=5, sval2=sval2-ifnull(3, 0)+ifnull(1, 0), rcount=rcount where id=1", -// "/update _vt.vreplication set pos=", -// "commit", -// }, -// table: "dst2", -// data: [][]string{ -// {"1", "5", "1", "1"}, -// }, -// }, { -// // delete with insertOnDup -// input: "delete from src2 where id=1", -// output: []string{ -// "begin", -// "update dst2 set val1=null, sval2=sval2-ifnull(1, 0), rcount=rcount-1 where id=1", -// "/update _vt.vreplication set pos=", -// "commit", -// }, -// table: "dst2", -// data: [][]string{ -// {"1", "", "0", "0"}, -// }, -// }, { -// // insert with insertIgnore -// input: "insert into src3 values(1, 'aaa')", -// output: []string{ -// "begin", -// "insert ignore into dst3(id,val) values (1,'aaa')", -// "/update _vt.vreplication set pos=", -// "commit", -// }, -// table: "dst3", -// data: [][]string{ -// {"1", "aaa"}, -// }, -// }, { -// // update with insertIgnore -// input: "update src3 set val='bbb'", -// output: []string{ -// "begin", -// "insert ignore into dst3(id,val) values (1,'bbb')", -// "/update _vt.vreplication set pos=", -// "commit", -// }, -// table: "dst3", -// data: [][]string{ -// {"1", "aaa"}, -// }, -// }, { -// // delete with insertIgnore -// input: "delete from src3 where id=1", -// output: []string{ -// "begin", -// "/update _vt.vreplication set pos=", -// "commit", -// }, -// table: "dst3", -// data: [][]string{ -// {"1", "aaa"}, -// }, -// }, { -// // insert: regular expression filter -// input: "insert into yes values(1, 'aaa')", -// output: []string{ -// "begin", -// "insert into yes(id,val) values (1,'aaa')", -// "/update _vt.vreplication set pos=", -// "commit", -// }, -// table: "yes", -// data: [][]string{ -// {"1", "aaa"}, -// }, -// }, { -// // update: regular expression filter -// input: "update yes set val='bbb'", -// output: []string{ -// "begin", -// "update yes set val='bbb' where id=1", -// "/update _vt.vreplication set pos=", -// "commit", -// }, -// table: "yes", -// data: [][]string{ -// {"1", "bbb"}, -// }, -// }, { -// // table should not match a rule -// input: "insert into no values(1, 'aaa')", -// output: []string{}, -// }, { -// // nopk: insert -// input: "insert into nopk values(1, 'aaa')", -// output: []string{ -// "begin", -// "insert into nopk(id,val) values (1,'aaa')", -// "/update _vt.vreplication set pos=", -// "commit", -// }, -// table: "nopk", -// data: [][]string{ -// {"1", "aaa"}, -// }, -// }, { -// // nopk: update -// input: "update nopk set val='bbb' where id=1", -// output: []string{ -// "begin", -// "delete from nopk where id=1 and val='aaa'", -// "insert into nopk(id,val) values (1,'bbb')", -// "/update _vt.vreplication set pos=", -// "commit", -// }, -// table: "nopk", -// data: [][]string{ -// {"1", "bbb"}, -// }, -// }, { -// // nopk: delete -// input: "delete from nopk where id=1", -// output: []string{ -// "begin", -// "delete from nopk where id=1 and val='bbb'", -// "/update _vt.vreplication set pos=", -// "commit", -// }, -// table: "nopk", -// data: [][]string{}, -// }} - -// for _, tcases := range testcases { -// execStatements(t, []string{tcases.input}) -// expectDBClientQueries(t, tcases.output) -// if tcases.table != "" { -// expectData(t, tcases.table, tcases.data) -// } -// } -// } - -// func TestPlayerKeywordNames(t *testing.T) { -// defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) - -// execStatements(t, []string{ -// "create table `begin`(`primary` int, `column` varbinary(128), primary key(`primary`))", -// fmt.Sprintf("create table %s.`begin`(`primary` int, `column` varbinary(128), primary key(`primary`))", vrepldb), -// "create table `rollback`(`primary` int, `column` varbinary(128), primary key(`primary`))", -// fmt.Sprintf("create table %s.`rollback`(`primary` int, `column` varbinary(128), primary key(`primary`))", vrepldb), -// "create table `commit`(`primary` int, `column` varbinary(128), primary key(`primary`))", -// fmt.Sprintf("create table %s.`commit`(`primary` int, `column` varbinary(128), primary key(`primary`))", vrepldb), -// }) -// defer execStatements(t, []string{ -// "drop table `begin`", -// fmt.Sprintf("drop table %s.`begin`", vrepldb), -// "drop table `rollback`", -// fmt.Sprintf("drop table %s.`rollback`", vrepldb), -// "drop table `commit`", -// fmt.Sprintf("drop table %s.`commit`", vrepldb), -// }) -// env.SchemaEngine.Reload(context.Background()) - -// filter := &binlogdatapb.Filter{ -// Rules: []*binlogdatapb.Rule{{ -// Match: "begin", -// Filter: "select * from `begin`", -// }, { -// Match: "rollback", -// Filter: "select `primary`, `column` from `rollback`", -// }, { -// Match: "commit", -// Filter: "select `primary`+1 as `primary`, concat(`column`, 'a') as `column` from `commit`", -// }}, -// } -// cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") -// defer cancel() - -// testcases := []struct { -// input string -// output []string -// table string -// data [][]string -// }{{ -// input: "insert into `begin` values(1, 'aaa')", -// output: []string{ -// "begin", -// "insert into `begin`(`primary`,`column`) values (1,'aaa')", -// "/update _vt.vreplication set pos=", -// "commit", -// }, -// table: "begin", -// data: [][]string{ -// {"1", "aaa"}, -// }, -// }, { -// input: "update `begin` set `column`='bbb'", -// output: []string{ -// "begin", -// "update `begin` set `column`='bbb' where `primary`=1", -// "/update _vt.vreplication set pos=", -// "commit", -// }, -// table: "begin", -// data: [][]string{ -// {"1", "bbb"}, -// }, -// }, { -// input: "delete from `begin` where `primary`=1", -// output: []string{ -// "begin", -// "delete from `begin` where `primary`=1", -// "/update _vt.vreplication set pos=", -// "commit", -// }, -// table: "begin", -// data: [][]string{}, -// }, { -// input: "insert into `rollback` values(1, 'aaa')", -// output: []string{ -// "begin", -// "insert into `rollback`(`primary`,`column`) values (1,'aaa')", -// "/update _vt.vreplication set pos=", -// "commit", -// }, -// table: "rollback", -// data: [][]string{ -// {"1", "aaa"}, -// }, -// }, { -// input: "update `rollback` set `column`='bbb'", -// output: []string{ -// "begin", -// "update `rollback` set `column`='bbb' where `primary`=1", -// "/update _vt.vreplication set pos=", -// "commit", -// }, -// table: "rollback", -// data: [][]string{ -// {"1", "bbb"}, -// }, -// }, { -// input: "delete from `rollback` where `primary`=1", -// output: []string{ -// "begin", -// "delete from `rollback` where `primary`=1", -// "/update _vt.vreplication set pos=", -// "commit", -// }, -// table: "rollback", -// data: [][]string{}, -// }, { -// input: "insert into `commit` values(1, 'aaa')", -// output: []string{ -// "begin", -// "insert into `commit`(`primary`,`column`) values (1 + 1,concat('aaa', 'a'))", -// "/update _vt.vreplication set pos=", -// "commit", -// }, -// table: "commit", -// data: [][]string{ -// {"2", "aaaa"}, -// }, -// }, { -// input: "update `commit` set `column`='bbb' where `primary`=1", -// output: []string{ -// "begin", -// "update `commit` set `column`=concat('bbb', 'a') where `primary`=(1 + 1)", -// "/update _vt.vreplication set pos=", -// "commit", -// }, -// table: "commit", -// data: [][]string{ -// {"2", "bbba"}, -// }, -// }, { -// input: "update `commit` set `primary`=2 where `primary`=1", -// output: []string{ -// "begin", -// "delete from `commit` where `primary`=(1 + 1)", -// "insert into `commit`(`primary`,`column`) values (2 + 1,concat('bbb', 'a'))", -// "/update _vt.vreplication set pos=", -// "commit", -// }, -// table: "commit", -// data: [][]string{ -// {"3", "bbba"}, -// }, -// }, { -// input: "delete from `commit` where `primary`=2", -// output: []string{ -// "begin", -// "delete from `commit` where `primary`=(2 + 1)", -// "/update _vt.vreplication set pos=", -// "commit", -// }, -// table: "commit", -// data: [][]string{}, -// }} - -// for _, tcases := range testcases { -// execStatements(t, []string{tcases.input}) -// expectDBClientQueries(t, tcases.output) -// if tcases.table != "" { -// expectData(t, tcases.table, tcases.data) -// } -// } -// } -// func TestUnicode(t *testing.T) { -// defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) - -// execStatements(t, []string{ -// "create table src1(id int, val varchar(128) COLLATE utf8_unicode_ci, primary key(id))", -// fmt.Sprintf("create table %s.dst1(id int, val varchar(128) COLLATE utf8_unicode_ci, primary key(id)) DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci", vrepldb), -// }) -// defer execStatements(t, []string{ -// "drop table src1", -// fmt.Sprintf("drop table %s.dst1", vrepldb), -// }) -// env.SchemaEngine.Reload(context.Background()) - -// filter := &binlogdatapb.Filter{ -// Rules: []*binlogdatapb.Rule{{ -// Match: "dst1", -// Filter: "select * from src1", -// }}, -// } -// cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") -// defer cancel() - -// testcases := []struct { -// input string -// output []string -// table string -// data [][]string -// }{{ -// // insert with insertNormal -// input: "insert into src1 values(1, '👍')", -// output: []string{ -// "begin", -// // We should expect the "Mojibaked" version. -// "insert into dst1(id,val) values (1,'ðŸ‘\u008d')", -// "/update _vt.vreplication set pos=", -// "commit", -// }, -// table: "dst1", -// data: [][]string{ -// {"1", "👍"}, -// }, -// }} - -// // We need a latin1 connection. -// conn, err := env.Mysqld.GetDbaConnection() -// if err != nil { -// t.Fatal(err) -// } -// defer conn.Close() - -// if _, err := conn.ExecuteFetch("set names latin1", 10000, false); err != nil { -// t.Fatal(err) -// } - -// for _, tcases := range testcases { -// if _, err := conn.ExecuteFetch(tcases.input, 10000, false); err != nil { -// t.Error(err) -// } -// expectDBClientQueries(t, tcases.output) -// if tcases.table != "" { -// customExpectData(t, tcases.table, tcases.data, func(ctx context.Context, query string) (*sqltypes.Result, error) { -// return conn.ExecuteFetch(query, 10000, true) -// }) -// } -// } -// } - -// func TestPlayerUpdates(t *testing.T) { -// defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) - -// execStatements(t, []string{ -// "create table t1(id int, grouped int, ungrouped int, summed int, primary key(id))", -// fmt.Sprintf("create table %s.t1(id int, grouped int, ungrouped int, summed int, rcount int, primary key(id))", vrepldb), -// }) -// defer execStatements(t, []string{ -// "drop table t1", -// fmt.Sprintf("drop table %s.t1", vrepldb), -// }) -// env.SchemaEngine.Reload(context.Background()) - -// filter := &binlogdatapb.Filter{ -// Rules: []*binlogdatapb.Rule{{ -// Match: "t1", -// Filter: "select id, grouped, ungrouped, sum(summed) as summed, count(*) as rcount from t1 group by id, grouped", -// }}, -// } -// cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") -// defer cancel() - -// testcases := []struct { -// input string -// output string -// table string -// data [][]string -// }{{ -// // Start with all nulls -// input: "insert into t1 values(1, null, null, null)", -// output: "insert into t1(id,grouped,ungrouped,summed,rcount) values (1,null,null,ifnull(null, 0),1) on duplicate key update ungrouped=values(ungrouped), summed=summed+ifnull(values(summed), 0), rcount=rcount+1", -// table: "t1", -// data: [][]string{ -// {"1", "", "", "0", "1"}, -// }, -// }, { -// // null to null values -// input: "update t1 set grouped=1 where id=1", -// output: "update t1 set ungrouped=null, summed=summed-ifnull(null, 0)+ifnull(null, 0), rcount=rcount where id=1", -// table: "t1", -// data: [][]string{ -// {"1", "", "", "0", "1"}, -// }, -// }, { -// // null to non-null values -// input: "update t1 set ungrouped=1, summed=1 where id=1", -// output: "update t1 set ungrouped=1, summed=summed-ifnull(null, 0)+ifnull(1, 0), rcount=rcount where id=1", -// table: "t1", -// data: [][]string{ -// {"1", "", "1", "1", "1"}, -// }, -// }, { -// // non-null to non-null values -// input: "update t1 set ungrouped=2, summed=2 where id=1", -// output: "update t1 set ungrouped=2, summed=summed-ifnull(1, 0)+ifnull(2, 0), rcount=rcount where id=1", -// table: "t1", -// data: [][]string{ -// {"1", "", "2", "2", "1"}, -// }, -// }, { -// // non-null to null values -// input: "update t1 set ungrouped=null, summed=null where id=1", -// output: "update t1 set ungrouped=null, summed=summed-ifnull(2, 0)+ifnull(null, 0), rcount=rcount where id=1", -// table: "t1", -// data: [][]string{ -// {"1", "", "", "0", "1"}, -// }, -// }, { -// // insert non-null values -// input: "insert into t1 values(2, 2, 3, 4)", -// output: "insert into t1(id,grouped,ungrouped,summed,rcount) values (2,2,3,ifnull(4, 0),1) on duplicate key update ungrouped=values(ungrouped), summed=summed+ifnull(values(summed), 0), rcount=rcount+1", -// table: "t1", -// data: [][]string{ -// {"1", "", "", "0", "1"}, -// {"2", "2", "3", "4", "1"}, -// }, -// }, { -// // delete non-null values -// input: "delete from t1 where id=2", -// output: "update t1 set ungrouped=null, summed=summed-ifnull(4, 0), rcount=rcount-1 where id=2", -// table: "t1", -// data: [][]string{ -// {"1", "", "", "0", "1"}, -// {"2", "2", "", "0", "0"}, -// }, -// }} - -// for _, tcases := range testcases { -// execStatements(t, []string{tcases.input}) -// output := []string{ -// "begin", -// tcases.output, -// "/update _vt.vreplication set pos=", -// "commit", -// } -// if tcases.output == "" { -// output = []string{ -// "begin", -// "/update _vt.vreplication set pos=", -// "commit", -// } -// } -// expectDBClientQueries(t, output) -// if tcases.table != "" { -// expectData(t, tcases.table, tcases.data) -// } -// } -// } - -// func TestPlayerRowMove(t *testing.T) { -// defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) - -// execStatements(t, []string{ -// "create table src(id int, val1 int, val2 int, primary key(id))", -// fmt.Sprintf("create table %s.dst(val1 int, sval2 int, rcount int, primary key(val1))", vrepldb), -// }) -// defer execStatements(t, []string{ -// "drop table src", -// fmt.Sprintf("drop table %s.dst", vrepldb), -// }) -// env.SchemaEngine.Reload(context.Background()) - -// filter := &binlogdatapb.Filter{ -// Rules: []*binlogdatapb.Rule{{ -// Match: "dst", -// Filter: "select val1, sum(val2) as sval2, count(*) as rcount from src group by val1", -// }}, -// } -// cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") -// defer cancel() - -// execStatements(t, []string{ -// "insert into src values(1, 1, 1), (2, 2, 2), (3, 2, 3)", -// }) -// expectDBClientQueries(t, []string{ -// "begin", -// "insert into dst(val1,sval2,rcount) values (1,ifnull(1, 0),1) on duplicate key update sval2=sval2+ifnull(values(sval2), 0), rcount=rcount+1", -// "insert into dst(val1,sval2,rcount) values (2,ifnull(2, 0),1) on duplicate key update sval2=sval2+ifnull(values(sval2), 0), rcount=rcount+1", -// "insert into dst(val1,sval2,rcount) values (2,ifnull(3, 0),1) on duplicate key update sval2=sval2+ifnull(values(sval2), 0), rcount=rcount+1", -// "/update _vt.vreplication set pos=", -// "commit", -// }) -// expectData(t, "dst", [][]string{ -// {"1", "1", "1"}, -// {"2", "5", "2"}, -// }) - -// execStatements(t, []string{ -// "update src set val1=1, val2=4 where id=3", -// }) -// expectDBClientQueries(t, []string{ -// "begin", -// "update dst set sval2=sval2-ifnull(3, 0), rcount=rcount-1 where val1=2", -// "insert into dst(val1,sval2,rcount) values (1,ifnull(4, 0),1) on duplicate key update sval2=sval2+ifnull(values(sval2), 0), rcount=rcount+1", -// "/update _vt.vreplication set pos=", -// "commit", -// }) -// expectData(t, "dst", [][]string{ -// {"1", "5", "2"}, -// {"2", "2", "1"}, -// }) -// } - -// func TestPlayerTypes(t *testing.T) { -// defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) - -// execStatements(t, []string{ -// "create table vitess_ints(tiny tinyint, tinyu tinyint unsigned, small smallint, smallu smallint unsigned, medium mediumint, mediumu mediumint unsigned, normal int, normalu int unsigned, big bigint, bigu bigint unsigned, y year, primary key(tiny))", -// fmt.Sprintf("create table %s.vitess_ints(tiny tinyint, tinyu tinyint unsigned, small smallint, smallu smallint unsigned, medium mediumint, mediumu mediumint unsigned, normal int, normalu int unsigned, big bigint, bigu bigint unsigned, y year, primary key(tiny))", vrepldb), -// "create table vitess_fracts(id int, deci decimal(5,2), num numeric(5,2), f float, d double, primary key(id))", -// fmt.Sprintf("create table %s.vitess_fracts(id int, deci decimal(5,2), num numeric(5,2), f float, d double, primary key(id))", vrepldb), -// "create table vitess_strings(vb varbinary(16), c char(16), vc varchar(16), b binary(4), tb tinyblob, bl blob, ttx tinytext, tx text, en enum('a','b'), s set('a','b'), primary key(vb))", -// fmt.Sprintf("create table %s.vitess_strings(vb varbinary(16), c char(16), vc varchar(16), b binary(4), tb tinyblob, bl blob, ttx tinytext, tx text, en enum('a','b'), s set('a','b'), primary key(vb))", vrepldb), -// "create table vitess_misc(id int, b bit(8), d date, dt datetime, t time, g geometry, primary key(id))", -// fmt.Sprintf("create table %s.vitess_misc(id int, b bit(8), d date, dt datetime, t time, g geometry, primary key(id))", vrepldb), -// "create table vitess_null(id int, val varbinary(128), primary key(id))", -// fmt.Sprintf("create table %s.vitess_null(id int, val varbinary(128), primary key(id))", vrepldb), -// "create table src1(id int, val varbinary(128), primary key(id))", -// fmt.Sprintf("create table %s.src1(id int, val varbinary(128), primary key(id))", vrepldb), -// "create table binary_pk(b binary(4), val varbinary(4), primary key(b))", -// fmt.Sprintf("create table %s.binary_pk(b binary(4), val varbinary(4), primary key(b))", vrepldb), -// }) -// defer execStatements(t, []string{ -// "drop table vitess_ints", -// fmt.Sprintf("drop table %s.vitess_ints", vrepldb), -// "drop table vitess_fracts", -// fmt.Sprintf("drop table %s.vitess_fracts", vrepldb), -// "drop table vitess_strings", -// fmt.Sprintf("drop table %s.vitess_strings", vrepldb), -// "drop table vitess_misc", -// fmt.Sprintf("drop table %s.vitess_misc", vrepldb), -// "drop table vitess_null", -// fmt.Sprintf("drop table %s.vitess_null", vrepldb), -// "drop table binary_pk", -// fmt.Sprintf("drop table %s.binary_pk", vrepldb), -// }) -// env.SchemaEngine.Reload(context.Background()) - -// filter := &binlogdatapb.Filter{ -// Rules: []*binlogdatapb.Rule{{ -// Match: "/.*", -// }}, -// } -// cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") -// defer cancel() -// testcases := []struct { -// input string -// output string -// table string -// data [][]string -// }{{ -// input: "insert into vitess_ints values(-128, 255, -32768, 65535, -8388608, 16777215, -2147483648, 4294967295, -9223372036854775808, 18446744073709551615, 2012)", -// output: "insert into vitess_ints(tiny,tinyu,small,smallu,medium,mediumu,normal,normalu,big,bigu,y) values (-128,255,-32768,65535,-8388608,16777215,-2147483648,4294967295,-9223372036854775808,18446744073709551615,2012)", -// table: "vitess_ints", -// data: [][]string{ -// {"-128", "255", "-32768", "65535", "-8388608", "16777215", "-2147483648", "4294967295", "-9223372036854775808", "18446744073709551615", "2012"}, -// }, -// }, { -// input: "insert into vitess_fracts values(1, 1.99, 2.99, 3.99, 4.99)", -// output: "insert into vitess_fracts(id,deci,num,f,d) values (1,1.99,2.99,3.99E+00,4.99E+00)", -// table: "vitess_fracts", -// data: [][]string{ -// {"1", "1.99", "2.99", "3.99", "4.99"}, -// }, -// }, { -// input: "insert into vitess_strings values('a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'a', 'a,b')", -// output: "insert into vitess_strings(vb,c,vc,b,tb,bl,ttx,tx,en,s) values ('a','b','c','d\\0\\0\\0','e','f','g','h','1','3')", -// table: "vitess_strings", -// data: [][]string{ -// {"a", "b", "c", "d\x00\x00\x00", "e", "f", "g", "h", "a", "a,b"}, -// }, -// }, { -// input: "insert into vitess_misc values(1, '\x01', '2012-01-01', '2012-01-01 15:45:45', '15:45:45', point(1, 2))", -// output: "insert into vitess_misc(id,b,d,dt,t,g) values (1,b'00000001','2012-01-01','2012-01-01 15:45:45','15:45:45','\\0\\0\\0\\0\x01\x01\\0\\0\\0\\0\\0\\0\\0\\0\\0\xf0?\\0\\0\\0\\0\\0\\0\\0@')", -// table: "vitess_misc", -// data: [][]string{ -// {"1", "\x01", "2012-01-01", "2012-01-01 15:45:45", "15:45:45", "\x00\x00\x00\x00\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf0?\x00\x00\x00\x00\x00\x00\x00@"}, -// }, -// }, { -// input: "insert into vitess_null values(1, null)", -// output: "insert into vitess_null(id,val) values (1,null)", -// table: "vitess_null", -// data: [][]string{ -// {"1", ""}, -// }, -// }, { -// input: "insert into binary_pk values('a', 'aaa')", -// output: "insert into binary_pk(b,val) values ('a\\0\\0\\0','aaa')", -// table: "binary_pk", -// data: [][]string{ -// {"a\x00\x00\x00", "aaa"}, -// }, -// }, { -// // Binary pk is a special case: https://github.com/vitessio/vitess/issues/3984 -// input: "update binary_pk set val='bbb' where b='a\\0\\0\\0'", -// output: "update binary_pk set val='bbb' where b='a\\0\\0\\0'", -// table: "binary_pk", -// data: [][]string{ -// {"a\x00\x00\x00", "bbb"}, -// }, -// }} - -// for _, tcases := range testcases { -// execStatements(t, []string{tcases.input}) -// want := []string{ -// "begin", -// tcases.output, -// "/update _vt.vreplication set pos=", -// "commit", -// } -// expectDBClientQueries(t, want) -// if tcases.table != "" { -// expectData(t, tcases.table, tcases.data) -// } -// } -// } - -// func TestPlayerDDL(t *testing.T) { -// defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) -// execStatements(t, []string{ -// "create table t1(id int, primary key(id))", -// fmt.Sprintf("create table %s.t1(id int, primary key(id))", vrepldb), -// }) -// defer execStatements(t, []string{ -// "drop table t1", -// fmt.Sprintf("drop table %s.t1", vrepldb), -// }) -// env.SchemaEngine.Reload(context.Background()) - -// filter := &binlogdatapb.Filter{ -// Rules: []*binlogdatapb.Rule{{ -// Match: "/.*", -// }}, -// } - -// cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") -// // Issue a dummy change to ensure vreplication is initialized. Otherwise there -// // is a race between the DDLs and the schema loader of vstreamer. -// // Root cause seems to be with MySQL where t1 shows up in information_schema before -// // the actual table is created. -// execStatements(t, []string{"insert into t1 values(1)"}) -// expectDBClientQueries(t, []string{ -// "begin", -// "insert into t1(id) values (1)", -// "/update _vt.vreplication set pos=", -// "commit", -// }) - -// execStatements(t, []string{"alter table t1 add column val varchar(128)"}) -// execStatements(t, []string{"alter table t1 drop column val"}) -// expectDBClientQueries(t, []string{ -// "/update _vt.vreplication set pos=", -// "/update _vt.vreplication set pos=", -// }) -// cancel() - -// cancel, id := startVReplication(t, filter, binlogdatapb.OnDDLAction_STOP, "") -// execStatements(t, []string{"alter table t1 add column val varchar(128)"}) -// pos1 := masterPosition(t) -// execStatements(t, []string{"alter table t1 drop column val"}) -// pos2 := masterPosition(t) -// // The stop position must be the GTID of the first DDL -// expectDBClientQueries(t, []string{ -// "begin", -// fmt.Sprintf("/update _vt.vreplication set pos='%s'", pos1), -// "/update _vt.vreplication set state='Stopped'", -// "commit", -// }) -// // Restart vreplication -// if _, err := playerEngine.Exec(fmt.Sprintf(`update _vt.vreplication set state = 'Running', message='' where id=%d`, id)); err != nil { -// t.Fatal(err) -// } -// // It should stop at the next DDL -// expectDBClientQueries(t, []string{ -// "/update.*'Running'", -// // Second update is from vreplicator. -// "/update.*'Running'", -// "begin", -// fmt.Sprintf("/update.*'%s'", pos2), -// "/update _vt.vreplication set state='Stopped'", -// "commit", -// }) -// cancel() - -// execStatements(t, []string{fmt.Sprintf("alter table %s.t1 add column val2 varchar(128)", vrepldb)}) -// cancel, _ = startVReplication(t, filter, binlogdatapb.OnDDLAction_EXEC, "") -// execStatements(t, []string{"alter table t1 add column val1 varchar(128)"}) -// expectDBClientQueries(t, []string{ -// "alter table t1 add column val1 varchar(128)", -// "/update _vt.vreplication set pos=", -// }) -// execStatements(t, []string{"alter table t1 add column val2 varchar(128)"}) -// expectDBClientQueries(t, []string{ -// "alter table t1 add column val2 varchar(128)", -// "/update _vt.vreplication set message='Duplicate", -// }) -// cancel() - -// execStatements(t, []string{ -// "alter table t1 drop column val1", -// "alter table t1 drop column val2", -// fmt.Sprintf("alter table %s.t1 drop column val1", vrepldb), -// }) - -// execStatements(t, []string{fmt.Sprintf("create table %s.t2(id int, primary key(id))", vrepldb)}) -// cancel, _ = startVReplication(t, filter, binlogdatapb.OnDDLAction_EXEC_IGNORE, "") -// execStatements(t, []string{"alter table t1 add column val1 varchar(128)"}) -// expectDBClientQueries(t, []string{ -// "alter table t1 add column val1 varchar(128)", -// "/update _vt.vreplication set pos=", -// }) -// execStatements(t, []string{"alter table t1 add column val2 varchar(128)"}) -// expectDBClientQueries(t, []string{ -// "alter table t1 add column val2 varchar(128)", -// "/update _vt.vreplication set pos=", -// }) -// cancel() -// } - -// func TestPlayerStopPos(t *testing.T) { -// defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) - -// execStatements(t, []string{ -// "create table yes(id int, val varbinary(128), primary key(id))", -// fmt.Sprintf("create table %s.yes(id int, val varbinary(128), primary key(id))", vrepldb), -// "create table no(id int, val varbinary(128), primary key(id))", -// }) -// defer execStatements(t, []string{ -// "drop table yes", -// fmt.Sprintf("drop table %s.yes", vrepldb), -// "drop table no", -// }) -// env.SchemaEngine.Reload(context.Background()) - -// filter := &binlogdatapb.Filter{ -// Rules: []*binlogdatapb.Rule{{ -// Match: "/yes", -// }}, -// } -// bls := &binlogdatapb.BinlogSource{ -// Keyspace: env.KeyspaceName, -// Shard: env.ShardName, -// Filter: filter, -// OnDdl: binlogdatapb.OnDDLAction_IGNORE, -// } -// startPos := masterPosition(t) -// query := binlogplayer.CreateVReplicationState("test", bls, startPos, binlogplayer.BlpStopped, vrepldb) -// qr, err := playerEngine.Exec(query) -// if err != nil { -// t.Fatal(err) -// } -// id := uint32(qr.InsertID) -// for q := range globalDBQueries { -// if strings.HasPrefix(q, "insert into _vt.vreplication") { -// break -// } -// } - -// // Test normal stop. -// execStatements(t, []string{ -// "insert into yes values(1, 'aaa')", -// }) -// stopPos := masterPosition(t) -// query = binlogplayer.StartVReplicationUntil(id, stopPos) -// if _, err := playerEngine.Exec(query); err != nil { -// t.Fatal(err) -// } -// expectDBClientQueries(t, []string{ -// "/update.*'Running'", -// // Second update is from vreplicator. -// "/update.*'Running'", -// "begin", -// "insert into yes(id,val) values (1,'aaa')", -// fmt.Sprintf("/update.*'%s'", stopPos), -// "/update.*'Stopped'", -// "commit", -// }) - -// // Test stopping at empty transaction. -// execStatements(t, []string{ -// "insert into no values(2, 'aaa')", -// "insert into no values(3, 'aaa')", -// }) -// stopPos = masterPosition(t) -// execStatements(t, []string{ -// "insert into no values(4, 'aaa')", -// }) -// query = binlogplayer.StartVReplicationUntil(id, stopPos) -// if _, err := playerEngine.Exec(query); err != nil { -// t.Fatal(err) -// } -// expectDBClientQueries(t, []string{ -// "/update.*'Running'", -// // Second update is from vreplicator. -// "/update.*'Running'", -// "begin", -// // Since 'no' generates empty transactions that are skipped by -// // vplayer, a commit is done only for the stop position event. -// fmt.Sprintf("/update.*'%s'", stopPos), -// "/update.*'Stopped'", -// "commit", -// }) - -// // Test stopping when position is already reached. -// query = binlogplayer.StartVReplicationUntil(id, stopPos) -// if _, err := playerEngine.Exec(query); err != nil { -// t.Fatal(err) -// } -// expectDBClientQueries(t, []string{ -// "/update.*'Running'", -// // Second update is from vreplicator. -// "/update.*'Running'", -// "/update.*'Stopped'.*already reached", -// }) -// } - -// func TestPlayerIdleUpdate(t *testing.T) { -// defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) - -// savedIdleTimeout := idleTimeout -// defer func() { idleTimeout = savedIdleTimeout }() -// idleTimeout = 100 * time.Millisecond - -// execStatements(t, []string{ -// "create table t1(id int, val varbinary(128), primary key(id))", -// fmt.Sprintf("create table %s.t1(id int, val varbinary(128), primary key(id))", vrepldb), -// }) -// defer execStatements(t, []string{ -// "drop table t1", -// fmt.Sprintf("drop table %s.t1", vrepldb), -// }) -// env.SchemaEngine.Reload(context.Background()) - -// filter := &binlogdatapb.Filter{ -// Rules: []*binlogdatapb.Rule{{ -// Match: "/.*", -// }}, -// } -// cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") -// defer cancel() - -// execStatements(t, []string{ -// "insert into t1 values(1, 'aaa')", -// }) -// start := time.Now() -// expectDBClientQueries(t, []string{ -// "begin", -// "insert into t1(id,val) values (1,'aaa')", -// "/update _vt.vreplication set pos=", -// "commit", -// }) -// // The above write will generate a new binlog event, and -// // that event will loopback into player as an empty event. -// // But it must not get saved until idleTimeout has passed. -// // The exact positions are hard to verify because of this -// // loopback mechanism. -// expectDBClientQueries(t, []string{ -// "/update _vt.vreplication set pos=", -// }) -// if duration := time.Since(start); duration < idleTimeout { -// t.Errorf("duration: %v, must be at least %v", duration, idleTimeout) -// } -// } - -// func TestPlayerSplitTransaction(t *testing.T) { -// defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) -// flag.Set("vstream_packet_size", "10") -// defer flag.Set("vstream_packet_size", "10000") - -// execStatements(t, []string{ -// "create table t1(id int, val varbinary(128), primary key(id))", -// fmt.Sprintf("create table %s.t1(id int, val varbinary(128), primary key(id))", vrepldb), -// }) -// defer execStatements(t, []string{ -// "drop table t1", -// fmt.Sprintf("drop table %s.t1", vrepldb), -// }) -// env.SchemaEngine.Reload(context.Background()) - -// filter := &binlogdatapb.Filter{ -// Rules: []*binlogdatapb.Rule{{ -// Match: "/.*", -// }}, -// } -// cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") -// defer cancel() - -// execStatements(t, []string{ -// "begin", -// "insert into t1 values(1, '123456')", -// "insert into t1 values(2, '789012')", -// "commit", -// }) -// // Because the packet size is 10, this is received as two events, -// // but still combined as one transaction. -// expectDBClientQueries(t, []string{ -// "begin", -// "insert into t1(id,val) values (1,'123456')", -// "insert into t1(id,val) values (2,'789012')", -// "/update _vt.vreplication set pos=", -// "commit", -// }) -// } - -// func TestPlayerLockErrors(t *testing.T) { -// defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) - -// execStatements(t, []string{ -// "create table t1(id int, val varbinary(128), primary key(id))", -// fmt.Sprintf("create table %s.t1(id int, val varbinary(128), primary key(id))", vrepldb), -// }) -// defer execStatements(t, []string{ -// "drop table t1", -// fmt.Sprintf("drop table %s.t1", vrepldb), -// }) -// env.SchemaEngine.Reload(context.Background()) - -// filter := &binlogdatapb.Filter{ -// Rules: []*binlogdatapb.Rule{{ -// Match: "/.*", -// }}, -// } -// cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") -// defer cancel() - -// execStatements(t, []string{ -// "begin", -// "insert into t1 values(1, 'aaa')", -// "insert into t1 values(2, 'bbb')", -// "commit", -// }) -// expectDBClientQueries(t, []string{ -// "begin", -// "insert into t1(id,val) values (1,'aaa')", -// "insert into t1(id,val) values (2,'bbb')", -// "/update _vt.vreplication set pos=", -// "commit", -// }) - -// vconn := &realDBClient{nolog: true} -// if err := vconn.Connect(); err != nil { -// t.Error(err) -// } -// defer vconn.Close() - -// // Start a transaction and lock the second row. -// if _, err := vconn.ExecuteFetch("begin", 1); err != nil { -// t.Error(err) -// } -// if _, err := vconn.ExecuteFetch("update t1 set val='bbb' where id=2", 1); err != nil { -// t.Error(err) -// } - -// execStatements(t, []string{ -// "begin", -// "update t1 set val='ccc' where id=1", -// "update t1 set val='ccc' where id=2", -// "commit", -// }) -// // The innodb lock wait timeout is set to 1s. -// expectDBClientQueries(t, []string{ -// "begin", -// "update t1 set val='ccc' where id=1", -// "update t1 set val='ccc' where id=2", -// "rollback", -// }) - -// // Release the lock, and watch the retry go through. -// _, _ = vconn.ExecuteFetch("rollback", 1) -// expectDBClientQueries(t, []string{ -// "begin", -// "update t1 set val='ccc' where id=1", -// "update t1 set val='ccc' where id=2", -// "/update _vt.vreplication set pos=", -// "commit", -// }) -// } - -// func TestPlayerCancelOnLock(t *testing.T) { -// defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) - -// execStatements(t, []string{ -// "create table t1(id int, val varbinary(128), primary key(id))", -// fmt.Sprintf("create table %s.t1(id int, val varbinary(128), primary key(id))", vrepldb), -// }) -// defer execStatements(t, []string{ -// "drop table t1", -// fmt.Sprintf("drop table %s.t1", vrepldb), -// }) -// env.SchemaEngine.Reload(context.Background()) - -// filter := &binlogdatapb.Filter{ -// Rules: []*binlogdatapb.Rule{{ -// Match: "/.*", -// }}, -// } -// cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") -// defer cancel() - -// execStatements(t, []string{ -// "begin", -// "insert into t1 values(1, 'aaa')", -// "commit", -// }) -// expectDBClientQueries(t, []string{ -// "begin", -// "insert into t1(id,val) values (1,'aaa')", -// "/update _vt.vreplication set pos=", -// "commit", -// }) - -// vconn := &realDBClient{nolog: true} -// if err := vconn.Connect(); err != nil { -// t.Error(err) -// } -// defer vconn.Close() - -// // Start a transaction and lock the row. -// if _, err := vconn.ExecuteFetch("begin", 1); err != nil { -// t.Error(err) -// } -// if _, err := vconn.ExecuteFetch("update t1 set val='bbb' where id=1", 1); err != nil { -// t.Error(err) -// } - -// execStatements(t, []string{ -// "begin", -// "update t1 set val='ccc' where id=1", -// "commit", -// }) -// // The innodb lock wait timeout is set to 1s. -// expectDBClientQueries(t, []string{ -// "begin", -// "update t1 set val='ccc' where id=1", -// "rollback", -// }) - -// // VReplication should not get stuck if you cancel now. -// done := make(chan bool) -// go func() { -// cancel() -// close(done) -// }() -// select { -// case <-done: -// case <-time.After(5 * time.Second): -// t.Error("cancel is hung") -// } -// } - -// func TestPlayerBatching(t *testing.T) { -// defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) - -// execStatements(t, []string{ -// "create table t1(id int, val varbinary(128), primary key(id))", -// fmt.Sprintf("create table %s.t1(id int, val varbinary(128), primary key(id))", vrepldb), -// }) -// defer execStatements(t, []string{ -// "drop table t1", -// fmt.Sprintf("drop table %s.t1", vrepldb), -// }) -// env.SchemaEngine.Reload(context.Background()) - -// filter := &binlogdatapb.Filter{ -// Rules: []*binlogdatapb.Rule{{ -// Match: "/.*", -// }}, -// } -// cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_EXEC, "") -// defer cancel() - -// execStatements(t, []string{ -// "insert into t1 values(1, 'aaa')", -// }) -// expectDBClientQueries(t, []string{ -// "begin", -// "insert into t1(id,val) values (1,'aaa')", -// "/update _vt.vreplication set pos=", -// "commit", -// }) - -// vconn := &realDBClient{nolog: true} -// if err := vconn.Connect(); err != nil { -// t.Error(err) -// } -// defer vconn.Close() - -// // Start a transaction and lock the row. -// if _, err := vconn.ExecuteFetch("begin", 1); err != nil { -// t.Error(err) -// } -// if _, err := vconn.ExecuteFetch("update t1 set val='bbb' where id=1", 1); err != nil { -// t.Error(err) -// } - -// // create one transaction -// execStatements(t, []string{ -// "update t1 set val='ccc' where id=1", -// }) -// // Wait for the begin. The update will be blocked. -// expectDBClientQueries(t, []string{ -// "begin", -// }) - -// // Create two more transactions. They will go and wait in the relayLog. -// execStatements(t, []string{ -// "insert into t1 values(2, 'aaa')", -// "insert into t1 values(3, 'aaa')", -// "alter table t1 add column val2 varbinary(128)", -// "alter table t1 drop column val2", -// }) - -// // Release the lock. -// _, _ = vconn.ExecuteFetch("rollback", 1) -// // First transaction will complete. The other two -// // transactions must be batched into one. But the -// // DDLs should be on their own. -// expectDBClientQueries(t, []string{ -// "update t1 set val='ccc' where id=1", -// "/update _vt.vreplication set pos=", -// "commit", -// "begin", -// "insert into t1(id,val) values (2,'aaa')", -// "insert into t1(id,val) values (3,'aaa')", -// "/update _vt.vreplication set pos=", -// "commit", -// "alter table t1 add column val2 varbinary(128)", -// "/update _vt.vreplication set pos=", -// "alter table t1 drop column val2", -// "/update _vt.vreplication set pos=", -// }) -// } - -// func TestPlayerRelayLogMaxSize(t *testing.T) { -// defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) - -// for i := 0; i < 2; i++ { -// // First iteration checks max size, second checks max items -// func() { -// switch i { -// case 0: -// savedSize := relayLogMaxSize -// defer func() { relayLogMaxSize = savedSize }() -// relayLogMaxSize = 10 -// case 1: -// savedLen := relayLogMaxItems -// defer func() { relayLogMaxItems = savedLen }() -// relayLogMaxItems = 2 -// } - -// execStatements(t, []string{ -// "create table t1(id int, val varbinary(128), primary key(id))", -// fmt.Sprintf("create table %s.t1(id int, val varbinary(128), primary key(id))", vrepldb), -// }) -// defer execStatements(t, []string{ -// "drop table t1", -// fmt.Sprintf("drop table %s.t1", vrepldb), -// }) -// env.SchemaEngine.Reload(context.Background()) - -// filter := &binlogdatapb.Filter{ -// Rules: []*binlogdatapb.Rule{{ -// Match: "/.*", -// }}, -// } -// cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") -// defer cancel() - -// execStatements(t, []string{ -// "insert into t1 values(1, '123456')", -// }) -// expectDBClientQueries(t, []string{ -// "begin", -// "insert into t1(id,val) values (1,'123456')", -// "/update _vt.vreplication set pos=", -// "commit", -// }) - -// vconn := &realDBClient{nolog: true} -// if err := vconn.Connect(); err != nil { -// t.Error(err) -// } -// defer vconn.Close() - -// // Start a transaction and lock the row. -// if _, err := vconn.ExecuteFetch("begin", 1); err != nil { -// t.Error(err) -// } -// if _, err := vconn.ExecuteFetch("update t1 set val='bbb' where id=1", 1); err != nil { -// t.Error(err) -// } - -// // create one transaction -// execStatements(t, []string{ -// "update t1 set val='ccc' where id=1", -// }) -// // Wait for the begin. The update will be blocked. -// expectDBClientQueries(t, []string{ -// "begin", -// }) - -// // Create two more transactions. They will go and wait in the relayLog. -// execStatements(t, []string{ -// "insert into t1 values(2, '789012')", -// "insert into t1 values(3, '345678')", -// "insert into t1 values(4, '901234')", -// }) - -// // Release the lock. -// _, _ = vconn.ExecuteFetch("rollback", 1) -// // First transaction will complete. The other two -// // transactions must be batched into one. The last transaction -// // will wait to be sent to the relay until the player fetches -// // them. -// expectDBClientQueries(t, []string{ -// "update t1 set val='ccc' where id=1", -// "/update _vt.vreplication set pos=", -// "commit", -// "begin", -// "insert into t1(id,val) values (2,'789012')", -// "insert into t1(id,val) values (3,'345678')", -// "/update _vt.vreplication set pos=", -// "commit", -// "begin", -// "insert into t1(id,val) values (4,'901234')", -// "/update _vt.vreplication set pos=", -// "commit", -// }) -// }() -// } -// } - -// func TestRestartOnVStreamEnd(t *testing.T) { -// defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) - -// savedDelay := *retryDelay -// defer func() { *retryDelay = savedDelay }() -// *retryDelay = 1 * time.Millisecond - -// execStatements(t, []string{ -// "create table t1(id int, val varbinary(128), primary key(id))", -// fmt.Sprintf("create table %s.t1(id int, val varbinary(128), primary key(id))", vrepldb), -// }) -// defer execStatements(t, []string{ -// "drop table t1", -// fmt.Sprintf("drop table %s.t1", vrepldb), -// }) -// env.SchemaEngine.Reload(context.Background()) - -// filter := &binlogdatapb.Filter{ -// Rules: []*binlogdatapb.Rule{{ -// Match: "/.*", -// }}, -// } -// cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") -// defer cancel() - -// execStatements(t, []string{ -// "insert into t1 values(1, 'aaa')", -// }) -// expectDBClientQueries(t, []string{ -// "begin", -// "insert into t1(id,val) values (1,'aaa')", -// "/update _vt.vreplication set pos=", -// "commit", -// }) - -// streamerEngine.Close() -// expectDBClientQueries(t, []string{ -// "/update _vt.vreplication set message='vstream ended'", -// }) -// if err := streamerEngine.Open(env.KeyspaceName, env.ShardName); err != nil { -// t.Fatal(err) -// } - -// execStatements(t, []string{ -// "insert into t1 values(2, 'aaa')", -// }) -// expectDBClientQueries(t, []string{ -// "/update _vt.vreplication set state='Running'", -// "begin", -// "insert into t1(id,val) values (2,'aaa')", -// "/update _vt.vreplication set pos=", -// "commit", -// }) -// } - -// func TestTimestamp(t *testing.T) { -// defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) - -// execStatements(t, []string{ -// "create table t1(id int, ts timestamp, dt datetime)", -// fmt.Sprintf("create table %s.t1(id int, ts timestamp, dt datetime)", vrepldb), -// }) -// defer execStatements(t, []string{ -// "drop table t1", -// fmt.Sprintf("drop table %s.t1", vrepldb), -// }) -// env.SchemaEngine.Reload(context.Background()) - -// filter := &binlogdatapb.Filter{ -// Rules: []*binlogdatapb.Rule{{ -// Match: "/.*", -// }}, -// } -// cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") -// defer cancel() - -// qr, err := env.Mysqld.FetchSuperQuery(context.Background(), "select now()") -// if err != nil { -// t.Fatal(err) -// } -// want := qr.Rows[0][0].ToString() -// t.Logf("want: %s", want) - -// execStatements(t, []string{ -// fmt.Sprintf("insert into t1 values(1, '%s', '%s')", want, want), -// }) -// expectDBClientQueries(t, []string{ -// "begin", -// // The insert value for ts will be in UTC. -// // We'll check the row instead. -// "/insert into t1", -// "/update _vt.vreplication set pos=", -// "commit", -// }) - -// expectData(t, "t1", [][]string{{"1", want, want}}) -// } +func TestPlayerFilters(t *testing.T) { + defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) + + execStatements(t, []string{ + "create table src1(id int, val varbinary(128), primary key(id))", + fmt.Sprintf("create table %s.dst1(id int, val varbinary(128), primary key(id))", vrepldb), + "create table src2(id int, val1 int, val2 int, primary key(id))", + fmt.Sprintf("create table %s.dst2(id int, val1 int, sval2 int, rcount int, primary key(id))", vrepldb), + "create table src3(id int, val varbinary(128), primary key(id))", + fmt.Sprintf("create table %s.dst3(id int, val varbinary(128), primary key(id))", vrepldb), + "create table yes(id int, val varbinary(128), primary key(id))", + fmt.Sprintf("create table %s.yes(id int, val varbinary(128), primary key(id))", vrepldb), + "create table no(id int, val varbinary(128), primary key(id))", + "create table nopk(id int, val varbinary(128))", + fmt.Sprintf("create table %s.nopk(id int, val varbinary(128))", vrepldb), + }) + defer execStatements(t, []string{ + "drop table src1", + fmt.Sprintf("drop table %s.dst1", vrepldb), + "drop table src2", + fmt.Sprintf("drop table %s.dst2", vrepldb), + "drop table src3", + fmt.Sprintf("drop table %s.dst3", vrepldb), + "drop table yes", + fmt.Sprintf("drop table %s.yes", vrepldb), + "drop table no", + "drop table nopk", + fmt.Sprintf("drop table %s.nopk", vrepldb), + }) + env.SchemaEngine.Reload(context.Background()) + + filter := &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "dst1", + Filter: "select * from src1", + }, { + Match: "dst2", + Filter: "select id, val1, sum(val2) as sval2, count(*) as rcount from src2 group by id", + }, { + Match: "dst3", + Filter: "select id, val from src3 group by id, val", + }, { + Match: "/yes", + }, { + Match: "/nopk", + }}, + } + cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") + defer cancel() + + testcases := []struct { + input string + output []string + table string + data [][]string + }{{ + // insert with insertNormal + input: "insert into src1 values(1, 'aaa')", + output: []string{ + "begin", + "insert into dst1(id,val) values (1,'aaa')", + "/update _vt.vreplication set pos=", + "commit", + }, + table: "dst1", + data: [][]string{ + {"1", "aaa"}, + }, + }, { + // update with insertNormal + input: "update src1 set val='bbb'", + output: []string{ + "begin", + "update dst1 set val='bbb' where id=1", + "/update _vt.vreplication set pos=", + "commit", + }, + table: "dst1", + data: [][]string{ + {"1", "bbb"}, + }, + }, { + // delete with insertNormal + input: "delete from src1 where id=1", + output: []string{ + "begin", + "delete from dst1 where id=1", + "/update _vt.vreplication set pos=", + "commit", + }, + table: "dst1", + data: [][]string{}, + }, { + // insert with insertOnDup + input: "insert into src2 values(1, 2, 3)", + output: []string{ + "begin", + "insert into dst2(id,val1,sval2,rcount) values (1,2,ifnull(3, 0),1) on duplicate key update val1=values(val1), sval2=sval2+ifnull(values(sval2), 0), rcount=rcount+1", + "/update _vt.vreplication set pos=", + "commit", + }, + table: "dst2", + data: [][]string{ + {"1", "2", "3", "1"}, + }, + }, { + // update with insertOnDup + input: "update src2 set val1=5, val2=1 where id=1", + output: []string{ + "begin", + "update dst2 set val1=5, sval2=sval2-ifnull(3, 0)+ifnull(1, 0), rcount=rcount where id=1", + "/update _vt.vreplication set pos=", + "commit", + }, + table: "dst2", + data: [][]string{ + {"1", "5", "1", "1"}, + }, + }, { + // delete with insertOnDup + input: "delete from src2 where id=1", + output: []string{ + "begin", + "update dst2 set val1=null, sval2=sval2-ifnull(1, 0), rcount=rcount-1 where id=1", + "/update _vt.vreplication set pos=", + "commit", + }, + table: "dst2", + data: [][]string{ + {"1", "", "0", "0"}, + }, + }, { + // insert with insertIgnore + input: "insert into src3 values(1, 'aaa')", + output: []string{ + "begin", + "insert ignore into dst3(id,val) values (1,'aaa')", + "/update _vt.vreplication set pos=", + "commit", + }, + table: "dst3", + data: [][]string{ + {"1", "aaa"}, + }, + }, { + // update with insertIgnore + input: "update src3 set val='bbb'", + output: []string{ + "begin", + "insert ignore into dst3(id,val) values (1,'bbb')", + "/update _vt.vreplication set pos=", + "commit", + }, + table: "dst3", + data: [][]string{ + {"1", "aaa"}, + }, + }, { + // delete with insertIgnore + input: "delete from src3 where id=1", + output: []string{ + "begin", + "/update _vt.vreplication set pos=", + "commit", + }, + table: "dst3", + data: [][]string{ + {"1", "aaa"}, + }, + }, { + // insert: regular expression filter + input: "insert into yes values(1, 'aaa')", + output: []string{ + "begin", + "insert into yes(id,val) values (1,'aaa')", + "/update _vt.vreplication set pos=", + "commit", + }, + table: "yes", + data: [][]string{ + {"1", "aaa"}, + }, + }, { + // update: regular expression filter + input: "update yes set val='bbb'", + output: []string{ + "begin", + "update yes set val='bbb' where id=1", + "/update _vt.vreplication set pos=", + "commit", + }, + table: "yes", + data: [][]string{ + {"1", "bbb"}, + }, + }, { + // table should not match a rule + input: "insert into no values(1, 'aaa')", + output: []string{}, + }, { + // nopk: insert + input: "insert into nopk values(1, 'aaa')", + output: []string{ + "begin", + "insert into nopk(id,val) values (1,'aaa')", + "/update _vt.vreplication set pos=", + "commit", + }, + table: "nopk", + data: [][]string{ + {"1", "aaa"}, + }, + }, { + // nopk: update + input: "update nopk set val='bbb' where id=1", + output: []string{ + "begin", + "delete from nopk where id=1 and val='aaa'", + "insert into nopk(id,val) values (1,'bbb')", + "/update _vt.vreplication set pos=", + "commit", + }, + table: "nopk", + data: [][]string{ + {"1", "bbb"}, + }, + }, { + // nopk: delete + input: "delete from nopk where id=1", + output: []string{ + "begin", + "delete from nopk where id=1 and val='bbb'", + "/update _vt.vreplication set pos=", + "commit", + }, + table: "nopk", + data: [][]string{}, + }} + + for _, tcases := range testcases { + execStatements(t, []string{tcases.input}) + expectDBClientQueries(t, tcases.output) + if tcases.table != "" { + expectData(t, tcases.table, tcases.data) + } + } +} + +func TestPlayerKeywordNames(t *testing.T) { + defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) + + execStatements(t, []string{ + "create table `begin`(`primary` int, `column` varbinary(128), primary key(`primary`))", + fmt.Sprintf("create table %s.`begin`(`primary` int, `column` varbinary(128), primary key(`primary`))", vrepldb), + "create table `rollback`(`primary` int, `column` varbinary(128), primary key(`primary`))", + fmt.Sprintf("create table %s.`rollback`(`primary` int, `column` varbinary(128), primary key(`primary`))", vrepldb), + "create table `commit`(`primary` int, `column` varbinary(128), primary key(`primary`))", + fmt.Sprintf("create table %s.`commit`(`primary` int, `column` varbinary(128), primary key(`primary`))", vrepldb), + }) + defer execStatements(t, []string{ + "drop table `begin`", + fmt.Sprintf("drop table %s.`begin`", vrepldb), + "drop table `rollback`", + fmt.Sprintf("drop table %s.`rollback`", vrepldb), + "drop table `commit`", + fmt.Sprintf("drop table %s.`commit`", vrepldb), + }) + env.SchemaEngine.Reload(context.Background()) + + filter := &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "begin", + Filter: "select * from `begin`", + }, { + Match: "rollback", + Filter: "select `primary`, `column` from `rollback`", + }, { + Match: "commit", + Filter: "select `primary`+1 as `primary`, concat(`column`, 'a') as `column` from `commit`", + }}, + } + cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") + defer cancel() + + testcases := []struct { + input string + output []string + table string + data [][]string + }{{ + input: "insert into `begin` values(1, 'aaa')", + output: []string{ + "begin", + "insert into `begin`(`primary`,`column`) values (1,'aaa')", + "/update _vt.vreplication set pos=", + "commit", + }, + table: "begin", + data: [][]string{ + {"1", "aaa"}, + }, + }, { + input: "update `begin` set `column`='bbb'", + output: []string{ + "begin", + "update `begin` set `column`='bbb' where `primary`=1", + "/update _vt.vreplication set pos=", + "commit", + }, + table: "begin", + data: [][]string{ + {"1", "bbb"}, + }, + }, { + input: "delete from `begin` where `primary`=1", + output: []string{ + "begin", + "delete from `begin` where `primary`=1", + "/update _vt.vreplication set pos=", + "commit", + }, + table: "begin", + data: [][]string{}, + }, { + input: "insert into `rollback` values(1, 'aaa')", + output: []string{ + "begin", + "insert into `rollback`(`primary`,`column`) values (1,'aaa')", + "/update _vt.vreplication set pos=", + "commit", + }, + table: "rollback", + data: [][]string{ + {"1", "aaa"}, + }, + }, { + input: "update `rollback` set `column`='bbb'", + output: []string{ + "begin", + "update `rollback` set `column`='bbb' where `primary`=1", + "/update _vt.vreplication set pos=", + "commit", + }, + table: "rollback", + data: [][]string{ + {"1", "bbb"}, + }, + }, { + input: "delete from `rollback` where `primary`=1", + output: []string{ + "begin", + "delete from `rollback` where `primary`=1", + "/update _vt.vreplication set pos=", + "commit", + }, + table: "rollback", + data: [][]string{}, + }, { + input: "insert into `commit` values(1, 'aaa')", + output: []string{ + "begin", + "insert into `commit`(`primary`,`column`) values (1 + 1,concat('aaa', 'a'))", + "/update _vt.vreplication set pos=", + "commit", + }, + table: "commit", + data: [][]string{ + {"2", "aaaa"}, + }, + }, { + input: "update `commit` set `column`='bbb' where `primary`=1", + output: []string{ + "begin", + "update `commit` set `column`=concat('bbb', 'a') where `primary`=(1 + 1)", + "/update _vt.vreplication set pos=", + "commit", + }, + table: "commit", + data: [][]string{ + {"2", "bbba"}, + }, + }, { + input: "update `commit` set `primary`=2 where `primary`=1", + output: []string{ + "begin", + "delete from `commit` where `primary`=(1 + 1)", + "insert into `commit`(`primary`,`column`) values (2 + 1,concat('bbb', 'a'))", + "/update _vt.vreplication set pos=", + "commit", + }, + table: "commit", + data: [][]string{ + {"3", "bbba"}, + }, + }, { + input: "delete from `commit` where `primary`=2", + output: []string{ + "begin", + "delete from `commit` where `primary`=(2 + 1)", + "/update _vt.vreplication set pos=", + "commit", + }, + table: "commit", + data: [][]string{}, + }} + + for _, tcases := range testcases { + execStatements(t, []string{tcases.input}) + expectDBClientQueries(t, tcases.output) + if tcases.table != "" { + expectData(t, tcases.table, tcases.data) + } + } +} +func TestUnicode(t *testing.T) { + defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) + + execStatements(t, []string{ + "create table src1(id int, val varchar(128) COLLATE utf8_unicode_ci, primary key(id))", + fmt.Sprintf("create table %s.dst1(id int, val varchar(128) COLLATE utf8_unicode_ci, primary key(id)) DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci", vrepldb), + }) + defer execStatements(t, []string{ + "drop table src1", + fmt.Sprintf("drop table %s.dst1", vrepldb), + }) + env.SchemaEngine.Reload(context.Background()) + + filter := &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "dst1", + Filter: "select * from src1", + }}, + } + cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") + defer cancel() + + testcases := []struct { + input string + output []string + table string + data [][]string + }{{ + // insert with insertNormal + input: "insert into src1 values(1, '👍')", + output: []string{ + "begin", + // We should expect the "Mojibaked" version. + "insert into dst1(id,val) values (1,'ðŸ‘\u008d')", + "/update _vt.vreplication set pos=", + "commit", + }, + table: "dst1", + data: [][]string{ + {"1", "👍"}, + }, + }} + + // We need a latin1 connection. + conn, err := env.Mysqld.GetDbaConnection() + if err != nil { + t.Fatal(err) + } + defer conn.Close() + + if _, err := conn.ExecuteFetch("set names latin1", 10000, false); err != nil { + t.Fatal(err) + } + + for _, tcases := range testcases { + if _, err := conn.ExecuteFetch(tcases.input, 10000, false); err != nil { + t.Error(err) + } + expectDBClientQueries(t, tcases.output) + if tcases.table != "" { + customExpectData(t, tcases.table, tcases.data, func(ctx context.Context, query string) (*sqltypes.Result, error) { + return conn.ExecuteFetch(query, 10000, true) + }) + } + } +} + +func TestPlayerUpdates(t *testing.T) { + defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) + + execStatements(t, []string{ + "create table t1(id int, grouped int, ungrouped int, summed int, primary key(id))", + fmt.Sprintf("create table %s.t1(id int, grouped int, ungrouped int, summed int, rcount int, primary key(id))", vrepldb), + }) + defer execStatements(t, []string{ + "drop table t1", + fmt.Sprintf("drop table %s.t1", vrepldb), + }) + env.SchemaEngine.Reload(context.Background()) + + filter := &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "t1", + Filter: "select id, grouped, ungrouped, sum(summed) as summed, count(*) as rcount from t1 group by id, grouped", + }}, + } + cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") + defer cancel() + + testcases := []struct { + input string + output string + table string + data [][]string + }{{ + // Start with all nulls + input: "insert into t1 values(1, null, null, null)", + output: "insert into t1(id,grouped,ungrouped,summed,rcount) values (1,null,null,ifnull(null, 0),1) on duplicate key update ungrouped=values(ungrouped), summed=summed+ifnull(values(summed), 0), rcount=rcount+1", + table: "t1", + data: [][]string{ + {"1", "", "", "0", "1"}, + }, + }, { + // null to null values + input: "update t1 set grouped=1 where id=1", + output: "update t1 set ungrouped=null, summed=summed-ifnull(null, 0)+ifnull(null, 0), rcount=rcount where id=1", + table: "t1", + data: [][]string{ + {"1", "", "", "0", "1"}, + }, + }, { + // null to non-null values + input: "update t1 set ungrouped=1, summed=1 where id=1", + output: "update t1 set ungrouped=1, summed=summed-ifnull(null, 0)+ifnull(1, 0), rcount=rcount where id=1", + table: "t1", + data: [][]string{ + {"1", "", "1", "1", "1"}, + }, + }, { + // non-null to non-null values + input: "update t1 set ungrouped=2, summed=2 where id=1", + output: "update t1 set ungrouped=2, summed=summed-ifnull(1, 0)+ifnull(2, 0), rcount=rcount where id=1", + table: "t1", + data: [][]string{ + {"1", "", "2", "2", "1"}, + }, + }, { + // non-null to null values + input: "update t1 set ungrouped=null, summed=null where id=1", + output: "update t1 set ungrouped=null, summed=summed-ifnull(2, 0)+ifnull(null, 0), rcount=rcount where id=1", + table: "t1", + data: [][]string{ + {"1", "", "", "0", "1"}, + }, + }, { + // insert non-null values + input: "insert into t1 values(2, 2, 3, 4)", + output: "insert into t1(id,grouped,ungrouped,summed,rcount) values (2,2,3,ifnull(4, 0),1) on duplicate key update ungrouped=values(ungrouped), summed=summed+ifnull(values(summed), 0), rcount=rcount+1", + table: "t1", + data: [][]string{ + {"1", "", "", "0", "1"}, + {"2", "2", "3", "4", "1"}, + }, + }, { + // delete non-null values + input: "delete from t1 where id=2", + output: "update t1 set ungrouped=null, summed=summed-ifnull(4, 0), rcount=rcount-1 where id=2", + table: "t1", + data: [][]string{ + {"1", "", "", "0", "1"}, + {"2", "2", "", "0", "0"}, + }, + }} + + for _, tcases := range testcases { + execStatements(t, []string{tcases.input}) + output := []string{ + "begin", + tcases.output, + "/update _vt.vreplication set pos=", + "commit", + } + if tcases.output == "" { + output = []string{ + "begin", + "/update _vt.vreplication set pos=", + "commit", + } + } + expectDBClientQueries(t, output) + if tcases.table != "" { + expectData(t, tcases.table, tcases.data) + } + } +} + +func TestPlayerRowMove(t *testing.T) { + defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) + + execStatements(t, []string{ + "create table src(id int, val1 int, val2 int, primary key(id))", + fmt.Sprintf("create table %s.dst(val1 int, sval2 int, rcount int, primary key(val1))", vrepldb), + }) + defer execStatements(t, []string{ + "drop table src", + fmt.Sprintf("drop table %s.dst", vrepldb), + }) + env.SchemaEngine.Reload(context.Background()) + + filter := &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "dst", + Filter: "select val1, sum(val2) as sval2, count(*) as rcount from src group by val1", + }}, + } + cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") + defer cancel() + + execStatements(t, []string{ + "insert into src values(1, 1, 1), (2, 2, 2), (3, 2, 3)", + }) + expectDBClientQueries(t, []string{ + "begin", + "insert into dst(val1,sval2,rcount) values (1,ifnull(1, 0),1) on duplicate key update sval2=sval2+ifnull(values(sval2), 0), rcount=rcount+1", + "insert into dst(val1,sval2,rcount) values (2,ifnull(2, 0),1) on duplicate key update sval2=sval2+ifnull(values(sval2), 0), rcount=rcount+1", + "insert into dst(val1,sval2,rcount) values (2,ifnull(3, 0),1) on duplicate key update sval2=sval2+ifnull(values(sval2), 0), rcount=rcount+1", + "/update _vt.vreplication set pos=", + "commit", + }) + expectData(t, "dst", [][]string{ + {"1", "1", "1"}, + {"2", "5", "2"}, + }) + + execStatements(t, []string{ + "update src set val1=1, val2=4 where id=3", + }) + expectDBClientQueries(t, []string{ + "begin", + "update dst set sval2=sval2-ifnull(3, 0), rcount=rcount-1 where val1=2", + "insert into dst(val1,sval2,rcount) values (1,ifnull(4, 0),1) on duplicate key update sval2=sval2+ifnull(values(sval2), 0), rcount=rcount+1", + "/update _vt.vreplication set pos=", + "commit", + }) + expectData(t, "dst", [][]string{ + {"1", "5", "2"}, + {"2", "2", "1"}, + }) +} + +func TestPlayerTypes(t *testing.T) { + defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) + + execStatements(t, []string{ + "create table vitess_ints(tiny tinyint, tinyu tinyint unsigned, small smallint, smallu smallint unsigned, medium mediumint, mediumu mediumint unsigned, normal int, normalu int unsigned, big bigint, bigu bigint unsigned, y year, primary key(tiny))", + fmt.Sprintf("create table %s.vitess_ints(tiny tinyint, tinyu tinyint unsigned, small smallint, smallu smallint unsigned, medium mediumint, mediumu mediumint unsigned, normal int, normalu int unsigned, big bigint, bigu bigint unsigned, y year, primary key(tiny))", vrepldb), + "create table vitess_fracts(id int, deci decimal(5,2), num numeric(5,2), f float, d double, primary key(id))", + fmt.Sprintf("create table %s.vitess_fracts(id int, deci decimal(5,2), num numeric(5,2), f float, d double, primary key(id))", vrepldb), + "create table vitess_strings(vb varbinary(16), c char(16), vc varchar(16), b binary(4), tb tinyblob, bl blob, ttx tinytext, tx text, en enum('a','b'), s set('a','b'), primary key(vb))", + fmt.Sprintf("create table %s.vitess_strings(vb varbinary(16), c char(16), vc varchar(16), b binary(4), tb tinyblob, bl blob, ttx tinytext, tx text, en enum('a','b'), s set('a','b'), primary key(vb))", vrepldb), + "create table vitess_misc(id int, b bit(8), d date, dt datetime, t time, g geometry, primary key(id))", + fmt.Sprintf("create table %s.vitess_misc(id int, b bit(8), d date, dt datetime, t time, g geometry, primary key(id))", vrepldb), + "create table vitess_null(id int, val varbinary(128), primary key(id))", + fmt.Sprintf("create table %s.vitess_null(id int, val varbinary(128), primary key(id))", vrepldb), + "create table src1(id int, val varbinary(128), primary key(id))", + fmt.Sprintf("create table %s.src1(id int, val varbinary(128), primary key(id))", vrepldb), + "create table binary_pk(b binary(4), val varbinary(4), primary key(b))", + fmt.Sprintf("create table %s.binary_pk(b binary(4), val varbinary(4), primary key(b))", vrepldb), + }) + defer execStatements(t, []string{ + "drop table vitess_ints", + fmt.Sprintf("drop table %s.vitess_ints", vrepldb), + "drop table vitess_fracts", + fmt.Sprintf("drop table %s.vitess_fracts", vrepldb), + "drop table vitess_strings", + fmt.Sprintf("drop table %s.vitess_strings", vrepldb), + "drop table vitess_misc", + fmt.Sprintf("drop table %s.vitess_misc", vrepldb), + "drop table vitess_null", + fmt.Sprintf("drop table %s.vitess_null", vrepldb), + "drop table binary_pk", + fmt.Sprintf("drop table %s.binary_pk", vrepldb), + }) + env.SchemaEngine.Reload(context.Background()) + + filter := &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "/.*", + }}, + } + cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") + defer cancel() + testcases := []struct { + input string + output string + table string + data [][]string + }{{ + input: "insert into vitess_ints values(-128, 255, -32768, 65535, -8388608, 16777215, -2147483648, 4294967295, -9223372036854775808, 18446744073709551615, 2012)", + output: "insert into vitess_ints(tiny,tinyu,small,smallu,medium,mediumu,normal,normalu,big,bigu,y) values (-128,255,-32768,65535,-8388608,16777215,-2147483648,4294967295,-9223372036854775808,18446744073709551615,2012)", + table: "vitess_ints", + data: [][]string{ + {"-128", "255", "-32768", "65535", "-8388608", "16777215", "-2147483648", "4294967295", "-9223372036854775808", "18446744073709551615", "2012"}, + }, + }, { + input: "insert into vitess_fracts values(1, 1.99, 2.99, 3.99, 4.99)", + output: "insert into vitess_fracts(id,deci,num,f,d) values (1,1.99,2.99,3.99E+00,4.99E+00)", + table: "vitess_fracts", + data: [][]string{ + {"1", "1.99", "2.99", "3.99", "4.99"}, + }, + }, { + input: "insert into vitess_strings values('a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'a', 'a,b')", + output: "insert into vitess_strings(vb,c,vc,b,tb,bl,ttx,tx,en,s) values ('a','b','c','d\\0\\0\\0','e','f','g','h','1','3')", + table: "vitess_strings", + data: [][]string{ + {"a", "b", "c", "d\x00\x00\x00", "e", "f", "g", "h", "a", "a,b"}, + }, + }, { + input: "insert into vitess_misc values(1, '\x01', '2012-01-01', '2012-01-01 15:45:45', '15:45:45', point(1, 2))", + output: "insert into vitess_misc(id,b,d,dt,t,g) values (1,b'00000001','2012-01-01','2012-01-01 15:45:45','15:45:45','\\0\\0\\0\\0\x01\x01\\0\\0\\0\\0\\0\\0\\0\\0\\0\xf0?\\0\\0\\0\\0\\0\\0\\0@')", + table: "vitess_misc", + data: [][]string{ + {"1", "\x01", "2012-01-01", "2012-01-01 15:45:45", "15:45:45", "\x00\x00\x00\x00\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf0?\x00\x00\x00\x00\x00\x00\x00@"}, + }, + }, { + input: "insert into vitess_null values(1, null)", + output: "insert into vitess_null(id,val) values (1,null)", + table: "vitess_null", + data: [][]string{ + {"1", ""}, + }, + }, { + input: "insert into binary_pk values('a', 'aaa')", + output: "insert into binary_pk(b,val) values ('a\\0\\0\\0','aaa')", + table: "binary_pk", + data: [][]string{ + {"a\x00\x00\x00", "aaa"}, + }, + }, { + // Binary pk is a special case: https://github.com/vitessio/vitess/issues/3984 + input: "update binary_pk set val='bbb' where b='a\\0\\0\\0'", + output: "update binary_pk set val='bbb' where b='a\\0\\0\\0'", + table: "binary_pk", + data: [][]string{ + {"a\x00\x00\x00", "bbb"}, + }, + }} + + for _, tcases := range testcases { + execStatements(t, []string{tcases.input}) + want := []string{ + "begin", + tcases.output, + "/update _vt.vreplication set pos=", + "commit", + } + expectDBClientQueries(t, want) + if tcases.table != "" { + expectData(t, tcases.table, tcases.data) + } + } +} + +func TestPlayerDDL(t *testing.T) { + defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) + execStatements(t, []string{ + "create table t1(id int, primary key(id))", + fmt.Sprintf("create table %s.t1(id int, primary key(id))", vrepldb), + }) + defer execStatements(t, []string{ + "drop table t1", + fmt.Sprintf("drop table %s.t1", vrepldb), + }) + env.SchemaEngine.Reload(context.Background()) + + filter := &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "/.*", + }}, + } + + cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") + // Issue a dummy change to ensure vreplication is initialized. Otherwise there + // is a race between the DDLs and the schema loader of vstreamer. + // Root cause seems to be with MySQL where t1 shows up in information_schema before + // the actual table is created. + execStatements(t, []string{"insert into t1 values(1)"}) + expectDBClientQueries(t, []string{ + "begin", + "insert into t1(id) values (1)", + "/update _vt.vreplication set pos=", + "commit", + }) + + execStatements(t, []string{"alter table t1 add column val varchar(128)"}) + execStatements(t, []string{"alter table t1 drop column val"}) + expectDBClientQueries(t, []string{ + "/update _vt.vreplication set pos=", + "/update _vt.vreplication set pos=", + }) + cancel() + + cancel, id := startVReplication(t, filter, binlogdatapb.OnDDLAction_STOP, "") + execStatements(t, []string{"alter table t1 add column val varchar(128)"}) + pos1 := masterPosition(t) + execStatements(t, []string{"alter table t1 drop column val"}) + pos2 := masterPosition(t) + // The stop position must be the GTID of the first DDL + expectDBClientQueries(t, []string{ + "begin", + fmt.Sprintf("/update _vt.vreplication set pos='%s'", pos1), + "/update _vt.vreplication set state='Stopped'", + "commit", + }) + // Restart vreplication + if _, err := playerEngine.Exec(fmt.Sprintf(`update _vt.vreplication set state = 'Running', message='' where id=%d`, id)); err != nil { + t.Fatal(err) + } + // It should stop at the next DDL + expectDBClientQueries(t, []string{ + "/update.*'Running'", + // Second update is from vreplicator. + "/update.*'Running'", + "begin", + fmt.Sprintf("/update.*'%s'", pos2), + "/update _vt.vreplication set state='Stopped'", + "commit", + }) + cancel() + + execStatements(t, []string{fmt.Sprintf("alter table %s.t1 add column val2 varchar(128)", vrepldb)}) + cancel, _ = startVReplication(t, filter, binlogdatapb.OnDDLAction_EXEC, "") + execStatements(t, []string{"alter table t1 add column val1 varchar(128)"}) + expectDBClientQueries(t, []string{ + "alter table t1 add column val1 varchar(128)", + "/update _vt.vreplication set pos=", + }) + execStatements(t, []string{"alter table t1 add column val2 varchar(128)"}) + expectDBClientQueries(t, []string{ + "alter table t1 add column val2 varchar(128)", + "/update _vt.vreplication set message='Duplicate", + }) + cancel() + + execStatements(t, []string{ + "alter table t1 drop column val1", + "alter table t1 drop column val2", + fmt.Sprintf("alter table %s.t1 drop column val1", vrepldb), + }) + + execStatements(t, []string{fmt.Sprintf("create table %s.t2(id int, primary key(id))", vrepldb)}) + cancel, _ = startVReplication(t, filter, binlogdatapb.OnDDLAction_EXEC_IGNORE, "") + execStatements(t, []string{"alter table t1 add column val1 varchar(128)"}) + expectDBClientQueries(t, []string{ + "alter table t1 add column val1 varchar(128)", + "/update _vt.vreplication set pos=", + }) + execStatements(t, []string{"alter table t1 add column val2 varchar(128)"}) + expectDBClientQueries(t, []string{ + "alter table t1 add column val2 varchar(128)", + "/update _vt.vreplication set pos=", + }) + cancel() +} + +func TestPlayerStopPos(t *testing.T) { + defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) + + execStatements(t, []string{ + "create table yes(id int, val varbinary(128), primary key(id))", + fmt.Sprintf("create table %s.yes(id int, val varbinary(128), primary key(id))", vrepldb), + "create table no(id int, val varbinary(128), primary key(id))", + }) + defer execStatements(t, []string{ + "drop table yes", + fmt.Sprintf("drop table %s.yes", vrepldb), + "drop table no", + }) + env.SchemaEngine.Reload(context.Background()) + + filter := &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "/yes", + }}, + } + bls := &binlogdatapb.BinlogSource{ + Keyspace: env.KeyspaceName, + Shard: env.ShardName, + Filter: filter, + OnDdl: binlogdatapb.OnDDLAction_IGNORE, + } + startPos := masterPosition(t) + query := binlogplayer.CreateVReplicationState("test", bls, startPos, binlogplayer.BlpStopped, vrepldb) + qr, err := playerEngine.Exec(query) + if err != nil { + t.Fatal(err) + } + id := uint32(qr.InsertID) + for q := range globalDBQueries { + if strings.HasPrefix(q, "insert into _vt.vreplication") { + break + } + } + + // Test normal stop. + execStatements(t, []string{ + "insert into yes values(1, 'aaa')", + }) + stopPos := masterPosition(t) + query = binlogplayer.StartVReplicationUntil(id, stopPos) + if _, err := playerEngine.Exec(query); err != nil { + t.Fatal(err) + } + expectDBClientQueries(t, []string{ + "/update.*'Running'", + // Second update is from vreplicator. + "/update.*'Running'", + "begin", + "insert into yes(id,val) values (1,'aaa')", + fmt.Sprintf("/update.*'%s'", stopPos), + "/update.*'Stopped'", + "commit", + }) + + // Test stopping at empty transaction. + execStatements(t, []string{ + "insert into no values(2, 'aaa')", + "insert into no values(3, 'aaa')", + }) + stopPos = masterPosition(t) + execStatements(t, []string{ + "insert into no values(4, 'aaa')", + }) + query = binlogplayer.StartVReplicationUntil(id, stopPos) + if _, err := playerEngine.Exec(query); err != nil { + t.Fatal(err) + } + expectDBClientQueries(t, []string{ + "/update.*'Running'", + // Second update is from vreplicator. + "/update.*'Running'", + "begin", + // Since 'no' generates empty transactions that are skipped by + // vplayer, a commit is done only for the stop position event. + fmt.Sprintf("/update.*'%s'", stopPos), + "/update.*'Stopped'", + "commit", + }) + + // Test stopping when position is already reached. + query = binlogplayer.StartVReplicationUntil(id, stopPos) + if _, err := playerEngine.Exec(query); err != nil { + t.Fatal(err) + } + expectDBClientQueries(t, []string{ + "/update.*'Running'", + // Second update is from vreplicator. + "/update.*'Running'", + "/update.*'Stopped'.*already reached", + }) +} + +func TestPlayerIdleUpdate(t *testing.T) { + defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) + + savedIdleTimeout := idleTimeout + defer func() { idleTimeout = savedIdleTimeout }() + idleTimeout = 100 * time.Millisecond + + execStatements(t, []string{ + "create table t1(id int, val varbinary(128), primary key(id))", + fmt.Sprintf("create table %s.t1(id int, val varbinary(128), primary key(id))", vrepldb), + }) + defer execStatements(t, []string{ + "drop table t1", + fmt.Sprintf("drop table %s.t1", vrepldb), + }) + env.SchemaEngine.Reload(context.Background()) + + filter := &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "/.*", + }}, + } + cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") + defer cancel() + + execStatements(t, []string{ + "insert into t1 values(1, 'aaa')", + }) + start := time.Now() + expectDBClientQueries(t, []string{ + "begin", + "insert into t1(id,val) values (1,'aaa')", + "/update _vt.vreplication set pos=", + "commit", + }) + // The above write will generate a new binlog event, and + // that event will loopback into player as an empty event. + // But it must not get saved until idleTimeout has passed. + // The exact positions are hard to verify because of this + // loopback mechanism. + expectDBClientQueries(t, []string{ + "/update _vt.vreplication set pos=", + }) + if duration := time.Since(start); duration < idleTimeout { + t.Errorf("duration: %v, must be at least %v", duration, idleTimeout) + } +} + +func TestPlayerSplitTransaction(t *testing.T) { + defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) + flag.Set("vstream_packet_size", "10") + defer flag.Set("vstream_packet_size", "10000") + + execStatements(t, []string{ + "create table t1(id int, val varbinary(128), primary key(id))", + fmt.Sprintf("create table %s.t1(id int, val varbinary(128), primary key(id))", vrepldb), + }) + defer execStatements(t, []string{ + "drop table t1", + fmt.Sprintf("drop table %s.t1", vrepldb), + }) + env.SchemaEngine.Reload(context.Background()) + + filter := &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "/.*", + }}, + } + cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") + defer cancel() + + execStatements(t, []string{ + "begin", + "insert into t1 values(1, '123456')", + "insert into t1 values(2, '789012')", + "commit", + }) + // Because the packet size is 10, this is received as two events, + // but still combined as one transaction. + expectDBClientQueries(t, []string{ + "begin", + "insert into t1(id,val) values (1,'123456')", + "insert into t1(id,val) values (2,'789012')", + "/update _vt.vreplication set pos=", + "commit", + }) +} + +func TestPlayerLockErrors(t *testing.T) { + defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) + + execStatements(t, []string{ + "create table t1(id int, val varbinary(128), primary key(id))", + fmt.Sprintf("create table %s.t1(id int, val varbinary(128), primary key(id))", vrepldb), + }) + defer execStatements(t, []string{ + "drop table t1", + fmt.Sprintf("drop table %s.t1", vrepldb), + }) + env.SchemaEngine.Reload(context.Background()) + + filter := &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "/.*", + }}, + } + cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") + defer cancel() + + execStatements(t, []string{ + "begin", + "insert into t1 values(1, 'aaa')", + "insert into t1 values(2, 'bbb')", + "commit", + }) + expectDBClientQueries(t, []string{ + "begin", + "insert into t1(id,val) values (1,'aaa')", + "insert into t1(id,val) values (2,'bbb')", + "/update _vt.vreplication set pos=", + "commit", + }) + + vconn := &realDBClient{nolog: true} + if err := vconn.Connect(); err != nil { + t.Error(err) + } + defer vconn.Close() + + // Start a transaction and lock the second row. + if _, err := vconn.ExecuteFetch("begin", 1); err != nil { + t.Error(err) + } + if _, err := vconn.ExecuteFetch("update t1 set val='bbb' where id=2", 1); err != nil { + t.Error(err) + } + + execStatements(t, []string{ + "begin", + "update t1 set val='ccc' where id=1", + "update t1 set val='ccc' where id=2", + "commit", + }) + // The innodb lock wait timeout is set to 1s. + expectDBClientQueries(t, []string{ + "begin", + "update t1 set val='ccc' where id=1", + "update t1 set val='ccc' where id=2", + "rollback", + }) + + // Release the lock, and watch the retry go through. + _, _ = vconn.ExecuteFetch("rollback", 1) + expectDBClientQueries(t, []string{ + "begin", + "update t1 set val='ccc' where id=1", + "update t1 set val='ccc' where id=2", + "/update _vt.vreplication set pos=", + "commit", + }) +} + +func TestPlayerCancelOnLock(t *testing.T) { + defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) + + execStatements(t, []string{ + "create table t1(id int, val varbinary(128), primary key(id))", + fmt.Sprintf("create table %s.t1(id int, val varbinary(128), primary key(id))", vrepldb), + }) + defer execStatements(t, []string{ + "drop table t1", + fmt.Sprintf("drop table %s.t1", vrepldb), + }) + env.SchemaEngine.Reload(context.Background()) + + filter := &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "/.*", + }}, + } + cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") + defer cancel() + + execStatements(t, []string{ + "begin", + "insert into t1 values(1, 'aaa')", + "commit", + }) + expectDBClientQueries(t, []string{ + "begin", + "insert into t1(id,val) values (1,'aaa')", + "/update _vt.vreplication set pos=", + "commit", + }) + + vconn := &realDBClient{nolog: true} + if err := vconn.Connect(); err != nil { + t.Error(err) + } + defer vconn.Close() + + // Start a transaction and lock the row. + if _, err := vconn.ExecuteFetch("begin", 1); err != nil { + t.Error(err) + } + if _, err := vconn.ExecuteFetch("update t1 set val='bbb' where id=1", 1); err != nil { + t.Error(err) + } + + execStatements(t, []string{ + "begin", + "update t1 set val='ccc' where id=1", + "commit", + }) + // The innodb lock wait timeout is set to 1s. + expectDBClientQueries(t, []string{ + "begin", + "update t1 set val='ccc' where id=1", + "rollback", + }) + + // VReplication should not get stuck if you cancel now. + done := make(chan bool) + go func() { + cancel() + close(done) + }() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Error("cancel is hung") + } +} + +func TestPlayerBatching(t *testing.T) { + defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) + + execStatements(t, []string{ + "create table t1(id int, val varbinary(128), primary key(id))", + fmt.Sprintf("create table %s.t1(id int, val varbinary(128), primary key(id))", vrepldb), + }) + defer execStatements(t, []string{ + "drop table t1", + fmt.Sprintf("drop table %s.t1", vrepldb), + }) + env.SchemaEngine.Reload(context.Background()) + + filter := &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "/.*", + }}, + } + cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_EXEC, "") + defer cancel() + + execStatements(t, []string{ + "insert into t1 values(1, 'aaa')", + }) + expectDBClientQueries(t, []string{ + "begin", + "insert into t1(id,val) values (1,'aaa')", + "/update _vt.vreplication set pos=", + "commit", + }) + + vconn := &realDBClient{nolog: true} + if err := vconn.Connect(); err != nil { + t.Error(err) + } + defer vconn.Close() + + // Start a transaction and lock the row. + if _, err := vconn.ExecuteFetch("begin", 1); err != nil { + t.Error(err) + } + if _, err := vconn.ExecuteFetch("update t1 set val='bbb' where id=1", 1); err != nil { + t.Error(err) + } + + // create one transaction + execStatements(t, []string{ + "update t1 set val='ccc' where id=1", + }) + // Wait for the begin. The update will be blocked. + expectDBClientQueries(t, []string{ + "begin", + }) + + // Create two more transactions. They will go and wait in the relayLog. + execStatements(t, []string{ + "insert into t1 values(2, 'aaa')", + "insert into t1 values(3, 'aaa')", + "alter table t1 add column val2 varbinary(128)", + "alter table t1 drop column val2", + }) + + // Release the lock. + _, _ = vconn.ExecuteFetch("rollback", 1) + // First transaction will complete. The other two + // transactions must be batched into one. But the + // DDLs should be on their own. + expectDBClientQueries(t, []string{ + "update t1 set val='ccc' where id=1", + "/update _vt.vreplication set pos=", + "commit", + "begin", + "insert into t1(id,val) values (2,'aaa')", + "insert into t1(id,val) values (3,'aaa')", + "/update _vt.vreplication set pos=", + "commit", + "alter table t1 add column val2 varbinary(128)", + "/update _vt.vreplication set pos=", + "alter table t1 drop column val2", + "/update _vt.vreplication set pos=", + }) +} + +func TestPlayerRelayLogMaxSize(t *testing.T) { + defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) + + for i := 0; i < 2; i++ { + // First iteration checks max size, second checks max items + func() { + switch i { + case 0: + savedSize := relayLogMaxSize + defer func() { relayLogMaxSize = savedSize }() + relayLogMaxSize = 10 + case 1: + savedLen := relayLogMaxItems + defer func() { relayLogMaxItems = savedLen }() + relayLogMaxItems = 2 + } + + execStatements(t, []string{ + "create table t1(id int, val varbinary(128), primary key(id))", + fmt.Sprintf("create table %s.t1(id int, val varbinary(128), primary key(id))", vrepldb), + }) + defer execStatements(t, []string{ + "drop table t1", + fmt.Sprintf("drop table %s.t1", vrepldb), + }) + env.SchemaEngine.Reload(context.Background()) + + filter := &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "/.*", + }}, + } + cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") + defer cancel() + + execStatements(t, []string{ + "insert into t1 values(1, '123456')", + }) + expectDBClientQueries(t, []string{ + "begin", + "insert into t1(id,val) values (1,'123456')", + "/update _vt.vreplication set pos=", + "commit", + }) + + vconn := &realDBClient{nolog: true} + if err := vconn.Connect(); err != nil { + t.Error(err) + } + defer vconn.Close() + + // Start a transaction and lock the row. + if _, err := vconn.ExecuteFetch("begin", 1); err != nil { + t.Error(err) + } + if _, err := vconn.ExecuteFetch("update t1 set val='bbb' where id=1", 1); err != nil { + t.Error(err) + } + + // create one transaction + execStatements(t, []string{ + "update t1 set val='ccc' where id=1", + }) + // Wait for the begin. The update will be blocked. + expectDBClientQueries(t, []string{ + "begin", + }) + + // Create two more transactions. They will go and wait in the relayLog. + execStatements(t, []string{ + "insert into t1 values(2, '789012')", + "insert into t1 values(3, '345678')", + "insert into t1 values(4, '901234')", + }) + + // Release the lock. + _, _ = vconn.ExecuteFetch("rollback", 1) + // First transaction will complete. The other two + // transactions must be batched into one. The last transaction + // will wait to be sent to the relay until the player fetches + // them. + expectDBClientQueries(t, []string{ + "update t1 set val='ccc' where id=1", + "/update _vt.vreplication set pos=", + "commit", + "begin", + "insert into t1(id,val) values (2,'789012')", + "insert into t1(id,val) values (3,'345678')", + "/update _vt.vreplication set pos=", + "commit", + "begin", + "insert into t1(id,val) values (4,'901234')", + "/update _vt.vreplication set pos=", + "commit", + }) + }() + } +} + +func TestRestartOnVStreamEnd(t *testing.T) { + defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) + + savedDelay := *retryDelay + defer func() { *retryDelay = savedDelay }() + *retryDelay = 1 * time.Millisecond + + execStatements(t, []string{ + "create table t1(id int, val varbinary(128), primary key(id))", + fmt.Sprintf("create table %s.t1(id int, val varbinary(128), primary key(id))", vrepldb), + }) + defer execStatements(t, []string{ + "drop table t1", + fmt.Sprintf("drop table %s.t1", vrepldb), + }) + env.SchemaEngine.Reload(context.Background()) + + filter := &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "/.*", + }}, + } + cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") + defer cancel() + + execStatements(t, []string{ + "insert into t1 values(1, 'aaa')", + }) + expectDBClientQueries(t, []string{ + "begin", + "insert into t1(id,val) values (1,'aaa')", + "/update _vt.vreplication set pos=", + "commit", + }) + + streamerEngine.Close() + expectDBClientQueries(t, []string{ + "/update _vt.vreplication set message='vstream ended'", + }) + if err := streamerEngine.Open(env.KeyspaceName, env.ShardName); err != nil { + t.Fatal(err) + } + + execStatements(t, []string{ + "insert into t1 values(2, 'aaa')", + }) + expectDBClientQueries(t, []string{ + "/update _vt.vreplication set state='Running'", + "begin", + "insert into t1(id,val) values (2,'aaa')", + "/update _vt.vreplication set pos=", + "commit", + }) +} + +func TestTimestamp(t *testing.T) { + defer deleteTablet(addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true)) + + execStatements(t, []string{ + "create table t1(id int, ts timestamp, dt datetime)", + fmt.Sprintf("create table %s.t1(id int, ts timestamp, dt datetime)", vrepldb), + }) + defer execStatements(t, []string{ + "drop table t1", + fmt.Sprintf("drop table %s.t1", vrepldb), + }) + env.SchemaEngine.Reload(context.Background()) + + filter := &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "/.*", + }}, + } + cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") + defer cancel() + + qr, err := env.Mysqld.FetchSuperQuery(context.Background(), "select now()") + if err != nil { + t.Fatal(err) + } + want := qr.Rows[0][0].ToString() + t.Logf("want: %s", want) + + execStatements(t, []string{ + fmt.Sprintf("insert into t1 values(1, '%s', '%s')", want, want), + }) + expectDBClientQueries(t, []string{ + "begin", + // The insert value for ts will be in UTC. + // We'll check the row instead. + "/insert into t1", + "/update _vt.vreplication set pos=", + "commit", + }) + + expectData(t, "t1", [][]string{{"1", want, want}}) +} func execStatements(t *testing.T, queries []string) { t.Helper() From b674d5ee0fa58d63ffe21795c28c74d45d3b3ab1 Mon Sep 17 00:00:00 2001 From: Rafael Chacon Date: Wed, 6 Nov 2019 10:08:16 -0800 Subject: [PATCH 16/28] Adds tests for vstreamer_client Signed-off-by: Rafael Chacon --- .../vreplication/vplayer_test.go | 189 ++++-- .../vreplication/vstreamer_client.go | 6 +- .../vreplication/vstreamer_client_test.go | 553 ++++++++++++++++++ go/vt/vttablet/tabletserver/schema/engine.go | 7 + .../vttablet/tabletserver/vstreamer/engine.go | 7 + 5 files changed, 702 insertions(+), 60 deletions(-) create mode 100644 go/vt/vttablet/tabletmanager/vreplication/vstreamer_client_test.go diff --git a/go/vt/vttablet/tabletmanager/vreplication/vplayer_test.go b/go/vt/vttablet/tabletmanager/vreplication/vplayer_test.go index 8cf648cb0db..428517f1670 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vplayer_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vplayer_test.go @@ -86,7 +86,7 @@ func TestMySQLVstreamerClient(t *testing.T) { ExternalMysql: "erepl", } - cancel, _ := startVReplicationV2(t, filter, bls, "") + cancel, _ := startVReplication(t, bls, "") defer cancel() testcases := []struct { @@ -179,7 +179,13 @@ func TestPlayerFilters(t *testing.T) { Match: "/nopk", }}, } - cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") + bls := &binlogdatapb.BinlogSource{ + Keyspace: env.KeyspaceName, + Shard: env.ShardName, + Filter: filter, + OnDdl: binlogdatapb.OnDDLAction_IGNORE, + } + cancel, _ := startVReplication(t, bls, "") defer cancel() testcases := []struct { @@ -413,7 +419,15 @@ func TestPlayerKeywordNames(t *testing.T) { Filter: "select `primary`+1 as `primary`, concat(`column`, 'a') as `column` from `commit`", }}, } - cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") + + bls := &binlogdatapb.BinlogSource{ + Keyspace: env.KeyspaceName, + Shard: env.ShardName, + Filter: filter, + OnDdl: binlogdatapb.OnDDLAction_IGNORE, + } + + cancel, _ := startVReplication(t, bls, "") defer cancel() testcases := []struct { @@ -565,7 +579,13 @@ func TestUnicode(t *testing.T) { Filter: "select * from src1", }}, } - cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") + bls := &binlogdatapb.BinlogSource{ + Keyspace: env.KeyspaceName, + Shard: env.ShardName, + Filter: filter, + OnDdl: binlogdatapb.OnDDLAction_IGNORE, + } + cancel, _ := startVReplication(t, bls, "") defer cancel() testcases := []struct { @@ -632,7 +652,13 @@ func TestPlayerUpdates(t *testing.T) { Filter: "select id, grouped, ungrouped, sum(summed) as summed, count(*) as rcount from t1 group by id, grouped", }}, } - cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") + bls := &binlogdatapb.BinlogSource{ + Keyspace: env.KeyspaceName, + Shard: env.ShardName, + Filter: filter, + OnDdl: binlogdatapb.OnDDLAction_IGNORE, + } + cancel, _ := startVReplication(t, bls, "") defer cancel() testcases := []struct { @@ -741,7 +767,13 @@ func TestPlayerRowMove(t *testing.T) { Filter: "select val1, sum(val2) as sval2, count(*) as rcount from src group by val1", }}, } - cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") + bls := &binlogdatapb.BinlogSource{ + Keyspace: env.KeyspaceName, + Shard: env.ShardName, + Filter: filter, + OnDdl: binlogdatapb.OnDDLAction_IGNORE, + } + cancel, _ := startVReplication(t, bls, "") defer cancel() execStatements(t, []string{ @@ -816,7 +848,13 @@ func TestPlayerTypes(t *testing.T) { Match: "/.*", }}, } - cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") + bls := &binlogdatapb.BinlogSource{ + Keyspace: env.KeyspaceName, + Shard: env.ShardName, + Filter: filter, + OnDdl: binlogdatapb.OnDDLAction_IGNORE, + } + cancel, _ := startVReplication(t, bls, "") defer cancel() testcases := []struct { input string @@ -907,8 +945,13 @@ func TestPlayerDDL(t *testing.T) { Match: "/.*", }}, } - - cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") + bls := &binlogdatapb.BinlogSource{ + Keyspace: env.KeyspaceName, + Shard: env.ShardName, + Filter: filter, + OnDdl: binlogdatapb.OnDDLAction_IGNORE, + } + cancel, _ := startVReplication(t, bls, "") // Issue a dummy change to ensure vreplication is initialized. Otherwise there // is a race between the DDLs and the schema loader of vstreamer. // Root cause seems to be with MySQL where t1 shows up in information_schema before @@ -928,8 +971,13 @@ func TestPlayerDDL(t *testing.T) { "/update _vt.vreplication set pos=", }) cancel() - - cancel, id := startVReplication(t, filter, binlogdatapb.OnDDLAction_STOP, "") + bls = &binlogdatapb.BinlogSource{ + Keyspace: env.KeyspaceName, + Shard: env.ShardName, + Filter: filter, + OnDdl: binlogdatapb.OnDDLAction_STOP, + } + cancel, id := startVReplication(t, bls, "") execStatements(t, []string{"alter table t1 add column val varchar(128)"}) pos1 := masterPosition(t) execStatements(t, []string{"alter table t1 drop column val"}) @@ -956,9 +1004,14 @@ func TestPlayerDDL(t *testing.T) { "commit", }) cancel() - + bls = &binlogdatapb.BinlogSource{ + Keyspace: env.KeyspaceName, + Shard: env.ShardName, + Filter: filter, + OnDdl: binlogdatapb.OnDDLAction_EXEC, + } execStatements(t, []string{fmt.Sprintf("alter table %s.t1 add column val2 varchar(128)", vrepldb)}) - cancel, _ = startVReplication(t, filter, binlogdatapb.OnDDLAction_EXEC, "") + cancel, _ = startVReplication(t, bls, "") execStatements(t, []string{"alter table t1 add column val1 varchar(128)"}) expectDBClientQueries(t, []string{ "alter table t1 add column val1 varchar(128)", @@ -977,8 +1030,14 @@ func TestPlayerDDL(t *testing.T) { fmt.Sprintf("alter table %s.t1 drop column val1", vrepldb), }) + bls = &binlogdatapb.BinlogSource{ + Keyspace: env.KeyspaceName, + Shard: env.ShardName, + Filter: filter, + OnDdl: binlogdatapb.OnDDLAction_EXEC_IGNORE, + } execStatements(t, []string{fmt.Sprintf("create table %s.t2(id int, primary key(id))", vrepldb)}) - cancel, _ = startVReplication(t, filter, binlogdatapb.OnDDLAction_EXEC_IGNORE, "") + cancel, _ = startVReplication(t, bls, "") execStatements(t, []string{"alter table t1 add column val1 varchar(128)"}) expectDBClientQueries(t, []string{ "alter table t1 add column val1 varchar(128)", @@ -1111,7 +1170,13 @@ func TestPlayerIdleUpdate(t *testing.T) { Match: "/.*", }}, } - cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") + bls := &binlogdatapb.BinlogSource{ + Keyspace: env.KeyspaceName, + Shard: env.ShardName, + Filter: filter, + OnDdl: binlogdatapb.OnDDLAction_IGNORE, + } + cancel, _ := startVReplication(t, bls, "") defer cancel() execStatements(t, []string{ @@ -1157,7 +1222,13 @@ func TestPlayerSplitTransaction(t *testing.T) { Match: "/.*", }}, } - cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") + bls := &binlogdatapb.BinlogSource{ + Keyspace: env.KeyspaceName, + Shard: env.ShardName, + Filter: filter, + OnDdl: binlogdatapb.OnDDLAction_IGNORE, + } + cancel, _ := startVReplication(t, bls, "") defer cancel() execStatements(t, []string{ @@ -1195,7 +1266,13 @@ func TestPlayerLockErrors(t *testing.T) { Match: "/.*", }}, } - cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") + bls := &binlogdatapb.BinlogSource{ + Keyspace: env.KeyspaceName, + Shard: env.ShardName, + Filter: filter, + OnDdl: binlogdatapb.OnDDLAction_IGNORE, + } + cancel, _ := startVReplication(t, bls, "") defer cancel() execStatements(t, []string{ @@ -1269,7 +1346,13 @@ func TestPlayerCancelOnLock(t *testing.T) { Match: "/.*", }}, } - cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") + bls := &binlogdatapb.BinlogSource{ + Keyspace: env.KeyspaceName, + Shard: env.ShardName, + Filter: filter, + OnDdl: binlogdatapb.OnDDLAction_IGNORE, + } + cancel, _ := startVReplication(t, bls, "") defer cancel() execStatements(t, []string{ @@ -1341,7 +1424,13 @@ func TestPlayerBatching(t *testing.T) { Match: "/.*", }}, } - cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_EXEC, "") + bls := &binlogdatapb.BinlogSource{ + Keyspace: env.KeyspaceName, + Shard: env.ShardName, + Filter: filter, + OnDdl: binlogdatapb.OnDDLAction_EXEC, + } + cancel, _ := startVReplication(t, bls, "") defer cancel() execStatements(t, []string{ @@ -1438,7 +1527,13 @@ func TestPlayerRelayLogMaxSize(t *testing.T) { Match: "/.*", }}, } - cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") + bls := &binlogdatapb.BinlogSource{ + Keyspace: env.KeyspaceName, + Shard: env.ShardName, + Filter: filter, + OnDdl: binlogdatapb.OnDDLAction_IGNORE, + } + cancel, _ := startVReplication(t, bls, "") defer cancel() execStatements(t, []string{ @@ -1527,7 +1622,13 @@ func TestRestartOnVStreamEnd(t *testing.T) { Match: "/.*", }}, } - cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") + bls := &binlogdatapb.BinlogSource{ + Keyspace: env.KeyspaceName, + Shard: env.ShardName, + Filter: filter, + OnDdl: binlogdatapb.OnDDLAction_IGNORE, + } + cancel, _ := startVReplication(t, bls, "") defer cancel() execStatements(t, []string{ @@ -1578,7 +1679,14 @@ func TestTimestamp(t *testing.T) { Match: "/.*", }}, } - cancel, _ := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") + + bls := &binlogdatapb.BinlogSource{ + Keyspace: env.KeyspaceName, + Shard: env.ShardName, + Filter: filter, + OnDdl: binlogdatapb.OnDDLAction_IGNORE, + } + cancel, _ := startVReplication(t, bls, "") defer cancel() qr, err := env.Mysqld.FetchSuperQuery(context.Background(), "select now()") @@ -1610,7 +1718,7 @@ func execStatements(t *testing.T, queries []string) { } } -func startVReplicationV2(t *testing.T, filter *binlogdatapb.Filter, bls *binlogdatapb.BinlogSource, pos string) (cancelFunc func(), id int) { +func startVReplication(t *testing.T, bls *binlogdatapb.BinlogSource, pos string) (cancelFunc func(), id int) { t.Helper() if pos == "" { @@ -1639,41 +1747,6 @@ func startVReplicationV2(t *testing.T, filter *binlogdatapb.Filter, bls *binlogd }, int(qr.InsertID) } -func startVReplication(t *testing.T, filter *binlogdatapb.Filter, onddl binlogdatapb.OnDDLAction, pos string) (cancelFunc func(), id int) { - t.Helper() - - bls := &binlogdatapb.BinlogSource{ - Keyspace: env.KeyspaceName, - Shard: env.ShardName, - Filter: filter, - OnDdl: onddl, - } - if pos == "" { - pos = masterPosition(t) - } - query := binlogplayer.CreateVReplication("test", bls, pos, 9223372036854775807, 9223372036854775807, 0, vrepldb) - qr, err := playerEngine.Exec(query) - if err != nil { - t.Fatal(err) - } - expectDBClientQueries(t, []string{ - "/insert into _vt.vreplication", - "/update _vt.vreplication set state='Running'", - }) - - var once sync.Once - return func() { - t.Helper() - once.Do(func() { - query := fmt.Sprintf("delete from _vt.vreplication where id = %d", qr.InsertID) - if _, err := playerEngine.Exec(query); err != nil { - t.Fatal(err) - } - expectDeleteQueries(t) - }) - }, int(qr.InsertID) -} - func masterPosition(t *testing.T) string { t.Helper() pos, err := env.Mysqld.MasterPosition() diff --git a/go/vt/vttablet/tabletmanager/vreplication/vstreamer_client.go b/go/vt/vttablet/tabletmanager/vreplication/vstreamer_client.go index b35129342c8..cc7e0f19963 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vstreamer_client.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vstreamer_client.go @@ -173,7 +173,9 @@ func (vsClient *MySQLVStreamerClient) Open(ctx context.Context) (err error) { vsClient.vsEngine = vstreamer.NewEngine(mysqlSrvTopo, vsClient.sourceSe) vsClient.vsEngine.InitDBConfig(vsClient.sourceConnParams) - err = vsClient.vsEngine.Open("mysqlstreamer", "cell1") + // We don't really need a keyspace/cell as this is a dummy engine from the + // topology perspective + err = vsClient.vsEngine.Open("", "") if err != nil { return err } @@ -206,7 +208,7 @@ func (vsClient *MySQLVStreamerClient) VStream(ctx context.Context, startPos stri // VStreamRows part of the VStreamerClient interface func (vsClient *MySQLVStreamerClient) VStreamRows(ctx context.Context, query string, lastpk *querypb.QueryResult, send func(*binlogdatapb.VStreamRowsResponse) error) error { if !vsClient.isOpen { - return errors.New("Can't VStream without opening client") + return errors.New("Can't VStreamRows without opening client") } var row []sqltypes.Value if lastpk != nil { diff --git a/go/vt/vttablet/tabletmanager/vreplication/vstreamer_client_test.go b/go/vt/vttablet/tabletmanager/vreplication/vstreamer_client_test.go new file mode 100644 index 00000000000..0e26f26fc44 --- /dev/null +++ b/go/vt/vttablet/tabletmanager/vreplication/vstreamer_client_test.go @@ -0,0 +1,553 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vreplication + +import ( + "fmt" + "reflect" + "strings" + "sync" + "testing" + "time" + + "golang.org/x/net/context" + + "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/vt/vttablet/queryservice" + "vitess.io/vitess/go/vt/vttablet/tabletserver/schema" + "vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer" + + binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" + querypb "vitess.io/vitess/go/vt/proto/query" + topodatapb "vitess.io/vitess/go/vt/proto/topodata" +) + +func TestTabletVStreamerClientOpen(t *testing.T) { + tablet := addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true) + defer deleteTablet(tablet) + + type fields struct { + mu sync.Mutex + isOpen bool + tablet *topodatapb.Tablet + target *querypb.Target + tsQueryService queryservice.QueryService + } + type args struct { + ctx context.Context + } + tests := []struct { + name string + fields fields + args args + err string + }{ + { + name: "initializes streamer client", + fields: fields{ + tablet: tablet, + }, + args: args{ + ctx: context.Background(), + }, + }, + } + + for _, tcase := range tests { + t.Run(tcase.name, func(t *testing.T) { + vsClient := &TabletVStreamerClient{ + tablet: tcase.fields.tablet, + } + + err := vsClient.Open(tcase.args.ctx) + + if err != nil { + if !strings.Contains(err.Error(), tcase.err) { + t.Errorf("TabletVStreamerClient.Open() error:\n%v, want\n%v", err, tcase.err) + } + return + } + + if tcase.err != "" { + t.Errorf("TabletVStreamerClient.Open() error:\n%v, want\n%v", err, tcase.err) + } + + if !vsClient.isOpen { + t.Errorf("TabletVStreamerClient.Open() isOpen set to false, expected true") + } + + if vsClient.tablet == nil { + t.Errorf("TabletVStreamerClient.Open() expected sourceSe to be set") + } + }) + } +} + +func TestTabletVStreamerClientClose(t *testing.T) { + tablet := addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true) + defer deleteTablet(tablet) + + type fields struct { + mu sync.Mutex + isOpen bool + tablet *topodatapb.Tablet + target *querypb.Target + tsQueryService queryservice.QueryService + } + type args struct { + ctx context.Context + } + tests := []struct { + name string + fields fields + args args + err string + }{ + { + name: "closes engine correctly", + fields: fields{ + tablet: tablet, + }, + args: args{ + ctx: context.Background(), + }, + }, + } + + for _, tcase := range tests { + t.Run(tcase.name, func(t *testing.T) { + vsClient := &TabletVStreamerClient{ + tablet: tcase.fields.tablet, + } + + err := vsClient.Open(tcase.args.ctx) + if err != nil { + t.Errorf("Failed to Open vsClient") + return + } + + err = vsClient.Close(tcase.args.ctx) + + if tcase.err != "" { + t.Errorf("MySQLVStreamerClient.Close() error:\n%v, want\n%v", err, tcase.err) + } + + if vsClient.isOpen { + t.Errorf("MySQLVStreamerClient.Close() isOpen set to true, expected false") + } + }) + } +} + +func TestTabletVStreamerClientVStream(t *testing.T) { + tablet := addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true) + defer deleteTablet(tablet) + + vsClient := &TabletVStreamerClient{ + tablet: tablet, + } + + filter := &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "/.*", + }}, + } + eventsChan := make(chan *binlogdatapb.VEvent, 1000) + send := func(events []*binlogdatapb.VEvent) error { + fmt.Println(events) + fmt.Println(len(events)) + for _, e := range events { + eventsChan <- e + } + return nil + } + + execStatements(t, []string{ + "create table t1(id int, ts timestamp, dt datetime)", + fmt.Sprintf("create table %s.t1(id int, ts timestamp, dt datetime)", vrepldb), + }) + defer execStatements(t, []string{ + "drop table t1", + fmt.Sprintf("drop table %s.t1", vrepldb), + }) + + ctx := context.Background() + err := vsClient.Open(ctx) + if err != nil { + t.Errorf("Failed to Open vsClient") + return + } + + defer vsClient.Close(ctx) + + pos := masterPosition(t) + // This asserts that events are flowing through the VStream when using mysql client + go vsClient.VStream(ctx, pos, filter, send) + + qr, err := env.Mysqld.FetchSuperQuery(context.Background(), "select now()") + if err != nil { + t.Fatal(err) + } + want := qr.Rows[0][0].ToString() + execStatements(t, []string{ + fmt.Sprintf("insert into t1 values(1, '%s', '%s')", want, want), + }) + + select { + case got := <-eventsChan: + if got.Type != binlogdatapb.VEventType_GTID { + t.Errorf("Did not get expected events: want: %v, got: %v", binlogdatapb.VEventType_GTID, got.Type) + } + case <-time.After(5 * time.Second): + t.Errorf("no events received") + } +} + +func TestTabletVStreamerClientVStreamRows(t *testing.T) { + tablet := addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true) + defer deleteTablet(tablet) + + vsClient := &TabletVStreamerClient{ + tablet: tablet, + } + + eventsChan := make(chan *querypb.Row, 1000) + send := func(streamerResponse *binlogdatapb.VStreamRowsResponse) error { + for _, row := range streamerResponse.Rows { + eventsChan <- row + } + return nil + } + + execStatements(t, []string{ + "create table t1(id int, ts timestamp, dt datetime)", + fmt.Sprintf("create table %s.t1(id int, ts timestamp, dt datetime)", vrepldb), + }) + defer execStatements(t, []string{ + "drop table t1", + fmt.Sprintf("drop table %s.t1", vrepldb), + }) + + qr, err := env.Mysqld.FetchSuperQuery(context.Background(), "select now()") + if err != nil { + t.Fatal(err) + } + want := qr.Rows[0][0].ToString() + ctx := context.Background() + err = vsClient.Open(ctx) + if err != nil { + t.Errorf("Failed to Open vsClient") + return + } + + defer vsClient.Close(ctx) + + // This asserts that events are flowing through the VStream when using mysql client + go vsClient.VStreamRows(ctx, "select * from t1", nil, send) + + execStatements(t, []string{ + fmt.Sprintf("insert into t1 values(1, '%s', '%s')", want, want), + }) + + select { + case <-eventsChan: + // Success got expected + case <-time.After(5 * time.Second): + t.Errorf("no events received") + } +} + +func TestNewMySQLVStreamerClient(t *testing.T) { + type args struct { + sourceConnParams *mysql.ConnParams + } + tests := []struct { + name string + args args + want *MySQLVStreamerClient + }{ + { + name: "sets conn params for MySQLVStreamerClient ", + args: args{ + sourceConnParams: &mysql.ConnParams{ + Host: "testhost", + Port: 3306, + }, + }, + want: &MySQLVStreamerClient{ + sourceConnParams: &mysql.ConnParams{ + Host: "testhost", + Port: 3306, + }, + }, + }, + } + for _, tcase := range tests { + t.Run(tcase.name, func(t *testing.T) { + if got := NewMySQLVStreamerClient(tcase.args.sourceConnParams); !reflect.DeepEqual(got, tcase.want) { + t.Errorf("NewMySQLVStreamerClient() = %v, want %v", got, tcase.want) + } + }) + } +} + +func TestMySQLVStreamerClientOpen(t *testing.T) { + type fields struct { + isOpen bool + sourceConnParams *mysql.ConnParams + } + type args struct { + ctx context.Context + } + tests := []struct { + name string + fields fields + args args + err string + }{ + { + name: "initializes streamer correctly", + fields: fields{ + sourceConnParams: env.Dbcfgs.ExternalReplWithDB(), + }, + args: args{ + ctx: context.Background(), + }, + }, + { + name: "returns error when invalid conn params are provided", + fields: fields{ + sourceConnParams: &mysql.ConnParams{ + Host: "invalidhost", + Port: 3306, + }, + }, + args: args{ + ctx: context.Background(), + }, + err: "failed: dial tcp: lookup invalidhost", + }, + } + for _, tcase := range tests { + t.Run(tcase.name, func(t *testing.T) { + vsClient := &MySQLVStreamerClient{ + sourceConnParams: tcase.fields.sourceConnParams, + } + + err := vsClient.Open(tcase.args.ctx) + + if err != nil { + if !strings.Contains(err.Error(), tcase.err) { + t.Errorf("MySQLVStreamerClient.Open() error:\n%v, want\n%v", err, tcase.err) + } + return + } + + if tcase.err != "" { + t.Errorf("MySQLVStreamerClient.Open() error:\n%v, want\n%v", err, tcase.err) + } + + if !vsClient.isOpen { + t.Errorf("MySQLVStreamerClient.Open() isOpen set to false, expected true") + } + + if !vsClient.sourceSe.IsOpen() { + t.Errorf("MySQLVStreamerClient.Open() expected sourceSe to be opened") + } + + if !vsClient.vsEngine.IsOpen() { + t.Errorf("MySQLVStreamerClient.Open() expected vsEngine to be opened") + } + }) + } +} + +func TestMySQLVStreamerClientClose(t *testing.T) { + type fields struct { + mu sync.Mutex + isOpen bool + sourceConnParams *mysql.ConnParams + vsEngine *vstreamer.Engine + sourceSe *schema.Engine + } + type args struct { + ctx context.Context + } + + tests := []struct { + name string + fields fields + args args + err string + }{ + { + name: "closes engine correctly", + fields: fields{ + sourceConnParams: env.Dbcfgs.ExternalReplWithDB(), + }, + args: args{ + ctx: context.Background(), + }, + }, + } + + for _, tcase := range tests { + t.Run(tcase.name, func(t *testing.T) { + vsClient := &MySQLVStreamerClient{ + isOpen: tcase.fields.isOpen, + sourceConnParams: tcase.fields.sourceConnParams, + } + + err := vsClient.Open(tcase.args.ctx) + if err != nil { + t.Errorf("Failed to Open vsClient") + return + } + + err = vsClient.Close(tcase.args.ctx) + + if tcase.err != "" { + t.Errorf("MySQLVStreamerClient.Close() error:\n%v, want\n%v", err, tcase.err) + } + + if vsClient.isOpen { + t.Errorf("MySQLVStreamerClient.Close() isOpen set to true, expected false") + } + + if vsClient.sourceSe.IsOpen() { + t.Errorf("MySQLVStreamerClient.Close() expected sourceSe to be closed") + } + + if vsClient.vsEngine.IsOpen() { + t.Errorf("MySQLVStreamerClient.Close() expected vsEngine to be closed") + } + }) + } +} + +func TestMySQLVStreamerClientVStream(t *testing.T) { + vsClient := &MySQLVStreamerClient{ + sourceConnParams: env.Dbcfgs.ExternalReplWithDB(), + } + + filter := &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "/.*", + }}, + } + eventsChan := make(chan *binlogdatapb.VEvent, 1000) + send := func(events []*binlogdatapb.VEvent) error { + fmt.Println(events) + fmt.Println(len(events)) + for _, e := range events { + eventsChan <- e + } + return nil + } + + execStatements(t, []string{ + "create table t1(id int, ts timestamp, dt datetime)", + fmt.Sprintf("create table %s.t1(id int, ts timestamp, dt datetime)", vrepldb), + }) + defer execStatements(t, []string{ + "drop table t1", + fmt.Sprintf("drop table %s.t1", vrepldb), + }) + + ctx := context.Background() + err := vsClient.Open(ctx) + if err != nil { + t.Errorf("Failed to Open vsClient") + return + } + + defer vsClient.Close(ctx) + + pos := masterPosition(t) + // This asserts that events are flowing through the VStream when using mysql client + go vsClient.VStream(ctx, pos, filter, send) + + qr, err := env.Mysqld.FetchSuperQuery(context.Background(), "select now()") + if err != nil { + t.Fatal(err) + } + want := qr.Rows[0][0].ToString() + execStatements(t, []string{ + fmt.Sprintf("insert into t1 values(1, '%s', '%s')", want, want), + }) + + select { + case got := <-eventsChan: + if got.Type != binlogdatapb.VEventType_GTID { + t.Errorf("Did not get expected events: want: %v, got: %v", binlogdatapb.VEventType_GTID, got.Type) + } + case <-time.After(5 * time.Second): + t.Errorf("no events received") + } +} + +func TestMySQLVStreamerClientVStreamRows(t *testing.T) { + vsClient := &MySQLVStreamerClient{ + sourceConnParams: env.Dbcfgs.ExternalReplWithDB(), + } + + eventsChan := make(chan *querypb.Row, 1000) + send := func(streamerResponse *binlogdatapb.VStreamRowsResponse) error { + for _, row := range streamerResponse.Rows { + eventsChan <- row + } + return nil + } + + execStatements(t, []string{ + "create table t1(id int, ts timestamp, dt datetime)", + fmt.Sprintf("create table %s.t1(id int, ts timestamp, dt datetime)", vrepldb), + }) + defer execStatements(t, []string{ + "drop table t1", + fmt.Sprintf("drop table %s.t1", vrepldb), + }) + + qr, err := env.Mysqld.FetchSuperQuery(context.Background(), "select now()") + if err != nil { + t.Fatal(err) + } + want := qr.Rows[0][0].ToString() + + ctx := context.Background() + err = vsClient.Open(ctx) + if err != nil { + t.Errorf("Failed to Open vsClient") + return + } + + defer vsClient.Close(ctx) + + // This asserts that events are flowing through the VStream when using mysql client + go vsClient.VStreamRows(ctx, "select * from t1", nil, send) + + execStatements(t, []string{ + fmt.Sprintf("insert into t1 values(1, '%s', '%s')", want, want), + }) + + select { + case <-eventsChan: + // Success got expected + case <-time.After(5 * time.Second): + t.Errorf("no events received") + } +} diff --git a/go/vt/vttablet/tabletserver/schema/engine.go b/go/vt/vttablet/tabletserver/schema/engine.go index 36811afb76c..6110839e616 100644 --- a/go/vt/vttablet/tabletserver/schema/engine.go +++ b/go/vt/vttablet/tabletserver/schema/engine.go @@ -193,6 +193,13 @@ func (se *Engine) Open() error { return nil } +// IsOpen() checks if engine is open +func (se *Engine) IsOpen() bool { + se.mu.Lock() + defer se.mu.Unlock() + return se.isOpen +} + // Close shuts down Engine and is idempotent. // It can be re-opened after Close. func (se *Engine) Close() { diff --git a/go/vt/vttablet/tabletserver/vstreamer/engine.go b/go/vt/vttablet/tabletserver/vstreamer/engine.go index 1c393a2b028..dd269a9f588 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/engine.go +++ b/go/vt/vttablet/tabletserver/vstreamer/engine.go @@ -111,6 +111,13 @@ func (vse *Engine) Open(keyspace, cell string) error { return nil } +// IsOpen checks if the engine is opened +func (vse *Engine) IsOpen() bool { + vse.mu.Lock() + defer vse.mu.Unlock() + return vse.isOpen +} + // Close closes the Engine service. func (vse *Engine) Close() { func() { From 99884961bff1e8722592c1a2686c460e1718f422 Mon Sep 17 00:00:00 2001 From: Rafael Chacon Date: Wed, 6 Nov 2019 12:56:36 -0800 Subject: [PATCH 17/28] Do not pass source conn params around * At the moment we only support erpel user. Passing source conn params around was adding unnecessary complexity. * This cleans up that and makes it more explicit that only erepl user is supported. In the future we will add more flexibility in terms of what kind of users can be configured for external vreplication streams Signed-off-by: Rafael Chacon --- go/vt/vttablet/tabletmanager/action_agent.go | 7 ++-- .../tabletmanager/vreplication/controller.go | 21 ++++++------ .../vreplication/controller_test.go | 16 +++++----- .../tabletmanager/vreplication/engine.go | 32 +++++++++---------- .../tabletmanager/vreplication/engine_test.go | 16 +++++----- .../vreplication/framework_test.go | 4 ++- .../vreplication/vstreamer_client.go | 18 +++++++++-- .../vreplication/vstreamer_client_test.go | 17 ++-------- 8 files changed, 64 insertions(+), 67 deletions(-) diff --git a/go/vt/vttablet/tabletmanager/action_agent.go b/go/vt/vttablet/tabletmanager/action_agent.go index a4b6a0536a0..58fdad00218 100644 --- a/go/vt/vttablet/tabletmanager/action_agent.go +++ b/go/vt/vttablet/tabletmanager/action_agent.go @@ -287,11 +287,12 @@ func NewActionAgent( return nil, err } + vreplication.InitVStreamerClient(agent.DBConfigs) + // The db name is set by the Start function called above agent.VREngine = vreplication.NewEngine(ts, tabletAlias.Cell, mysqld, func() binlogplayer.DBClient { return binlogplayer.NewDBClient(agent.DBConfigs.FilteredWithDB()) }, - agent.DBConfigs.ExternalReplWithDB(), agent.DBConfigs.FilteredWithDB().DbName, ) servenv.OnTerm(agent.VREngine.Close) @@ -360,7 +361,7 @@ func NewTestActionAgent(batchCtx context.Context, ts *topo.Server, tabletAlias * Cnf: nil, MysqlDaemon: mysqlDaemon, DBConfigs: &dbconfigs.DBConfigs{}, - VREngine: vreplication.NewEngine(ts, tabletAlias.Cell, mysqlDaemon, binlogplayer.NewFakeDBClient, nil, ti.DbName()), + VREngine: vreplication.NewEngine(ts, tabletAlias.Cell, mysqlDaemon, binlogplayer.NewFakeDBClient, ti.DbName()), History: history.New(historyLength), _healthy: fmt.Errorf("healthcheck not run yet"), } @@ -399,7 +400,7 @@ func NewComboActionAgent(batchCtx context.Context, ts *topo.Server, tabletAlias Cnf: nil, MysqlDaemon: mysqlDaemon, DBConfigs: dbcfgs, - VREngine: vreplication.NewEngine(nil, "", nil, nil, nil, ""), + VREngine: vreplication.NewEngine(nil, "", nil, nil, ""), gotMysqlPort: true, History: history.New(historyLength), _healthy: fmt.Errorf("healthcheck not run yet"), diff --git a/go/vt/vttablet/tabletmanager/vreplication/controller.go b/go/vt/vttablet/tabletmanager/vreplication/controller.go index 246e8e04d95..ba329696c31 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/controller.go +++ b/go/vt/vttablet/tabletmanager/vreplication/controller.go @@ -27,7 +27,6 @@ import ( "github.com/golang/protobuf/proto" "golang.org/x/net/context" - "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/sync2" "vitess.io/vitess/go/tb" "vitess.io/vitess/go/vt/binlog/binlogplayer" @@ -45,10 +44,9 @@ var retryDelay = flag.Duration("vreplication_retry_delay", 5*time.Second, "delay // There is no mutex within a controller becaust its members are // either read-only or self-synchronized. type controller struct { - dbClientFactory func() binlogplayer.DBClient - sourceDbConnParams *mysql.ConnParams - mysqld mysqlctl.MysqlDaemon - blpStats *binlogplayer.Stats + dbClientFactory func() binlogplayer.DBClient + mysqld mysqlctl.MysqlDaemon + blpStats *binlogplayer.Stats id uint32 source binlogdatapb.BinlogSource @@ -64,16 +62,15 @@ type controller struct { // newController creates a new controller. Unless a stream is explicitly 'Stopped', // this function launches a goroutine to perform continuous vreplication. -func newController(ctx context.Context, params map[string]string, dbClientFactory func() binlogplayer.DBClient, sourceDbConnParams *mysql.ConnParams, mysqld mysqlctl.MysqlDaemon, ts *topo.Server, cell, tabletTypesStr string, blpStats *binlogplayer.Stats) (*controller, error) { +func newController(ctx context.Context, params map[string]string, dbClientFactory func() binlogplayer.DBClient, mysqld mysqlctl.MysqlDaemon, ts *topo.Server, cell, tabletTypesStr string, blpStats *binlogplayer.Stats) (*controller, error) { if blpStats == nil { blpStats = binlogplayer.NewStats() } ct := &controller{ - dbClientFactory: dbClientFactory, - sourceDbConnParams: sourceDbConnParams, - mysqld: mysqld, - blpStats: blpStats, - done: make(chan struct{}), + dbClientFactory: dbClientFactory, + mysqld: mysqld, + blpStats: blpStats, + done: make(chan struct{}), } // id @@ -215,7 +212,7 @@ func (ct *controller) runBlp(ctx context.Context) (err error) { if ct.source.GetExternalMysql() == "" { vsClient = NewTabletVStreamerClient(tablet) } else { - vsClient = NewMySQLVStreamerClient(ct.sourceDbConnParams) + vsClient = NewMySQLVStreamerClient() } vreplicator := NewVReplicator(ct.id, &ct.source, vsClient, ct.blpStats, dbClient, ct.mysqld) diff --git a/go/vt/vttablet/tabletmanager/vreplication/controller_test.go b/go/vt/vttablet/tabletmanager/vreplication/controller_test.go index c0644a7f3a6..f330985c4c9 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/controller_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/controller_test.go @@ -76,7 +76,7 @@ func TestControllerKeyRange(t *testing.T) { dbClientFactory := func() binlogplayer.DBClient { return dbClient } mysqld := &fakemysqldaemon.FakeMysqlDaemon{MysqlPort: 3306} - ct, err := newController(context.Background(), params, dbClientFactory, nil, mysqld, env.TopoServ, env.Cells[0], "replica", nil) + ct, err := newController(context.Background(), params, dbClientFactory, mysqld, env.TopoServ, env.Cells[0], "replica", nil) if err != nil { t.Fatal(err) } @@ -136,7 +136,7 @@ func TestControllerTables(t *testing.T) { }, } - ct, err := newController(context.Background(), params, dbClientFactory, nil, mysqld, env.TopoServ, env.Cells[0], "replica", nil) + ct, err := newController(context.Background(), params, dbClientFactory, mysqld, env.TopoServ, env.Cells[0], "replica", nil) if err != nil { t.Fatal(err) } @@ -153,7 +153,7 @@ func TestControllerBadID(t *testing.T) { params := map[string]string{ "id": "bad", } - _, err := newController(context.Background(), params, nil, nil, nil, nil, "", "", nil) + _, err := newController(context.Background(), params, nil, nil, nil, "", "", nil) want := `strconv.Atoi: parsing "bad": invalid syntax` if err == nil || err.Error() != want { t.Errorf("newController err: %v, want %v", err, want) @@ -166,7 +166,7 @@ func TestControllerStopped(t *testing.T) { "state": binlogplayer.BlpStopped, } - ct, err := newController(context.Background(), params, nil, nil, nil, nil, "", "", nil) + ct, err := newController(context.Background(), params, nil, nil, nil, "", "", nil) if err != nil { t.Fatal(err) } @@ -203,7 +203,7 @@ func TestControllerOverrides(t *testing.T) { dbClientFactory := func() binlogplayer.DBClient { return dbClient } mysqld := &fakemysqldaemon.FakeMysqlDaemon{MysqlPort: 3306} - ct, err := newController(context.Background(), params, dbClientFactory, nil, mysqld, env.TopoServ, env.Cells[0], "rdonly", nil) + ct, err := newController(context.Background(), params, dbClientFactory, mysqld, env.TopoServ, env.Cells[0], "rdonly", nil) if err != nil { t.Fatal(err) } @@ -227,7 +227,7 @@ func TestControllerCanceledContext(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() - ct, err := newController(ctx, params, nil, nil, nil, env.TopoServ, env.Cells[0], "rdonly", nil) + ct, err := newController(ctx, params, nil, nil, env.TopoServ, env.Cells[0], "rdonly", nil) if err != nil { t.Fatal(err) } @@ -269,7 +269,7 @@ func TestControllerRetry(t *testing.T) { dbClientFactory := func() binlogplayer.DBClient { return dbClient } mysqld := &fakemysqldaemon.FakeMysqlDaemon{MysqlPort: 3306} - ct, err := newController(context.Background(), params, dbClientFactory, nil, mysqld, env.TopoServ, env.Cells[0], "rdonly", nil) + ct, err := newController(context.Background(), params, dbClientFactory, mysqld, env.TopoServ, env.Cells[0], "rdonly", nil) if err != nil { t.Fatal(err) } @@ -315,7 +315,7 @@ func TestControllerStopPosition(t *testing.T) { dbClientFactory := func() binlogplayer.DBClient { return dbClient } mysqld := &fakemysqldaemon.FakeMysqlDaemon{MysqlPort: 3306} - ct, err := newController(context.Background(), params, dbClientFactory, nil, mysqld, env.TopoServ, env.Cells[0], "replica", nil) + ct, err := newController(context.Background(), params, dbClientFactory, mysqld, env.TopoServ, env.Cells[0], "replica", nil) if err != nil { t.Fatal(err) } diff --git a/go/vt/vttablet/tabletmanager/vreplication/engine.go b/go/vt/vttablet/tabletmanager/vreplication/engine.go index 2b72a33ea4e..4baad2ac62c 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/engine.go +++ b/go/vt/vttablet/tabletmanager/vreplication/engine.go @@ -77,25 +77,23 @@ type Engine struct { // cancel will cancel the root context, thereby all controllers. cancel context.CancelFunc - ts *topo.Server - cell string - mysqld mysqlctl.MysqlDaemon - dbClientFactory func() binlogplayer.DBClient - sourceDbConnParams *mysql.ConnParams - dbName string + ts *topo.Server + cell string + mysqld mysqlctl.MysqlDaemon + dbClientFactory func() binlogplayer.DBClient + dbName string } // NewEngine creates a new Engine. // A nil ts means that the Engine is disabled. -func NewEngine(ts *topo.Server, cell string, mysqld mysqlctl.MysqlDaemon, dbClientFactory func() binlogplayer.DBClient, sourceDbConnParams *mysql.ConnParams, dbName string) *Engine { +func NewEngine(ts *topo.Server, cell string, mysqld mysqlctl.MysqlDaemon, dbClientFactory func() binlogplayer.DBClient, dbName string) *Engine { vre := &Engine{ - controllers: make(map[int]*controller), - ts: ts, - cell: cell, - mysqld: mysqld, - dbClientFactory: dbClientFactory, - sourceDbConnParams: sourceDbConnParams, - dbName: dbName, + controllers: make(map[int]*controller), + ts: ts, + cell: cell, + mysqld: mysqld, + dbClientFactory: dbClientFactory, + dbName: dbName, } return vre } @@ -189,7 +187,7 @@ func (vre *Engine) initAll() error { return err } for _, row := range rows { - ct, err := newController(vre.ctx, row, vre.dbClientFactory, vre.sourceDbConnParams, vre.mysqld, vre.ts, vre.cell, *tabletTypesStr, nil) + ct, err := newController(vre.ctx, row, vre.dbClientFactory, vre.mysqld, vre.ts, vre.cell, *tabletTypesStr, nil) if err != nil { return err } @@ -282,7 +280,7 @@ func (vre *Engine) Exec(query string) (*sqltypes.Result, error) { if err != nil { return nil, err } - ct, err := newController(vre.ctx, params, vre.dbClientFactory, vre.sourceDbConnParams, vre.mysqld, vre.ts, vre.cell, *tabletTypesStr, nil) + ct, err := newController(vre.ctx, params, vre.dbClientFactory, vre.mysqld, vre.ts, vre.cell, *tabletTypesStr, nil) if err != nil { return nil, err } @@ -320,7 +318,7 @@ func (vre *Engine) Exec(query string) (*sqltypes.Result, error) { } // Create a new controller in place of the old one. // For continuity, the new controller inherits the previous stats. - ct, err := newController(vre.ctx, params, vre.dbClientFactory, vre.sourceDbConnParams, vre.mysqld, vre.ts, vre.cell, *tabletTypesStr, blpStats[id]) + ct, err := newController(vre.ctx, params, vre.dbClientFactory, vre.mysqld, vre.ts, vre.cell, *tabletTypesStr, blpStats[id]) if err != nil { return nil, err } diff --git a/go/vt/vttablet/tabletmanager/vreplication/engine_test.go b/go/vt/vttablet/tabletmanager/vreplication/engine_test.go index 8e7cff1a880..d16ca35cc4b 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/engine_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/engine_test.go @@ -41,7 +41,7 @@ func TestEngineOpen(t *testing.T) { // Test Insert - vre := NewEngine(env.TopoServ, env.Cells[0], mysqld, dbClientFactory, nil, dbClient.DBName()) + vre := NewEngine(env.TopoServ, env.Cells[0], mysqld, dbClientFactory, dbClient.DBName()) if vre.IsOpen() { t.Errorf("IsOpen: %v, want false", vre.IsOpen()) } @@ -89,7 +89,7 @@ func TestEngineExec(t *testing.T) { // Test Insert - vre := NewEngine(env.TopoServ, env.Cells[0], mysqld, dbClientFactory, nil, dbClient.DBName()) + vre := NewEngine(env.TopoServ, env.Cells[0], mysqld, dbClientFactory, dbClient.DBName()) dbClient.ExpectRequest("select * from _vt.vreplication where db_name='db'", &sqltypes.Result{}, nil) if err := vre.Open(context.Background()); err != nil { @@ -249,7 +249,7 @@ func TestEngineBadInsert(t *testing.T) { dbClientFactory := func() binlogplayer.DBClient { return dbClient } mysqld := &fakemysqldaemon.FakeMysqlDaemon{MysqlPort: 3306} - vre := NewEngine(env.TopoServ, env.Cells[0], mysqld, dbClientFactory, nil, dbClient.DBName()) + vre := NewEngine(env.TopoServ, env.Cells[0], mysqld, dbClientFactory, dbClient.DBName()) dbClient.ExpectRequest("select * from _vt.vreplication where db_name='db'", &sqltypes.Result{}, nil) if err := vre.Open(context.Background()); err != nil { @@ -279,7 +279,7 @@ func TestEngineSelect(t *testing.T) { dbClientFactory := func() binlogplayer.DBClient { return dbClient } mysqld := &fakemysqldaemon.FakeMysqlDaemon{MysqlPort: 3306} - vre := NewEngine(env.TopoServ, env.Cells[0], mysqld, dbClientFactory, nil, dbClient.DBName()) + vre := NewEngine(env.TopoServ, env.Cells[0], mysqld, dbClientFactory, dbClient.DBName()) dbClient.ExpectRequest("select * from _vt.vreplication where db_name='db'", &sqltypes.Result{}, nil) if err := vre.Open(context.Background()); err != nil { @@ -314,7 +314,7 @@ func TestWaitForPos(t *testing.T) { dbClient := binlogplayer.NewMockDBClient(t) mysqld := &fakemysqldaemon.FakeMysqlDaemon{MysqlPort: 3306} dbClientFactory := func() binlogplayer.DBClient { return dbClient } - vre := NewEngine(env.TopoServ, env.Cells[0], mysqld, dbClientFactory, nil, dbClient.DBName()) + vre := NewEngine(env.TopoServ, env.Cells[0], mysqld, dbClientFactory, dbClient.DBName()) dbClient.ExpectRequest("select * from _vt.vreplication where db_name='db'", &sqltypes.Result{}, nil) if err := vre.Open(context.Background()); err != nil { @@ -344,7 +344,7 @@ func TestWaitForPosError(t *testing.T) { dbClient := binlogplayer.NewMockDBClient(t) mysqld := &fakemysqldaemon.FakeMysqlDaemon{MysqlPort: 3306} dbClientFactory := func() binlogplayer.DBClient { return dbClient } - vre := NewEngine(env.TopoServ, env.Cells[0], mysqld, dbClientFactory, nil, dbClient.DBName()) + vre := NewEngine(env.TopoServ, env.Cells[0], mysqld, dbClientFactory, dbClient.DBName()) err := vre.WaitForPos(context.Background(), 1, "MariaDB/0-1-1084") want := `vreplication engine is closed` @@ -386,7 +386,7 @@ func TestWaitForPosCancel(t *testing.T) { dbClient := binlogplayer.NewMockDBClient(t) mysqld := &fakemysqldaemon.FakeMysqlDaemon{MysqlPort: 3306} dbClientFactory := func() binlogplayer.DBClient { return dbClient } - vre := NewEngine(env.TopoServ, env.Cells[0], mysqld, dbClientFactory, nil, dbClient.DBName()) + vre := NewEngine(env.TopoServ, env.Cells[0], mysqld, dbClientFactory, dbClient.DBName()) dbClient.ExpectRequest("select * from _vt.vreplication where db_name='db'", &sqltypes.Result{}, nil) if err := vre.Open(context.Background()); err != nil { @@ -433,7 +433,7 @@ func TestCreateDBAndTable(t *testing.T) { // Test Insert - vre := NewEngine(env.TopoServ, env.Cells[0], mysqld, dbClientFactory, nil, dbClient.DBName()) + vre := NewEngine(env.TopoServ, env.Cells[0], mysqld, dbClientFactory, dbClient.DBName()) tableNotFound := mysql.SQLError{Num: 1146, Message: "table not found"} dbClient.ExpectRequest("select * from _vt.vreplication where db_name='db'", nil, &tableNotFound) diff --git a/go/vt/vttablet/tabletmanager/vreplication/framework_test.go b/go/vt/vttablet/tabletmanager/vreplication/framework_test.go index 27f78253294..6ec1e08b608 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/framework_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/framework_test.go @@ -96,7 +96,9 @@ func TestMain(m *testing.M) { return 1 } - playerEngine = NewEngine(env.TopoServ, env.Cells[0], env.Mysqld, realDBClientFactory, env.Dbcfgs.ExternalReplWithDB(), vrepldb) + InitVStreamerClient(env.Dbcfgs) + + playerEngine = NewEngine(env.TopoServ, env.Cells[0], env.Mysqld, realDBClientFactory, vrepldb) if err := playerEngine.Open(context.Background()); err != nil { fmt.Fprintf(os.Stderr, "%v", err) return 1 diff --git a/go/vt/vttablet/tabletmanager/vreplication/vstreamer_client.go b/go/vt/vttablet/tabletmanager/vreplication/vstreamer_client.go index cc7e0f19963..2e9f5ae393f 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vstreamer_client.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vstreamer_client.go @@ -25,6 +25,7 @@ import ( "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/vt/dbconfigs" "vitess.io/vitess/go/vt/grpcclient" "vitess.io/vitess/go/vt/srvtopo" "vitess.io/vitess/go/vt/topo/memorytopo" @@ -45,6 +46,7 @@ var ( _ VStreamerClient = (*MySQLVStreamerClient)(nil) mysqlStreamerClientOnce sync.Once mysqlSrvTopo *srvtopo.ResilientServer + dbcfgs *dbconfigs.DBConfigs ) // VStreamerClient exposes the core interface of a vstreamer @@ -139,11 +141,15 @@ func (vsClient *TabletVStreamerClient) VStreamRows(ctx context.Context, query st // NewMySQLVStreamerClient is a vstream client that allows you to stream directly from MySQL. // In order to achieve this, the following creates a vstreamer Engine with a dummy in memorytopo. -func NewMySQLVStreamerClient(sourceConnParams *mysql.ConnParams) *MySQLVStreamerClient { +func NewMySQLVStreamerClient() *MySQLVStreamerClient { + if dbcfgs == nil { + panic("can't use MySQLVStreamerClient without calling InitVStreamerClient() ") + } + // TODO: For now external mysql streams can only be used with ExternalReplWithDB creds. + // In the future we will support multiple users. vsClient := &MySQLVStreamerClient{ - sourceConnParams: sourceConnParams, + sourceConnParams: dbcfgs.ExternalReplWithDB(), } - return vsClient } @@ -221,6 +227,12 @@ func (vsClient *MySQLVStreamerClient) VStreamRows(ctx context.Context, query str return vsClient.vsEngine.StreamRows(ctx, query, row, send) } +func InitVStreamerClient(cfg *dbconfigs.DBConfigs) { + // Make copy of config + dbcfgs = &dbconfigs.DBConfigs{} + *dbcfgs = *cfg +} + type checker struct{} var _ = connpool.MySQLChecker(checker{}) diff --git a/go/vt/vttablet/tabletmanager/vreplication/vstreamer_client_test.go b/go/vt/vttablet/tabletmanager/vreplication/vstreamer_client_test.go index 0e26f26fc44..27e4da55935 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vstreamer_client_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vstreamer_client_test.go @@ -272,33 +272,20 @@ func TestTabletVStreamerClientVStreamRows(t *testing.T) { } func TestNewMySQLVStreamerClient(t *testing.T) { - type args struct { - sourceConnParams *mysql.ConnParams - } tests := []struct { name string - args args want *MySQLVStreamerClient }{ { name: "sets conn params for MySQLVStreamerClient ", - args: args{ - sourceConnParams: &mysql.ConnParams{ - Host: "testhost", - Port: 3306, - }, - }, want: &MySQLVStreamerClient{ - sourceConnParams: &mysql.ConnParams{ - Host: "testhost", - Port: 3306, - }, + sourceConnParams: env.Dbcfgs.ExternalReplWithDB(), }, }, } for _, tcase := range tests { t.Run(tcase.name, func(t *testing.T) { - if got := NewMySQLVStreamerClient(tcase.args.sourceConnParams); !reflect.DeepEqual(got, tcase.want) { + if got := NewMySQLVStreamerClient(); !reflect.DeepEqual(got, tcase.want) { t.Errorf("NewMySQLVStreamerClient() = %v, want %v", got, tcase.want) } }) From 4bd4904e83cafdfc2752ecf55218b00d66312d57 Mon Sep 17 00:00:00 2001 From: Rafael Chacon Date: Wed, 6 Nov 2019 13:46:27 -0800 Subject: [PATCH 18/28] Style improvements * Fix typo in some comments. * Make VReplicator private again. This change is no longer needed. Originally we wanted "vtshovel" to be an external process. Given that this now hooks into the existent engine, there is no need to make this public. Signed-off-by: Rafael Chacon --- go/mysql/flavor.go | 2 +- go/vt/binlog/binlogplayer/binlog_player.go | 2 +- .../tabletmanager/vreplication/controller.go | 4 ++-- .../tabletmanager/vreplication/vcopier.go | 4 ++-- .../tabletmanager/vreplication/vplayer.go | 4 ++-- .../tabletmanager/vreplication/vreplicator.go | 20 +++++++++---------- .../tabletserver/vstreamer/vstreamer.go | 4 ++-- 7 files changed, 20 insertions(+), 20 deletions(-) diff --git a/go/mysql/flavor.go b/go/mysql/flavor.go index fd53d84babb..18c085333f5 100644 --- a/go/mysql/flavor.go +++ b/go/mysql/flavor.go @@ -62,7 +62,7 @@ type flavor interface { // stopSlave returns the command to stop the slave. stopSlaveCommand() string - // sendBinlogFileDumpCommand sends the packet required to start streaming from file:post + // sendBinlogFileDumpCommand sends the packet required to start streaming from file:pos sendBinlogFileDumpCommand(c *Conn, slaveID uint32, binlogFilename string, pos uint32) error // sendBinlogDumpCommand sends the packet required to start diff --git a/go/vt/binlog/binlogplayer/binlog_player.go b/go/vt/binlog/binlogplayer/binlog_player.go index 74792c3b871..1a7eef3a9f6 100644 --- a/go/vt/binlog/binlogplayer/binlog_player.go +++ b/go/vt/binlog/binlogplayer/binlog_player.go @@ -551,7 +551,7 @@ func ReadVRSettings(dbClient DBClient, uid uint32) (VRSettings, error) { return VRSettings{}, fmt.Errorf("failed to parse max_replication_lag column: %v", err) } startPos := vrRow[0].ToString() - // TODO: This will be removed when we start using filename:pos flavor and everythign will by a proper enconded mysql.Position + // TODO @rafael: This will be removed when we start using the non_gtid_flavor. In that case filename:pos flavor will be handled by the flavor with pseudo gtids. There won't be any need to have different kind of mysql positions. gtidStartPos, _ := mysql.DecodePosition(startPos) stopPos, err := mysql.DecodePosition(vrRow[1].ToString()) diff --git a/go/vt/vttablet/tabletmanager/vreplication/controller.go b/go/vt/vttablet/tabletmanager/vreplication/controller.go index ba329696c31..c1cd3ca475f 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/controller.go +++ b/go/vt/vttablet/tabletmanager/vreplication/controller.go @@ -215,8 +215,8 @@ func (ct *controller) runBlp(ctx context.Context) (err error) { vsClient = NewMySQLVStreamerClient() } - vreplicator := NewVReplicator(ct.id, &ct.source, vsClient, ct.blpStats, dbClient, ct.mysqld) - return vreplicator.Replicate(ctx) + vr := newVReplicator(ct.id, &ct.source, vsClient, ct.blpStats, dbClient, ct.mysqld) + return vr.Replicate(ctx) } return fmt.Errorf("missing source") } diff --git a/go/vt/vttablet/tabletmanager/vreplication/vcopier.go b/go/vt/vttablet/tabletmanager/vreplication/vcopier.go index 1785404d29e..2bf53348039 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vcopier.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vcopier.go @@ -38,11 +38,11 @@ import ( ) type vcopier struct { - vr *VReplicator + vr *vreplicator tablePlan *TablePlan } -func newVCopier(vr *VReplicator) *vcopier { +func newVCopier(vr *vreplicator) *vcopier { return &vcopier{ vr: vr, } diff --git a/go/vt/vttablet/tabletmanager/vreplication/vplayer.go b/go/vt/vttablet/tabletmanager/vreplication/vplayer.go index 4e5d6872190..331f800d2a9 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vplayer.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vplayer.go @@ -36,7 +36,7 @@ import ( ) type vplayer struct { - vr *VReplicator + vr *vreplicator startPos string gtidStartPos mysql.Position stopPos mysql.Position @@ -59,7 +59,7 @@ type vplayer struct { timeOffsetNs int64 } -func newVPlayer(vr *VReplicator, settings binlogplayer.VRSettings, copyState map[string]*sqltypes.Result, pausePos mysql.Position) *vplayer { +func newVPlayer(vr *vreplicator, settings binlogplayer.VRSettings, copyState map[string]*sqltypes.Result, pausePos mysql.Position) *vplayer { saveStop := true if !pausePos.IsZero() { settings.StopPos = pausePos diff --git a/go/vt/vttablet/tabletmanager/vreplication/vreplicator.go b/go/vt/vttablet/tabletmanager/vreplication/vreplicator.go index d64a87f1edd..996338a9182 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vreplicator.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vreplicator.go @@ -44,8 +44,8 @@ var ( replicaLagTolerance = 10 * time.Second ) -// VReplicator provides the core logic to start vreplication streams -type VReplicator struct { +// vreplicator provides the core logic to start vreplication streams +type vreplicator struct { id uint32 dbClient *vdbClient // source @@ -65,9 +65,9 @@ type SchemasLoader interface { GetSchema(dbName string, tables, excludeTables []string, includeViews bool) (*tabletmanagerdatapb.SchemaDefinition, error) } -// NewVReplicator creates a new vreplicator -func NewVReplicator(id uint32, source *binlogdatapb.BinlogSource, sourceVStreamer VStreamerClient, stats *binlogplayer.Stats, dbClient binlogplayer.DBClient, sl SchemasLoader) *VReplicator { - return &VReplicator{ +// newVReplicator creates a new vreplicator +func newVReplicator(id uint32, source *binlogdatapb.BinlogSource, sourceVStreamer VStreamerClient, stats *binlogplayer.Stats, dbClient binlogplayer.DBClient, sl SchemasLoader) *vreplicator { + return &vreplicator{ id: id, source: source, sourceVStreamer: sourceVStreamer, @@ -78,7 +78,7 @@ func NewVReplicator(id uint32, source *binlogdatapb.BinlogSource, sourceVStreame } // Replicate starts a vreplication stream. -func (vr *VReplicator) Replicate(ctx context.Context) error { +func (vr *vreplicator) Replicate(ctx context.Context) error { tableKeys, err := vr.buildTableKeys() if err != nil { return err @@ -117,7 +117,7 @@ func (vr *VReplicator) Replicate(ctx context.Context) error { } } -func (vr *VReplicator) buildTableKeys() (map[string][]string, error) { +func (vr *vreplicator) buildTableKeys() (map[string][]string, error) { schema, err := vr.sl.GetSchema(vr.dbClient.DBName(), []string{"/.*/"}, nil, false) if err != nil { return nil, err @@ -133,7 +133,7 @@ func (vr *VReplicator) buildTableKeys() (map[string][]string, error) { return tableKeys, nil } -func (vr *VReplicator) readSettings(ctx context.Context) (settings binlogplayer.VRSettings, numTablesToCopy int64, err error) { +func (vr *vreplicator) readSettings(ctx context.Context) (settings binlogplayer.VRSettings, numTablesToCopy int64, err error) { settings, err = binlogplayer.ReadVRSettings(vr.dbClient, vr.id) if err != nil { return settings, numTablesToCopy, fmt.Errorf("error reading VReplication settings: %v", err) @@ -168,7 +168,7 @@ func (vr *VReplicator) readSettings(ctx context.Context) (settings binlogplayer. return settings, numTablesToCopy, nil } -func (vr *VReplicator) setMessage(message string) error { +func (vr *vreplicator) setMessage(message string) error { vr.stats.History.Add(&binlogplayer.StatsHistoryRecord{ Time: time.Now(), Message: message, @@ -180,7 +180,7 @@ func (vr *VReplicator) setMessage(message string) error { return nil } -func (vr *VReplicator) setState(state, message string) error { +func (vr *vreplicator) setState(state, message string) error { return binlogplayer.SetVReplicationState(vr.dbClient, vr.id, state, message) } diff --git a/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go b/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go index b9c73a92fba..73467b84197 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go +++ b/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go @@ -315,8 +315,8 @@ func (vs *vstreamer) parseEvent(ev mysql.BinlogEvent) ([]*binlogdatapb.VEvent, e if err != nil { return nil, fmt.Errorf("can't get query from binlog event: %v, event data: %#v", err, ev) } - // Insert/Delete/Update are supported only to be used in the context of vtshovel where source databases - // could be using SBR. Vitess itself should never run into cases where it needs to consume non rbr statements. + // Insert/Delete/Update are supported only to be used in the context of external mysql streams where source databases + // could be using SBR. Vitess itself will never run into cases where it needs to consume non rbr statements. switch cat := sqlparser.Preview(q.SQL); cat { case sqlparser.StmtInsert: mustSend := mustSendStmt(q, vs.cp.DbName) From 21eeeb45eb1a3885c984861b2bca3c02d2228b82 Mon Sep 17 00:00:00 2001 From: Rafael Chacon Date: Wed, 13 Nov 2019 16:24:20 -0800 Subject: [PATCH 19/28] Abort on error when executing a DML in statement based replication Signed-off-by: Rafael Chacon --- go/vt/vttablet/tabletmanager/vreplication/vplayer.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/go/vt/vttablet/tabletmanager/vreplication/vplayer.go b/go/vt/vttablet/tabletmanager/vreplication/vplayer.go index 331f800d2a9..d78e8ca863a 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vplayer.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vplayer.go @@ -178,10 +178,8 @@ func (vp *vplayer) applyStmtEvent(ctx context.Context, event *binlogdatapb.VEven } } - if _, err := vp.vr.dbClient.ExecuteWithRetry(ctx, event.Dml); err != nil { - log.Warningf("Fail to run: %v. Got error: %v", event.Dml, err) - } - return nil + _, err := vp.vr.dbClient.ExecuteWithRetry(ctx, event.Dml) + return err } func (vp *vplayer) applyRowEvent(ctx context.Context, rowEvent *binlogdatapb.RowEvent) error { From 166570176d29c5890153cc429744b35d1edb7350 Mon Sep 17 00:00:00 2001 From: Rafael Chacon Date: Mon, 25 Nov 2019 15:14:50 -0800 Subject: [PATCH 20/28] Fixes per rebase with file:pos feature Signed-off-by: Rafael Chacon --- go/mysql/flavor.go | 10 -- go/mysql/flavor_mariadb.go | 5 - go/mysql/flavor_mysql.go | 5 - go/mysql/replication_position.go | 29 ----- go/mysql/replication_position_test.go | 42 ------- go/vt/binlog/binlogplayer/binlog_player.go | 16 +-- go/vt/binlog/slave_connection.go | 20 --- .../tabletmanager/vreplication/controller.go | 2 +- .../vreplication/framework_test.go | 16 +++ .../tabletmanager/vreplication/vcopier.go | 4 +- .../tabletmanager/vreplication/vplayer.go | 21 ++-- .../vreplication/vplayer_test.go | 115 ------------------ .../tabletmanager/vreplication/vreplicator.go | 24 ++-- .../vreplication/vstreamer_client_test.go | 16 +-- .../tabletserver/vstreamer/vstreamer.go | 10 -- 15 files changed, 50 insertions(+), 285 deletions(-) diff --git a/go/mysql/flavor.go b/go/mysql/flavor.go index c220587470c..001ef89df7b 100644 --- a/go/mysql/flavor.go +++ b/go/mysql/flavor.go @@ -62,9 +62,6 @@ type flavor interface { // stopSlave returns the command to stop the slave. stopSlaveCommand() string - // sendBinlogFileDumpCommand sends the packet required to start streaming from file:pos - sendBinlogFileDumpCommand(c *Conn, slaveID uint32, binlogFilename string, pos uint32) error - // sendBinlogDumpCommand sends the packet required to start // dumping binlogs from the specified location. sendBinlogDumpCommand(c *Conn, slaveID uint32, startPos Position) error @@ -178,13 +175,6 @@ func (c *Conn) StopSlaveCommand() string { return c.flavor.stopSlaveCommand() } -// SendBinlogFileDumpCommand sends the flavor-specific version of -// the COM_BINLOG_DUMP command to start dumping raw binlog -// events over a slave connection, starting at a given file position. -func (c *Conn) SendBinlogFileDumpCommand(slaveID uint32, binlogFilename string, pos uint32) error { - return c.flavor.sendBinlogFileDumpCommand(c, slaveID, binlogFilename, pos) -} - // SendBinlogDumpCommand sends the flavor-specific version of // the COM_BINLOG_DUMP command to start dumping raw binlog // events over a slave connection, starting at a given GTID. diff --git a/go/mysql/flavor_mariadb.go b/go/mysql/flavor_mariadb.go index 97ced86e5f2..98174aa9b48 100644 --- a/go/mysql/flavor_mariadb.go +++ b/go/mysql/flavor_mariadb.go @@ -55,11 +55,6 @@ func (mariadbFlavor) stopSlaveCommand() string { return "STOP SLAVE" } -// sendBinlogFileDumpCommand is part of the Flavor interface. -func (mariadbFlavor) sendBinlogFileDumpCommand(c *Conn, slaveID uint32, binlogFilename string, pos uint32) error { - panic("filename binglog not supported for mariadb") -} - // sendBinlogDumpCommand is part of the Flavor interface. func (mariadbFlavor) sendBinlogDumpCommand(c *Conn, slaveID uint32, startPos Position) error { // Tell the server that we understand GTIDs by setting our slave diff --git a/go/mysql/flavor_mysql.go b/go/mysql/flavor_mysql.go index d642bcbe584..6ef3a34eb38 100644 --- a/go/mysql/flavor_mysql.go +++ b/go/mysql/flavor_mysql.go @@ -53,11 +53,6 @@ func (mysqlFlavor) stopSlaveCommand() string { return "STOP SLAVE" } -// sendBinlogDumpCommand is part of the Flavor interface. -func (mysqlFlavor) sendBinlogFileDumpCommand(c *Conn, slaveID uint32, binlogFilename string, pos uint32) error { - return c.WriteComBinlogDump(slaveID, binlogFilename, pos, 0) -} - // sendBinlogDumpCommand is part of the Flavor interface. func (mysqlFlavor) sendBinlogDumpCommand(c *Conn, slaveID uint32, startPos Position) error { gtidSet, ok := startPos.GTIDSet.(Mysql56GTIDSet) diff --git a/go/mysql/replication_position.go b/go/mysql/replication_position.go index f0ab2c732cd..a103d22af95 100644 --- a/go/mysql/replication_position.go +++ b/go/mysql/replication_position.go @@ -19,7 +19,6 @@ package mysql import ( "encoding/json" "fmt" - "strconv" "strings" "vitess.io/vitess/go/vt/proto/vtrpc" @@ -36,12 +35,6 @@ const ( MaximumPositionSize = 64000 ) -// BinlogFilePos used to encode filename:pos. -type BinlogFilePos struct { - Name string - Pos uint32 -} - // Position represents the information necessary to describe which // transactions a server has seen, so that it can request a replication stream // from a new master that picks up where it left off. @@ -127,28 +120,6 @@ func EncodePosition(rp Position) string { return fmt.Sprintf("%s/%s", rp.GTIDSet.Flavor(), rp.GTIDSet.String()) } -// ParseFilePosition converts a string in the format file:pos -// to BinlogFilePos -func ParseFilePosition(s string) (rp BinlogFilePos, err error) { - if s == "" { - return rp, vterrors.Errorf(vtrpc.Code_INTERNAL, "parse error: unknown file:pos format %#v", s) - } - - parts := strings.SplitN(s, ":", 2) - if len(parts) != 2 { - return rp, vterrors.Errorf(vtrpc.Code_INTERNAL, "parse error: unknown file:pos format %#v", s) - } - - pos, err := strconv.Atoi(parts[1]) - if err != nil { - return rp, vterrors.Errorf(vtrpc.Code_INTERNAL, "parse error: pos is not a valid int %#v", s) - } - - rp.Name = parts[0] - rp.Pos = uint32(pos) - return rp, nil -} - // DecodePosition converts a string in the format returned by // EncodePosition back into a Position value with the // correct underlying flavor. diff --git a/go/mysql/replication_position_test.go b/go/mysql/replication_position_test.go index 35ee0a8d67c..7162810af34 100644 --- a/go/mysql/replication_position_test.go +++ b/go/mysql/replication_position_test.go @@ -208,48 +208,6 @@ func TestPositionAppendToZero(t *testing.T) { } } -func TestParseFilePositionInvalidInput(t *testing.T) { - input := "filenameinvalidpos" - rp, err := ParseFilePosition(input) - if err == nil { - t.Errorf("ParseFilePosition(%#v) expected error, got : %#v", input, rp) - } - - want := `parse error: unknown file:pos format` - got, ok := err.(error) - if !ok || !strings.HasPrefix(got.Error(), want) { - t.Errorf("wrong error, got %#v, want %#v", got, want) - } -} - -func TestParseFilePositionInvalidPos(t *testing.T) { - input := "filename:invalidpos" - rp, err := ParseFilePosition(input) - if err == nil { - t.Errorf("ParseFilePosition(%#v) expected error, got : %#v", input, rp) - } - - want := `parse error: pos is not a valid` - got, ok := err.(error) - if !ok || !strings.HasPrefix(got.Error(), want) { - t.Errorf("wrong error, got %#v, want %#v", got, want) - } -} - -func TestParseFilePosition(t *testing.T) { - input := "filename:2343" - want := BinlogFilePos{Name: "filename", Pos: 2343} - got, err := ParseFilePosition(input) - if err != nil { - t.Errorf("ParseFilePosition(%#v) unexpected error: %#v", input, err) - } - - if got.Name != want.Name || got.Pos != want.Pos { - t.Errorf("ParseFilePosition(%#v) = %#v, want %#v", input, got, want) - } - -} - func TestMustParsePosition(t *testing.T) { flavor := "fake flavor" gtidSetParsers[flavor] = func(s string) (GTIDSet, error) { diff --git a/go/vt/binlog/binlogplayer/binlog_player.go b/go/vt/binlog/binlogplayer/binlog_player.go index 1acc68ab2d5..fea1e002a45 100644 --- a/go/vt/binlog/binlogplayer/binlog_player.go +++ b/go/vt/binlog/binlogplayer/binlog_player.go @@ -203,9 +203,7 @@ func (blp *BinlogPlayer) applyEvents(ctx context.Context) error { return err } - if !settings.GtidStartPos.IsZero() { - blp.position = settings.GtidStartPos - } + blp.position = settings.StartPos blp.stopPosition = settings.StopPos t, err := throttler.NewThrottler( fmt.Sprintf("BinlogPlayer/%d", blp.uid), @@ -520,12 +518,11 @@ func SetVReplicationState(dbClient DBClient, uid uint32, state, message string) // VRSettings contains the settings of a vreplication table. type VRSettings struct { - StartPos string + StartPos mysql.Position StopPos mysql.Position MaxTPS int64 MaxReplicationLag int64 State string - GtidStartPos mysql.Position } // ReadVRSettings retrieves the throttler settings for @@ -550,10 +547,10 @@ func ReadVRSettings(dbClient DBClient, uid uint32) (VRSettings, error) { if err != nil { return VRSettings{}, fmt.Errorf("failed to parse max_replication_lag column: %v", err) } - startPos := vrRow[0].ToString() - // TODO @rafael: This will be removed when we start using the non_gtid_flavor. In that case filename:pos flavor will be handled by the flavor with pseudo gtids. There won't be any need to have different kind of mysql positions. - gtidStartPos, _ := mysql.DecodePosition(startPos) - + startPos, err := mysql.DecodePosition(vrRow[0].ToString()) + if err != nil { + return VRSettings{}, fmt.Errorf("failed to parse pos column: %v", err) + } stopPos, err := mysql.DecodePosition(vrRow[1].ToString()) if err != nil { return VRSettings{}, fmt.Errorf("failed to parse stop_pos column: %v", err) @@ -561,7 +558,6 @@ func ReadVRSettings(dbClient DBClient, uid uint32) (VRSettings, error) { return VRSettings{ StartPos: startPos, - GtidStartPos: gtidStartPos, StopPos: stopPos, MaxTPS: maxTPS, MaxReplicationLag: maxReplicationLag, diff --git a/go/vt/binlog/slave_connection.go b/go/vt/binlog/slave_connection.go index 3a3196d9570..c38e67af859 100644 --- a/go/vt/binlog/slave_connection.go +++ b/go/vt/binlog/slave_connection.go @@ -127,26 +127,6 @@ func (sc *SlaveConnection) StartBinlogDumpFromPosition(ctx context.Context, star return sc.streamEvents(ctx), nil } -// StartBinlogDumpFromFilePosition requests a replication binlog dump from -// the master mysqld at the given binlog filename:pos and then sends binlog -// events to the provided channel. -// The stream will continue in the background, waiting for new events if -// necessary, until the connection is closed, either by the master or -// by canceling the context. -// -// Note the context is valid and used until eventChan is closed. -func (sc *SlaveConnection) StartBinlogDumpFromFilePosition(ctx context.Context, binlogFilename string, pos uint32) (<-chan mysql.BinlogEvent, error) { - ctx, sc.cancel = context.WithCancel(ctx) - - log.Infof("sending binlog file dump command: binlogfilename=%v, pos=%v, slaveID=%v", binlogFilename, pos, sc.slaveID) - if err := sc.SendBinlogFileDumpCommand(sc.slaveID, binlogFilename, pos); err != nil { - log.Errorf("couldn't send binlog dump command: %v", err) - return nil, err - } - - return sc.streamEvents(ctx), nil -} - // streamEvents returns a channel on which events are streamed. func (sc *SlaveConnection) streamEvents(ctx context.Context) chan mysql.BinlogEvent { // FIXME(alainjobart) I think we can use a buffered channel for better performance. diff --git a/go/vt/vttablet/tabletmanager/vreplication/controller.go b/go/vt/vttablet/tabletmanager/vreplication/controller.go index 1a23c34c96d..6284f7fe2ce 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/controller.go +++ b/go/vt/vttablet/tabletmanager/vreplication/controller.go @@ -182,7 +182,7 @@ func (ct *controller) runBlp(ctx context.Context) (err error) { var tablet *topodatapb.Tablet if ct.source.GetExternalMysql() == "" { - tablet, err := ct.tabletPicker.PickForStreaming(ctx) + tablet, err = ct.tabletPicker.PickForStreaming(ctx) if err != nil { return err } diff --git a/go/vt/vttablet/tabletmanager/vreplication/framework_test.go b/go/vt/vttablet/tabletmanager/vreplication/framework_test.go index 8683eae08dc..6bd26b63f85 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/framework_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/framework_test.go @@ -124,6 +124,22 @@ func resetBinlogClient() { globalFBC = &fakeBinlogClient{} } +func masterPosition(t *testing.T) string { + t.Helper() + pos, err := env.Mysqld.MasterPosition() + if err != nil { + t.Fatal(err) + } + return mysql.EncodePosition(pos) +} + +func execStatements(t *testing.T, queries []string) { + t.Helper() + if err := env.Mysqld.ExecuteSuperQueryList(context.Background(), queries); err != nil { + t.Error(err) + } +} + //-------------------------------------- // Topos and tablets diff --git a/go/vt/vttablet/tabletmanager/vreplication/vcopier.go b/go/vt/vttablet/tabletmanager/vreplication/vcopier.go index 2bf53348039..b663efe6e03 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vcopier.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vcopier.go @@ -122,7 +122,7 @@ func (vc *vcopier) catchup(ctx context.Context, copyState map[string]*sqltypes.R } // If there's no start position, it means we're copying the // first table. So, there's nothing to catch up to. - if settings.GtidStartPos.IsZero() { + if settings.StartPos.IsZero() { return nil } @@ -288,7 +288,7 @@ func (vc *vcopier) fastForward(ctx context.Context, copyState map[string]*sqltyp if err != nil { return err } - if settings.GtidStartPos.IsZero() { + if settings.StartPos.IsZero() { update := binlogplayer.GenerateUpdatePos(vc.vr.id, pos, time.Now().Unix(), 0) _, err := vc.vr.dbClient.Execute(update) return err diff --git a/go/vt/vttablet/tabletmanager/vreplication/vplayer.go b/go/vt/vttablet/tabletmanager/vreplication/vplayer.go index 9df7bfc6638..e9e4679df79 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vplayer.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vplayer.go @@ -36,13 +36,11 @@ import ( ) type vplayer struct { - vr *vreplicator - startPos string - gtidStartPos mysql.Position - stopPos mysql.Position - startBinlogFilePos *mysql.BinlogFilePos - saveStop bool - copyState map[string]*sqltypes.Result + vr *vreplicator + startPos mysql.Position + stopPos mysql.Position + saveStop bool + copyState map[string]*sqltypes.Result replicatorPlan *ReplicatorPlan tablePlans map[string]*TablePlan @@ -68,8 +66,7 @@ func newVPlayer(vr *vreplicator, settings binlogplayer.VRSettings, copyState map return &vplayer{ vr: vr, startPos: settings.StartPos, - gtidStartPos: settings.GtidStartPos, - pos: settings.GtidStartPos, + pos: settings.StartPos, stopPos: settings.StopPos, saveStop: saveStop, copyState: copyState, @@ -80,9 +77,9 @@ func newVPlayer(vr *vreplicator, settings binlogplayer.VRSettings, copyState map // play is not resumable. If pausePos is set, play returns without updating the vreplication state. func (vp *vplayer) play(ctx context.Context) error { - if !vp.stopPos.IsZero() && vp.gtidStartPos.AtLeast(vp.stopPos) { + if !vp.stopPos.IsZero() && vp.startPos.AtLeast(vp.stopPos) { if vp.saveStop { - return vp.vr.setState(binlogplayer.BlpStopped, fmt.Sprintf("Stop position %v already reached: %v", vp.gtidStartPos, vp.stopPos)) + return vp.vr.setState(binlogplayer.BlpStopped, fmt.Sprintf("Stop position %v already reached: %v", vp.startPos, vp.stopPos)) } return nil } @@ -123,7 +120,7 @@ func (vp *vplayer) fetchAndApply(ctx context.Context) (err error) { streamErr := make(chan error, 1) go func() { - streamErr <- vp.vr.sourceVStreamer.VStream(ctx, vp.startPos, vp.replicatorPlan.VStreamFilter, func(events []*binlogdatapb.VEvent) error { + streamErr <- vp.vr.sourceVStreamer.VStream(ctx, mysql.EncodePosition(vp.startPos), vp.replicatorPlan.VStreamFilter, func(events []*binlogdatapb.VEvent) error { return relay.Send(events) }) }() diff --git a/go/vt/vttablet/tabletmanager/vreplication/vplayer_test.go b/go/vt/vttablet/tabletmanager/vreplication/vplayer_test.go index 40440ec7880..8e97468166b 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vplayer_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vplayer_test.go @@ -26,111 +26,12 @@ import ( "golang.org/x/net/context" - "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/binlog/binlogplayer" binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" ) -func TestMySQLVstreamerClient(t *testing.T) { - execStatements(t, []string{ - "create table src1(id int, val varbinary(128), primary key(id))", - fmt.Sprintf("create table %s.dst1(id int, val varbinary(128), primary key(id))", vrepldb), - "create table src2(id int, val1 int, val2 int, primary key(id))", - fmt.Sprintf("create table %s.dst2(id int, val1 int, sval2 int, rcount int, primary key(id))", vrepldb), - "create table src3(id int, val varbinary(128), primary key(id))", - fmt.Sprintf("create table %s.dst3(id int, val varbinary(128), primary key(id))", vrepldb), - "create table yes(id int, val varbinary(128), primary key(id))", - fmt.Sprintf("create table %s.yes(id int, val varbinary(128), primary key(id))", vrepldb), - "create table no(id int, val varbinary(128), primary key(id))", - "create table nopk(id int, val varbinary(128))", - fmt.Sprintf("create table %s.nopk(id int, val varbinary(128))", vrepldb), - }) - defer execStatements(t, []string{ - "drop table src1", - fmt.Sprintf("drop table %s.dst1", vrepldb), - "drop table src2", - fmt.Sprintf("drop table %s.dst2", vrepldb), - "drop table src3", - fmt.Sprintf("drop table %s.dst3", vrepldb), - "drop table yes", - fmt.Sprintf("drop table %s.yes", vrepldb), - "drop table no", - "drop table nopk", - fmt.Sprintf("drop table %s.nopk", vrepldb), - }) - env.SchemaEngine.Reload(context.Background()) - - filter := &binlogdatapb.Filter{ - Rules: []*binlogdatapb.Rule{{ - Match: "dst1", - Filter: "select * from src1", - }, { - Match: "dst2", - Filter: "select id, val1, sum(val2) as sval2, count(*) as rcount from src2 group by id", - }, { - Match: "dst3", - Filter: "select id, val from src3 group by id, val", - }, { - Match: "/yes", - }, { - Match: "/nopk", - }}, - } - - bls := &binlogdatapb.BinlogSource{ - Filter: filter, - OnDdl: binlogdatapb.OnDDLAction_IGNORE, - ExternalMysql: "erepl", - } - - cancel, _ := startVReplication(t, bls, "") - defer cancel() - - testcases := []struct { - input string - output []string - table string - data [][]string - }{{ - // insert with insertNormal - input: "insert into src1 values(1, 'aaa')", - output: []string{ - "begin", - "insert into dst1(id,val) values (1,'aaa')", - "/update _vt.vreplication set pos=", - "commit", - }, - table: "dst1", - data: [][]string{ - {"1", "aaa"}, - }, - }, { - // update with insertNormal - input: "update src1 set val='bbb'", - output: []string{ - "begin", - "update dst1 set val='bbb' where id=1", - "/update _vt.vreplication set pos=", - "commit", - }, - table: "dst1", - data: [][]string{ - {"1", "bbb"}, - }, - }} - - for _, tcases := range testcases { - execStatements(t, []string{tcases.input}) - expectDBClientQueries(t, tcases.output) - if tcases.table != "" { - expectData(t, tcases.table, tcases.data) - } - } - -} - func TestPlayerFilters(t *testing.T) { defer deleteTablet(addTablet(100)) @@ -1717,13 +1618,6 @@ func TestTimestamp(t *testing.T) { expectData(t, "t1", [][]string{{"1", want, want}}) } -func execStatements(t *testing.T, queries []string) { - t.Helper() - if err := env.Mysqld.ExecuteSuperQueryList(context.Background(), queries); err != nil { - t.Error(err) - } -} - func startVReplication(t *testing.T, bls *binlogdatapb.BinlogSource, pos string) (cancelFunc func(), id int) { t.Helper() @@ -1752,12 +1646,3 @@ func startVReplication(t *testing.T, bls *binlogdatapb.BinlogSource, pos string) }) }, int(qr.InsertID) } - -func masterPosition(t *testing.T) string { - t.Helper() - pos, err := env.Mysqld.MasterPosition() - if err != nil { - t.Fatal(err) - } - return mysql.EncodePosition(pos) -} diff --git a/go/vt/vttablet/tabletmanager/vreplication/vreplicator.go b/go/vt/vttablet/tabletmanager/vreplication/vreplicator.go index 996338a9182..2dc86792fc7 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vreplicator.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vreplicator.go @@ -27,9 +27,9 @@ import ( "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/binlog/binlogplayer" "vitess.io/vitess/go/vt/log" + "vitess.io/vitess/go/vt/mysqlctl" binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" - tabletmanagerdatapb "vitess.io/vitess/go/vt/proto/tabletmanagerdata" ) var ( @@ -54,26 +54,22 @@ type vreplicator struct { // target stats *binlogplayer.Stats - // sl is used to fetch the local schema. - sl SchemasLoader - tableKeys map[string][]string -} + // mysqld is used to fetch the local schema. + mysqld mysqlctl.MysqlDaemon -// SchemasLoader provides a way to load schemas for a vreplicator -type SchemasLoader interface { - GetSchema(dbName string, tables, excludeTables []string, includeViews bool) (*tabletmanagerdatapb.SchemaDefinition, error) + tableKeys map[string][]string } // newVReplicator creates a new vreplicator -func newVReplicator(id uint32, source *binlogdatapb.BinlogSource, sourceVStreamer VStreamerClient, stats *binlogplayer.Stats, dbClient binlogplayer.DBClient, sl SchemasLoader) *vreplicator { +func newVReplicator(id uint32, source *binlogdatapb.BinlogSource, sourceVStreamer VStreamerClient, stats *binlogplayer.Stats, dbClient binlogplayer.DBClient, mysqld mysqlctl.MysqlDaemon) *vreplicator { return &vreplicator{ id: id, source: source, sourceVStreamer: sourceVStreamer, stats: stats, dbClient: newVDBClient(dbClient, stats), - sl: sl, + mysqld: mysqld, } } @@ -95,16 +91,12 @@ func (vr *vreplicator) Replicate(ctx context.Context) error { return nil } - // TODO: This will get remove once we use filename:pos flavor - _, err = mysql.ParseFilePosition(settings.StartPos) - isFilePos := err == nil - switch { case numTablesToCopy != 0: if err := newVCopier(vr).copyNext(ctx, settings); err != nil { return err } - case settings.GtidStartPos.IsZero() && !isFilePos: + case settings.StartPos.IsZero(): if err := newVCopier(vr).initTablesForCopy(ctx); err != nil { return err } @@ -118,7 +110,7 @@ func (vr *vreplicator) Replicate(ctx context.Context) error { } func (vr *vreplicator) buildTableKeys() (map[string][]string, error) { - schema, err := vr.sl.GetSchema(vr.dbClient.DBName(), []string{"/.*/"}, nil, false) + schema, err := vr.mysqld.GetSchema(vr.dbClient.DBName(), []string{"/.*/"}, nil, false) if err != nil { return nil, err } diff --git a/go/vt/vttablet/tabletmanager/vreplication/vstreamer_client_test.go b/go/vt/vttablet/tabletmanager/vreplication/vstreamer_client_test.go index 27e4da55935..ea5d2c499fa 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vstreamer_client_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vstreamer_client_test.go @@ -37,7 +37,7 @@ import ( ) func TestTabletVStreamerClientOpen(t *testing.T) { - tablet := addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true) + tablet := addTablet(100) defer deleteTablet(tablet) type fields struct { @@ -98,7 +98,7 @@ func TestTabletVStreamerClientOpen(t *testing.T) { } func TestTabletVStreamerClientClose(t *testing.T) { - tablet := addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true) + tablet := addTablet(100) defer deleteTablet(tablet) type fields struct { @@ -154,7 +154,7 @@ func TestTabletVStreamerClientClose(t *testing.T) { } func TestTabletVStreamerClientVStream(t *testing.T) { - tablet := addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true) + tablet := addTablet(100) defer deleteTablet(tablet) vsClient := &TabletVStreamerClient{ @@ -209,8 +209,8 @@ func TestTabletVStreamerClientVStream(t *testing.T) { select { case got := <-eventsChan: - if got.Type != binlogdatapb.VEventType_GTID { - t.Errorf("Did not get expected events: want: %v, got: %v", binlogdatapb.VEventType_GTID, got.Type) + if got.Type != binlogdatapb.VEventType_BEGIN { + t.Errorf("Did not get expected events: want: %v, got: %v", binlogdatapb.VEventType_BEGIN, got.Type) } case <-time.After(5 * time.Second): t.Errorf("no events received") @@ -218,7 +218,7 @@ func TestTabletVStreamerClientVStream(t *testing.T) { } func TestTabletVStreamerClientVStreamRows(t *testing.T) { - tablet := addTablet(100, "0", topodatapb.TabletType_REPLICA, true, true) + tablet := addTablet(100) defer deleteTablet(tablet) vsClient := &TabletVStreamerClient{ @@ -479,8 +479,8 @@ func TestMySQLVStreamerClientVStream(t *testing.T) { select { case got := <-eventsChan: - if got.Type != binlogdatapb.VEventType_GTID { - t.Errorf("Did not get expected events: want: %v, got: %v", binlogdatapb.VEventType_GTID, got.Type) + if got.Type != binlogdatapb.VEventType_BEGIN { + t.Errorf("Did not get expected events: want: %v, got: %v", binlogdatapb.VEventType_BEGIN, got.Type) } case <-time.After(5 * time.Second): t.Errorf("no events received") diff --git a/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go b/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go index fb8eaa37f45..167fef2e720 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go +++ b/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go @@ -115,16 +115,6 @@ func (vs *vstreamer) Stream() error { } defer conn.Close() - // TODO: This case logic depending on startPos will disappear when filename:pos flavor is introduced - filePos, err := mysql.ParseFilePosition(vs.startPos) - if err == nil { - events, err := conn.StartBinlogDumpFromFilePosition(vs.ctx, filePos.Name, filePos.Pos) - if err != nil { - return wrapError(err, vs.startPos) - } - err = vs.parseEvents(vs.ctx, events) - return wrapError(err, vs.startPos) - } // Let's try to decode as gtidset pos, err := mysql.DecodePosition(vs.startPos) if err != nil { From 731f02ff9a218f7204976d70069462727e1f7fd7 Mon Sep 17 00:00:00 2001 From: Rafael Chacon Date: Tue, 26 Nov 2019 14:10:15 -0800 Subject: [PATCH 21/28] Revert no longer needed changes in vstream Signed-off-by: Rafael Chacon --- .../tabletserver/vstreamer/testenv/testenv.go | 2 +- .../tabletserver/vstreamer/vstreamer.go | 23 +++++++++---------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/go/vt/vttablet/tabletserver/vstreamer/testenv/testenv.go b/go/vt/vttablet/tabletserver/vstreamer/testenv/testenv.go index b43ac316af8..599cf3304e1 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/testenv/testenv.go +++ b/go/vt/vttablet/tabletserver/vstreamer/testenv/testenv.go @@ -106,7 +106,7 @@ func Init() (*Env, error) { te.Dbcfgs = dbconfigs.NewTestDBConfigs(te.cluster.MySQLConnParams(), te.cluster.MySQLAppDebugConnParams(), te.cluster.DbName()) te.Mysqld = mysqlctl.NewMysqld(te.Dbcfgs) te.SchemaEngine = schema.NewEngine(checker{}, tabletenv.DefaultQsConfig) - te.SchemaEngine.InitDBConfig(te.Dbcfgs.Dba()) + te.SchemaEngine.InitDBConfig(te.Dbcfgs.DbaWithDB()) // The first vschema should not be empty. Leads to Node not found error. // TODO(sougou): need to fix the bug. diff --git a/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go b/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go index 167fef2e720..a7d526d32cb 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go +++ b/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go @@ -103,31 +103,30 @@ func (vs *vstreamer) Cancel() { func (vs *vstreamer) Stream() error { defer vs.cancel() + pos, err := mysql.DecodePosition(vs.startPos) + if err != nil { + return err + } + vs.pos = pos + // Ensure se is Open. If vttablet came up in a non_serving role, // the schema engine may not have been initialized. if err := vs.se.Open(); err != nil { - return wrapError(err, vs.startPos) + return wrapError(err, vs.pos) } conn, err := binlog.NewSlaveConnection(vs.cp) if err != nil { - return wrapError(err, vs.startPos) + return wrapError(err, vs.pos) } defer conn.Close() - // Let's try to decode as gtidset - pos, err := mysql.DecodePosition(vs.startPos) - if err != nil { - return wrapError(err, vs.startPos) - } - - vs.pos = pos events, err := conn.StartBinlogDumpFromPosition(vs.ctx, vs.pos) if err != nil { - return wrapError(err, vs.startPos) + return wrapError(err, vs.pos) } err = vs.parseEvents(vs.ctx, events) - return wrapError(err, vs.pos.String()) + return wrapError(err, vs.pos) } func (vs *vstreamer) parseEvents(ctx context.Context, events <-chan mysql.BinlogEvent) error { @@ -558,7 +557,7 @@ func (vs *vstreamer) extractRowAndFilter(plan *streamerPlan, data []byte, dataCo return plan.filter(values) } -func wrapError(err error, stopPos string) error { +func wrapError(err error, stopPos mysql.Position) error { if err != nil { err = fmt.Errorf("stream error @ %v: %v", stopPos, err) log.Error(err) From ec632f206111f6c95fc190a2ab84585c2d508863 Mon Sep 17 00:00:00 2001 From: Rafael Chacon Date: Tue, 26 Nov 2019 16:08:42 -0800 Subject: [PATCH 22/28] Adds support to set flavor for specific connections Signed-off-by: Rafael Chacon --- go/vt/dbconfigs/dbconfigs.go | 1 + 1 file changed, 1 insertion(+) diff --git a/go/vt/dbconfigs/dbconfigs.go b/go/vt/dbconfigs/dbconfigs.go index a1636c0b387..87e0b074243 100644 --- a/go/vt/dbconfigs/dbconfigs.go +++ b/go/vt/dbconfigs/dbconfigs.go @@ -129,6 +129,7 @@ func registerPerUserFlags(dbc *userConfig, userKey string) { flag.StringVar(&dbc.param.SslCert, "db-config-"+userKey+"-ssl-cert", "", "deprecated: use db_ssl_cert") flag.StringVar(&dbc.param.SslKey, "db-config-"+userKey+"-ssl-key", "", "deprecated: use db_ssl_key") flag.StringVar(&dbc.param.ServerName, "db-config-"+userKey+"-server_name", "", "deprecated: use db_server_name") + flag.StringVar(&dbc.param.Flavor, "db-config-"+userKey+"-flavor", "", "deprecated: use db_flavor") flag.StringVar(&dbc.param.DeprecatedDBName, "db-config-"+userKey+"-dbname", "", "deprecated: dbname does not need to be explicitly configured") From d4460ee333056ced1f80cf08b8b55a47127b537b Mon Sep 17 00:00:00 2001 From: Rafael Chacon Date: Wed, 27 Nov 2019 11:15:34 -0800 Subject: [PATCH 23/28] Fixes per integration with file:pos rebase Signed-off-by: Rafael Chacon --- go/vt/dbconfigs/dbconfigs.go | 4 +- .../tabletmanager/vreplication/vplayer.go | 6 +-- .../vreplication/vstreamer_client.go | 37 +++++-------------- .../vreplication/vstreamer_client_test.go | 8 ---- .../vttablet/tabletserver/vstreamer/engine.go | 4 +- .../tabletserver/vstreamer/rowstreamer.go | 2 +- .../tabletserver/vstreamer/vstreamer.go | 2 +- 7 files changed, 17 insertions(+), 46 deletions(-) diff --git a/go/vt/dbconfigs/dbconfigs.go b/go/vt/dbconfigs/dbconfigs.go index 87e0b074243..c9df1ab26c8 100644 --- a/go/vt/dbconfigs/dbconfigs.go +++ b/go/vt/dbconfigs/dbconfigs.go @@ -277,7 +277,9 @@ func Init(defaultSocketFile string) (*DBConfigs, error) { if baseConfig.Flags != 0 { uc.param.Flags = baseConfig.Flags } - uc.param.Flavor = baseConfig.Flavor + if user != ExternalRepl { + uc.param.Flavor = baseConfig.Flavor + } if uc.useSSL { uc.param.SslCa = baseConfig.SslCa uc.param.SslCaPath = baseConfig.SslCaPath diff --git a/go/vt/vttablet/tabletmanager/vreplication/vplayer.go b/go/vt/vttablet/tabletmanager/vreplication/vplayer.go index e9e4679df79..ce9429eb623 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vplayer.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vplayer.go @@ -30,8 +30,6 @@ import ( "vitess.io/vitess/go/vt/binlog/binlogplayer" "vitess.io/vitess/go/vt/log" - // "vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer" - binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" ) @@ -138,8 +136,6 @@ func (vp *vplayer) fetchAndApply(ctx context.Context) (err error) { <-streamErr }() - log.Infof("error applying events: %v", err) - // If the apply thread ends with io.EOF, it means either the Engine // is shutting down and canceled the context, or stop position was reached. // If so, we return nil which will cause the controller to not retry. @@ -354,7 +350,7 @@ func (vp *vplayer) applyEvent(ctx context.Context, event *binlogdatapb.VEvent, m return err } case binlogdatapb.VEventType_ROW: - // This player is configured for row based replicaiton + // This player is configured for row based replication if err := vp.vr.dbClient.Begin(); err != nil { return err } diff --git a/go/vt/vttablet/tabletmanager/vreplication/vstreamer_client.go b/go/vt/vttablet/tabletmanager/vreplication/vstreamer_client.go index 2e9f5ae393f..746697f469c 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vstreamer_client.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vstreamer_client.go @@ -27,8 +27,7 @@ import ( "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/dbconfigs" "vitess.io/vitess/go/vt/grpcclient" - "vitess.io/vitess/go/vt/srvtopo" - "vitess.io/vitess/go/vt/topo/memorytopo" + "vitess.io/vitess/go/vt/vtgate/vindexes" "vitess.io/vitess/go/vt/vttablet/queryservice" "vitess.io/vitess/go/vt/vttablet/tabletconn" "vitess.io/vitess/go/vt/vttablet/tabletserver/connpool" @@ -42,11 +41,9 @@ import ( ) var ( - _ VStreamerClient = (*TabletVStreamerClient)(nil) - _ VStreamerClient = (*MySQLVStreamerClient)(nil) - mysqlStreamerClientOnce sync.Once - mysqlSrvTopo *srvtopo.ResilientServer - dbcfgs *dbconfigs.DBConfigs + _ VStreamerClient = (*TabletVStreamerClient)(nil) + _ VStreamerClient = (*MySQLVStreamerClient)(nil) + dbcfgs *dbconfigs.DBConfigs ) // VStreamerClient exposes the core interface of a vstreamer @@ -83,7 +80,6 @@ type MySQLVStreamerClient struct { isOpen bool sourceConnParams *mysql.ConnParams - vsEngine *vstreamer.Engine sourceSe *schema.Engine } @@ -162,12 +158,7 @@ func (vsClient *MySQLVStreamerClient) Open(ctx context.Context) (err error) { } vsClient.isOpen = true - mysqlStreamerClientOnce.Do(func() { - memorytopo := memorytopo.NewServer("mysqlstreamer") - mysqlSrvTopo = srvtopo.NewResilientServer(memorytopo, "") - }) - - // Let's create all the required components by vstreamer.Engine + // Let's create all the required components by vstreamer vsClient.sourceSe = schema.NewEngine(checker{}, tabletenv.DefaultQsConfig) vsClient.sourceSe.InitDBConfig(vsClient.sourceConnParams) @@ -175,17 +166,6 @@ func (vsClient *MySQLVStreamerClient) Open(ctx context.Context) (err error) { if err != nil { return err } - - vsClient.vsEngine = vstreamer.NewEngine(mysqlSrvTopo, vsClient.sourceSe) - vsClient.vsEngine.InitDBConfig(vsClient.sourceConnParams) - - // We don't really need a keyspace/cell as this is a dummy engine from the - // topology perspective - err = vsClient.vsEngine.Open("", "") - if err != nil { - return err - } - return nil } @@ -198,7 +178,6 @@ func (vsClient *MySQLVStreamerClient) Close(ctx context.Context) (err error) { } vsClient.isOpen = false - vsClient.vsEngine.Close() vsClient.sourceSe.Close() return nil } @@ -208,7 +187,8 @@ func (vsClient *MySQLVStreamerClient) VStream(ctx context.Context, startPos stri if !vsClient.isOpen { return errors.New("Can't VStream without opening client") } - return vsClient.vsEngine.Stream(ctx, startPos, filter, send) + streamer := vstreamer.NewVStreamer(ctx, vsClient.sourceConnParams, vsClient.sourceSe, startPos, filter, &vindexes.KeyspaceSchema{}, send) + return streamer.Stream() } // VStreamRows part of the VStreamerClient interface @@ -224,7 +204,8 @@ func (vsClient *MySQLVStreamerClient) VStreamRows(ctx context.Context, query str } row = r.Rows[0] } - return vsClient.vsEngine.StreamRows(ctx, query, row, send) + streamer := vstreamer.NewRowStreamer(ctx, vsClient.sourceConnParams, vsClient.sourceSe, query, row, &vindexes.KeyspaceSchema{}, send) + return streamer.Stream() } func InitVStreamerClient(cfg *dbconfigs.DBConfigs) { diff --git a/go/vt/vttablet/tabletmanager/vreplication/vstreamer_client_test.go b/go/vt/vttablet/tabletmanager/vreplication/vstreamer_client_test.go index ea5d2c499fa..ab2e19d1779 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vstreamer_client_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vstreamer_client_test.go @@ -355,10 +355,6 @@ func TestMySQLVStreamerClientOpen(t *testing.T) { if !vsClient.sourceSe.IsOpen() { t.Errorf("MySQLVStreamerClient.Open() expected sourceSe to be opened") } - - if !vsClient.vsEngine.IsOpen() { - t.Errorf("MySQLVStreamerClient.Open() expected vsEngine to be opened") - } }) } } @@ -418,10 +414,6 @@ func TestMySQLVStreamerClientClose(t *testing.T) { if vsClient.sourceSe.IsOpen() { t.Errorf("MySQLVStreamerClient.Close() expected sourceSe to be closed") } - - if vsClient.vsEngine.IsOpen() { - t.Errorf("MySQLVStreamerClient.Close() expected vsEngine to be closed") - } }) } } diff --git a/go/vt/vttablet/tabletserver/vstreamer/engine.go b/go/vt/vttablet/tabletserver/vstreamer/engine.go index 32424480f75..19b54c572d6 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/engine.go +++ b/go/vt/vttablet/tabletserver/vstreamer/engine.go @@ -166,7 +166,7 @@ func (vse *Engine) Stream(ctx context.Context, startPos string, filter *binlogda if !vse.isOpen { return nil, 0, errors.New("VStreamer is not open") } - streamer := newVStreamer(ctx, vse.cp, vse.se, startPos, filter, vse.kschema, send) + streamer := NewVStreamer(ctx, vse.cp, vse.se, startPos, filter, vse.kschema, send) idx := vse.streamIdx vse.streamers[idx] = streamer vse.streamIdx++ @@ -206,7 +206,7 @@ func (vse *Engine) StreamRows(ctx context.Context, query string, lastpk []sqltyp if !vse.isOpen { return nil, 0, errors.New("VStreamer is not open") } - rowStreamer := newRowStreamer(ctx, vse.cp, vse.se, query, lastpk, vse.kschema, send) + rowStreamer := NewRowStreamer(ctx, vse.cp, vse.se, query, lastpk, vse.kschema, send) idx := vse.streamIdx vse.rowStreamers[idx] = rowStreamer vse.streamIdx++ diff --git a/go/vt/vttablet/tabletserver/vstreamer/rowstreamer.go b/go/vt/vttablet/tabletserver/vstreamer/rowstreamer.go index c4114d451c0..3bdf0f2bdab 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/rowstreamer.go +++ b/go/vt/vttablet/tabletserver/vstreamer/rowstreamer.go @@ -48,7 +48,7 @@ type rowStreamer struct { sendQuery string } -func newRowStreamer(ctx context.Context, cp *mysql.ConnParams, se *schema.Engine, query string, lastpk []sqltypes.Value, kschema *vindexes.KeyspaceSchema, send func(*binlogdatapb.VStreamRowsResponse) error) *rowStreamer { +func NewRowStreamer(ctx context.Context, cp *mysql.ConnParams, se *schema.Engine, query string, lastpk []sqltypes.Value, kschema *vindexes.KeyspaceSchema, send func(*binlogdatapb.VStreamRowsResponse) error) *rowStreamer { ctx, cancel := context.WithCancel(ctx) return &rowStreamer{ ctx: ctx, diff --git a/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go b/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go index a7d526d32cb..79b31004dd6 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go +++ b/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go @@ -69,7 +69,7 @@ type streamerPlan struct { TableMap *mysql.TableMap } -func newVStreamer(ctx context.Context, cp *mysql.ConnParams, se *schema.Engine, startPos string, filter *binlogdatapb.Filter, kschema *vindexes.KeyspaceSchema, send func([]*binlogdatapb.VEvent) error) *vstreamer { +func NewVStreamer(ctx context.Context, cp *mysql.ConnParams, se *schema.Engine, startPos string, filter *binlogdatapb.Filter, kschema *vindexes.KeyspaceSchema, send func([]*binlogdatapb.VEvent) error) *vstreamer { ctx, cancel := context.WithCancel(ctx) return &vstreamer{ ctx: ctx, From 4f03489c95282640deee5a8a77b0655733176b3c Mon Sep 17 00:00:00 2001 From: Rafael Chacon Date: Wed, 27 Nov 2019 11:23:59 -0800 Subject: [PATCH 24/28] Remove test to make sure this is the last outstanding issue Signed-off-by: Rafael Chacon --- .../tabletserver/vstreamer/vstreamer_test.go | 39 ------------------- 1 file changed, 39 deletions(-) diff --git a/go/vt/vttablet/tabletserver/vstreamer/vstreamer_test.go b/go/vt/vttablet/tabletserver/vstreamer/vstreamer_test.go index dac50c5c62f..68ed41157e7 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/vstreamer_test.go +++ b/go/vt/vttablet/tabletserver/vstreamer/vstreamer_test.go @@ -852,45 +852,6 @@ func TestMinimalMode(t *testing.T) { } } -func TestStatementMode(t *testing.T) { - if testing.Short() { - t.Skip() - } - - execStatements(t, []string{ - "create table t1(id int, val1 varbinary(128), val2 varbinary(128), primary key(id))", - "insert into t1 values(1, 'aaa', 'bbb')", - }) - defer execStatements(t, []string{ - "drop table t1", - }) - engine.se.Reload(context.Background()) - - // Record position before the next few statements. - pos := masterPosition(t) - execStatements(t, []string{ - "set @@session.binlog_format='statement'", - "update t1 set val1='bbb' where id=1", - "set @@session.binlog_format='row'", - }) - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - ch := make(chan []*binlogdatapb.VEvent) - go func() { - for evs := range ch { - t.Errorf("received: %v", evs) - } - }() - defer close(ch) - err := vstream(ctx, t, pos, nil, ch) - want := "unexpected statement type" - if err == nil || !strings.Contains(err.Error(), want) { - t.Errorf("err: %v, must contain '%s'", err, want) - } -} - func runCases(t *testing.T, filter *binlogdatapb.Filter, testcases []testcase, postion string) { t.Helper() ctx, cancel := context.WithCancel(context.Background()) From b1a8772d8e36a18bf3de237ef4de0aee47b385cb Mon Sep 17 00:00:00 2001 From: Rafael Chacon Date: Wed, 27 Nov 2019 16:08:04 -0800 Subject: [PATCH 25/28] Fixes bug in filepos and adds test for statement mode * StripChecksum was changing the type of the event. This was a bug. * Adds test to vstreamer to reflect new support for statement based replication Signed-off-by: Rafael Chacon --- go/mysql/binlog_event_filepos.go | 14 ++++--- .../tabletserver/vstreamer/vstreamer_test.go | 38 +++++++++++++++++++ 2 files changed, 46 insertions(+), 6 deletions(-) diff --git a/go/mysql/binlog_event_filepos.go b/go/mysql/binlog_event_filepos.go index 5080a323691..9b2b6e1cef8 100644 --- a/go/mysql/binlog_event_filepos.go +++ b/go/mysql/binlog_event_filepos.go @@ -106,9 +106,11 @@ func (ev filePosQueryEvent) Query(BinlogFormat) (Query, error) { }, nil } -//---------------------------------------------------------------------------- +func (ev filePosQueryEvent) StripChecksum(f BinlogFormat) (BinlogEvent, []byte, error) { + return ev, nil, nil +} -var _ BinlogEvent = filePosFakeEvent{} +//---------------------------------------------------------------------------- // filePosFakeEvent is the base class for fake events. type filePosFakeEvent struct { @@ -207,10 +209,6 @@ func (ev filePosFakeEvent) Rows(BinlogFormat, *TableMap) (Rows, error) { return Rows{}, nil } -func (ev filePosFakeEvent) StripChecksum(f BinlogFormat) (BinlogEvent, []byte, error) { - return ev, nil, nil -} - func (ev filePosFakeEvent) IsPseudo() bool { return false } @@ -239,6 +237,10 @@ func (ev filePosGTIDEvent) IsGTID() bool { return true } +func (ev filePosGTIDEvent) StripChecksum(f BinlogFormat) (BinlogEvent, []byte, error) { + return ev, nil, nil +} + func (ev filePosGTIDEvent) GTID(BinlogFormat) (GTID, bool, error) { return ev.gtid, false, nil } diff --git a/go/vt/vttablet/tabletserver/vstreamer/vstreamer_test.go b/go/vt/vttablet/tabletserver/vstreamer/vstreamer_test.go index 68ed41157e7..8471db27eb5 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/vstreamer_test.go +++ b/go/vt/vttablet/tabletserver/vstreamer/vstreamer_test.go @@ -852,6 +852,44 @@ func TestMinimalMode(t *testing.T) { } } +func TestStatementMode(t *testing.T) { + if testing.Short() { + t.Skip() + } + execStatements(t, []string{ + "create table stream1(id int, val varbinary(128), primary key(id))", + "create table stream2(id int, val varbinary(128), primary key(id))", + }) + + engine.se.Reload(context.Background()) + + defer execStatements(t, []string{ + "drop table stream1", + "drop table stream2", + }) + + testcases := []testcase{{ + input: []string{ + "set @@session.binlog_format='STATEMENT'", + "begin", + "insert into stream1 values (1, 'aaa')", + "update stream1 set val='bbb' where id = 1", + "delete from stream1 where id = 1", + "commit", + "set @@session.binlog_format='ROW'", + }, + output: [][]string{{ + `begin`, + `type:INSERT dml:"insert into stream1 values (1, 'aaa')" `, + `type:UPDATE dml:"update stream1 set val='bbb' where id = 1" `, + `type:DELETE dml:"delete from stream1 where id = 1" `, + `gtid`, + `commit`, + }}, + }} + runCases(t, nil, testcases, "") +} + func runCases(t *testing.T, filter *binlogdatapb.Filter, testcases []testcase, postion string) { t.Helper() ctx, cancel := context.WithCancel(context.Background()) From 59785b6cbba3cd719f1349585b264d23f93c7afd Mon Sep 17 00:00:00 2001 From: Rafael Chacon Date: Wed, 27 Nov 2019 17:15:52 -0800 Subject: [PATCH 26/28] Fixes per changes upstream Signed-off-by: Rafael Chacon --- .../tabletmanager/vreplication/controller.go | 2 +- .../vreplication/journal_test.go | 43 ++++++++++++++++--- .../vreplication/vstreamer_client_test.go | 5 +++ 3 files changed, 44 insertions(+), 6 deletions(-) diff --git a/go/vt/vttablet/tabletmanager/vreplication/controller.go b/go/vt/vttablet/tabletmanager/vreplication/controller.go index 07514f11035..3df3b9369c1 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/controller.go +++ b/go/vt/vttablet/tabletmanager/vreplication/controller.go @@ -108,7 +108,7 @@ func newController(ctx context.Context, params map[string]string, dbClientFactor if v, ok := params["cell"]; ok { cell = v } - if v, ok := params["tablet_types"]; ok { + if v := params["tablet_types"]; v != "" { tabletTypesStr = v } tp, err := discovery.NewTabletPicker(ctx, ts, cell, ct.source.Keyspace, ct.source.Shard, tabletTypesStr, *healthcheckTopologyRefresh, *healthcheckRetryDelay, *healthcheckTimeout) diff --git a/go/vt/vttablet/tabletmanager/vreplication/journal_test.go b/go/vt/vttablet/tabletmanager/vreplication/journal_test.go index fc29f6d1ba0..32402c1fdb4 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/journal_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/journal_test.go @@ -43,7 +43,14 @@ func TestJournalOneToOne(t *testing.T) { Match: "t", }}, } - _, firstID := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") + bls := &binlogdatapb.BinlogSource{ + Keyspace: env.KeyspaceName, + Shard: env.ShardName, + Filter: filter, + OnDdl: binlogdatapb.OnDDLAction_IGNORE, + } + + _, firstID := startVReplication(t, bls, "") journal := &binlogdatapb.Journal{ Id: 1, @@ -98,7 +105,14 @@ func TestJournalOneToMany(t *testing.T) { Match: "t", }}, } - _, firstID := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") + bls := &binlogdatapb.BinlogSource{ + Keyspace: env.KeyspaceName, + Shard: env.ShardName, + Filter: filter, + OnDdl: binlogdatapb.OnDDLAction_IGNORE, + } + + _, firstID := startVReplication(t, bls, "") journal := &binlogdatapb.Journal{ Id: 1, @@ -158,7 +172,13 @@ func TestJournalTablePresent(t *testing.T) { Match: "t", }}, } - _, firstID := startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") + bls := &binlogdatapb.BinlogSource{ + Keyspace: env.KeyspaceName, + Shard: env.ShardName, + Filter: filter, + OnDdl: binlogdatapb.OnDDLAction_IGNORE, + } + _, firstID := startVReplication(t, bls, "") journal := &binlogdatapb.Journal{ Id: 1, @@ -213,7 +233,14 @@ func TestJournalTableNotPresent(t *testing.T) { Match: "t", }}, } - _, _ = startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") + bls := &binlogdatapb.BinlogSource{ + Keyspace: env.KeyspaceName, + Shard: env.ShardName, + Filter: filter, + OnDdl: binlogdatapb.OnDDLAction_IGNORE, + } + + _, _ = startVReplication(t, bls, "") journal := &binlogdatapb.Journal{ Id: 1, @@ -270,7 +297,13 @@ func TestJournalTableMixed(t *testing.T) { Match: "t1", }}, } - _, _ = startVReplication(t, filter, binlogdatapb.OnDDLAction_IGNORE, "") + bls := &binlogdatapb.BinlogSource{ + Keyspace: env.KeyspaceName, + Shard: env.ShardName, + Filter: filter, + OnDdl: binlogdatapb.OnDDLAction_IGNORE, + } + _, _ = startVReplication(t, bls, "") journal := &binlogdatapb.Journal{ Id: 1, diff --git a/go/vt/vttablet/tabletmanager/vreplication/vstreamer_client_test.go b/go/vt/vttablet/tabletmanager/vreplication/vstreamer_client_test.go index ab2e19d1779..e036f708bf0 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vstreamer_client_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vstreamer_client_test.go @@ -159,6 +159,11 @@ func TestTabletVStreamerClientVStream(t *testing.T) { vsClient := &TabletVStreamerClient{ tablet: tablet, + target: &querypb.Target{ + Keyspace: tablet.Keyspace, + Shard: tablet.Shard, + TabletType: tablet.Type, + }, } filter := &binlogdatapb.Filter{ From 5fd892557c0064a4a210320efb02eebf46e4ee46 Mon Sep 17 00:00:00 2001 From: Rafael Chacon Date: Sun, 1 Dec 2019 15:50:43 -0800 Subject: [PATCH 27/28] Adds more tests and fixes govet issues Signed-off-by: Rafael Chacon --- .../vreplication/vplayer_test.go | 87 +++++++++++++++++++ .../vreplication/vstreamer_client.go | 5 +- .../vreplication/vstreamer_client_test.go | 4 - 3 files changed, 89 insertions(+), 7 deletions(-) diff --git a/go/vt/vttablet/tabletmanager/vreplication/vplayer_test.go b/go/vt/vttablet/tabletmanager/vreplication/vplayer_test.go index 8e97468166b..0c2c19ce088 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vplayer_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vplayer_test.go @@ -32,6 +32,93 @@ import ( binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" ) +func TestPlayerStatementModeWithFilter(t *testing.T) { + defer deleteTablet(addTablet(100)) + + execStatements(t, []string{ + "create table src1(id int, val varbinary(128), primary key(id))", + }) + defer execStatements(t, []string{ + "drop table src1", + }) + env.SchemaEngine.Reload(context.Background()) + + filter := &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "dst1", + Filter: "select * from src1", + }}, + } + bls := &binlogdatapb.BinlogSource{ + Keyspace: env.KeyspaceName, + Shard: env.ShardName, + Filter: filter, + OnDdl: binlogdatapb.OnDDLAction_IGNORE, + } + cancel, _ := startVReplication(t, bls, "") + defer cancel() + + input := []string{ + "set @@session.binlog_format='STATEMENT'", + "insert into src1 values(1, 'aaa')", + "set @@session.binlog_format='ROW'", + } + + // It does not work when filter is enabled + output := []string{ + "begin", + "/update _vt.vreplication set message='Filter rules are not supported for SBR", + } + + execStatements(t, input) + expectDBClientQueries(t, output) +} + +func TestPlayerStatementMode(t *testing.T) { + defer deleteTablet(addTablet(100)) + + execStatements(t, []string{ + "create table src1(id int, val varbinary(128), primary key(id))", + fmt.Sprintf("create table %s.src1(id int, val varbinary(128), primary key(id))", vrepldb), + }) + defer execStatements(t, []string{ + "drop table src1", + fmt.Sprintf("drop table %s.src1", vrepldb), + }) + env.SchemaEngine.Reload(context.Background()) + + filter := &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "/.*", + Filter: "", + }}, + } + bls := &binlogdatapb.BinlogSource{ + Keyspace: env.KeyspaceName, + Shard: env.ShardName, + Filter: filter, + OnDdl: binlogdatapb.OnDDLAction_IGNORE, + } + cancel, _ := startVReplication(t, bls, "") + defer cancel() + + input := []string{ + "set @@session.binlog_format='STATEMENT'", + "insert into src1 values(1, 'aaa')", + "set @@session.binlog_format='ROW'", + } + + output := []string{ + "begin", + "insert into src1 values(1, 'aaa')", + "/update _vt.vreplication set pos=", + "commit", + } + + execStatements(t, input) + expectDBClientQueries(t, output) +} + func TestPlayerFilters(t *testing.T) { defer deleteTablet(addTablet(100)) diff --git a/go/vt/vttablet/tabletmanager/vreplication/vstreamer_client.go b/go/vt/vttablet/tabletmanager/vreplication/vstreamer_client.go index 746697f469c..96c4eeacb71 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vstreamer_client.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vstreamer_client.go @@ -208,10 +208,9 @@ func (vsClient *MySQLVStreamerClient) VStreamRows(ctx context.Context, query str return streamer.Stream() } +// InitVStreamerClient initializes config for vstreamer client func InitVStreamerClient(cfg *dbconfigs.DBConfigs) { - // Make copy of config - dbcfgs = &dbconfigs.DBConfigs{} - *dbcfgs = *cfg + dbcfgs = cfg } type checker struct{} diff --git a/go/vt/vttablet/tabletmanager/vreplication/vstreamer_client_test.go b/go/vt/vttablet/tabletmanager/vreplication/vstreamer_client_test.go index e036f708bf0..e7d52d58102 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vstreamer_client_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vstreamer_client_test.go @@ -20,7 +20,6 @@ import ( "fmt" "reflect" "strings" - "sync" "testing" "time" @@ -41,7 +40,6 @@ func TestTabletVStreamerClientOpen(t *testing.T) { defer deleteTablet(tablet) type fields struct { - mu sync.Mutex isOpen bool tablet *topodatapb.Tablet target *querypb.Target @@ -102,7 +100,6 @@ func TestTabletVStreamerClientClose(t *testing.T) { defer deleteTablet(tablet) type fields struct { - mu sync.Mutex isOpen bool tablet *topodatapb.Tablet target *querypb.Target @@ -366,7 +363,6 @@ func TestMySQLVStreamerClientOpen(t *testing.T) { func TestMySQLVStreamerClientClose(t *testing.T) { type fields struct { - mu sync.Mutex isOpen bool sourceConnParams *mysql.ConnParams vsEngine *vstreamer.Engine From 559a2109bc7c25dfd212c60bca3c2ed1a874fc92 Mon Sep 17 00:00:00 2001 From: Rafael Chacon Date: Wed, 4 Dec 2019 14:45:36 -0800 Subject: [PATCH 28/28] Re-running go-imports and addressing other comments from PR * Compute canAcceptStmtEvents when creating vplayer. Signed-off-by: Rafael Chacon --- go/vt/proto/automation/automation.pb.go | 3 ++- .../automationservice/automationservice.pb.go | 3 ++- go/vt/proto/binlogdata/binlogdata.pb.go | 3 ++- go/vt/proto/binlogservice/binlogservice.pb.go | 3 ++- go/vt/proto/logutil/logutil.pb.go | 3 ++- go/vt/proto/mysqlctl/mysqlctl.pb.go | 3 ++- go/vt/proto/query/query.pb.go | 3 ++- go/vt/proto/queryservice/queryservice.pb.go | 3 ++- .../replicationdata/replicationdata.pb.go | 3 ++- go/vt/proto/tableacl/tableacl.pb.go | 3 ++- .../tabletmanagerdata/tabletmanagerdata.pb.go | 3 ++- .../tabletmanagerservice.pb.go | 3 ++- go/vt/proto/throttlerdata/throttlerdata.pb.go | 3 ++- .../throttlerservice/throttlerservice.pb.go | 3 ++- go/vt/proto/topodata/topodata.pb.go | 3 ++- go/vt/proto/vschema/vschema.pb.go | 3 ++- go/vt/proto/vtctldata/vtctldata.pb.go | 3 ++- go/vt/proto/vtctlservice/vtctlservice.pb.go | 3 ++- go/vt/proto/vtgate/vtgate.pb.go | 3 ++- go/vt/proto/vtgateservice/vtgateservice.pb.go | 3 ++- go/vt/proto/vtrpc/vtrpc.pb.go | 3 ++- go/vt/proto/vttest/vttest.pb.go | 3 ++- go/vt/proto/vttime/time.pb.go | 3 ++- go/vt/proto/vtworkerdata/vtworkerdata.pb.go | 3 ++- .../vtworkerservice/vtworkerservice.pb.go | 3 ++- go/vt/proto/workflow/workflow.pb.go | 3 ++- .../tabletmanager/vreplication/vplayer.go | 22 +++++++++++++------ 27 files changed, 67 insertions(+), 33 deletions(-) diff --git a/go/vt/proto/automation/automation.pb.go b/go/vt/proto/automation/automation.pb.go index 527028c2c71..fe039ca84ef 100644 --- a/go/vt/proto/automation/automation.pb.go +++ b/go/vt/proto/automation/automation.pb.go @@ -5,8 +5,9 @@ package automation import ( fmt "fmt" - proto "github.com/golang/protobuf/proto" math "math" + + proto "github.com/golang/protobuf/proto" ) // Reference imports to suppress errors if they are not otherwise used. diff --git a/go/vt/proto/automationservice/automationservice.pb.go b/go/vt/proto/automationservice/automationservice.pb.go index e776217b6fb..bc2062f6620 100644 --- a/go/vt/proto/automationservice/automationservice.pb.go +++ b/go/vt/proto/automationservice/automationservice.pb.go @@ -6,11 +6,12 @@ package automationservice import ( context "context" fmt "fmt" + math "math" + proto "github.com/golang/protobuf/proto" grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" status "google.golang.org/grpc/status" - math "math" automation "vitess.io/vitess/go/vt/proto/automation" ) diff --git a/go/vt/proto/binlogdata/binlogdata.pb.go b/go/vt/proto/binlogdata/binlogdata.pb.go index 0e186a9dd44..22b36bc0c54 100644 --- a/go/vt/proto/binlogdata/binlogdata.pb.go +++ b/go/vt/proto/binlogdata/binlogdata.pb.go @@ -5,8 +5,9 @@ package binlogdata import ( fmt "fmt" - proto "github.com/golang/protobuf/proto" math "math" + + proto "github.com/golang/protobuf/proto" query "vitess.io/vitess/go/vt/proto/query" topodata "vitess.io/vitess/go/vt/proto/topodata" vtrpc "vitess.io/vitess/go/vt/proto/vtrpc" diff --git a/go/vt/proto/binlogservice/binlogservice.pb.go b/go/vt/proto/binlogservice/binlogservice.pb.go index f10fe87e8be..77312b91234 100644 --- a/go/vt/proto/binlogservice/binlogservice.pb.go +++ b/go/vt/proto/binlogservice/binlogservice.pb.go @@ -6,11 +6,12 @@ package binlogservice import ( context "context" fmt "fmt" + math "math" + proto "github.com/golang/protobuf/proto" grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" status "google.golang.org/grpc/status" - math "math" binlogdata "vitess.io/vitess/go/vt/proto/binlogdata" ) diff --git a/go/vt/proto/logutil/logutil.pb.go b/go/vt/proto/logutil/logutil.pb.go index 8b796b48d01..16f34fa049c 100644 --- a/go/vt/proto/logutil/logutil.pb.go +++ b/go/vt/proto/logutil/logutil.pb.go @@ -5,8 +5,9 @@ package logutil import ( fmt "fmt" - proto "github.com/golang/protobuf/proto" math "math" + + proto "github.com/golang/protobuf/proto" vttime "vitess.io/vitess/go/vt/proto/vttime" ) diff --git a/go/vt/proto/mysqlctl/mysqlctl.pb.go b/go/vt/proto/mysqlctl/mysqlctl.pb.go index f5c5fb76da1..2b80a9971d9 100644 --- a/go/vt/proto/mysqlctl/mysqlctl.pb.go +++ b/go/vt/proto/mysqlctl/mysqlctl.pb.go @@ -6,11 +6,12 @@ package mysqlctl import ( context "context" fmt "fmt" + math "math" + proto "github.com/golang/protobuf/proto" grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" status "google.golang.org/grpc/status" - math "math" ) // Reference imports to suppress errors if they are not otherwise used. diff --git a/go/vt/proto/query/query.pb.go b/go/vt/proto/query/query.pb.go index fe91f5a0525..d536cc1c24c 100644 --- a/go/vt/proto/query/query.pb.go +++ b/go/vt/proto/query/query.pb.go @@ -5,8 +5,9 @@ package query import ( fmt "fmt" - proto "github.com/golang/protobuf/proto" math "math" + + proto "github.com/golang/protobuf/proto" topodata "vitess.io/vitess/go/vt/proto/topodata" vtrpc "vitess.io/vitess/go/vt/proto/vtrpc" ) diff --git a/go/vt/proto/queryservice/queryservice.pb.go b/go/vt/proto/queryservice/queryservice.pb.go index b93c5d5b87d..74b816b4ede 100644 --- a/go/vt/proto/queryservice/queryservice.pb.go +++ b/go/vt/proto/queryservice/queryservice.pb.go @@ -6,11 +6,12 @@ package queryservice import ( context "context" fmt "fmt" + math "math" + proto "github.com/golang/protobuf/proto" grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" status "google.golang.org/grpc/status" - math "math" binlogdata "vitess.io/vitess/go/vt/proto/binlogdata" query "vitess.io/vitess/go/vt/proto/query" ) diff --git a/go/vt/proto/replicationdata/replicationdata.pb.go b/go/vt/proto/replicationdata/replicationdata.pb.go index 3fd65b925c9..3924d47bb4f 100644 --- a/go/vt/proto/replicationdata/replicationdata.pb.go +++ b/go/vt/proto/replicationdata/replicationdata.pb.go @@ -5,8 +5,9 @@ package replicationdata import ( fmt "fmt" - proto "github.com/golang/protobuf/proto" math "math" + + proto "github.com/golang/protobuf/proto" ) // Reference imports to suppress errors if they are not otherwise used. diff --git a/go/vt/proto/tableacl/tableacl.pb.go b/go/vt/proto/tableacl/tableacl.pb.go index bce52b0d6da..5fbfc778baa 100644 --- a/go/vt/proto/tableacl/tableacl.pb.go +++ b/go/vt/proto/tableacl/tableacl.pb.go @@ -5,8 +5,9 @@ package tableacl import ( fmt "fmt" - proto "github.com/golang/protobuf/proto" math "math" + + proto "github.com/golang/protobuf/proto" ) // Reference imports to suppress errors if they are not otherwise used. diff --git a/go/vt/proto/tabletmanagerdata/tabletmanagerdata.pb.go b/go/vt/proto/tabletmanagerdata/tabletmanagerdata.pb.go index fd1f76410e8..52c75655d2e 100644 --- a/go/vt/proto/tabletmanagerdata/tabletmanagerdata.pb.go +++ b/go/vt/proto/tabletmanagerdata/tabletmanagerdata.pb.go @@ -5,8 +5,9 @@ package tabletmanagerdata import ( fmt "fmt" - proto "github.com/golang/protobuf/proto" math "math" + + proto "github.com/golang/protobuf/proto" logutil "vitess.io/vitess/go/vt/proto/logutil" query "vitess.io/vitess/go/vt/proto/query" replicationdata "vitess.io/vitess/go/vt/proto/replicationdata" diff --git a/go/vt/proto/tabletmanagerservice/tabletmanagerservice.pb.go b/go/vt/proto/tabletmanagerservice/tabletmanagerservice.pb.go index 1ccd224e2d7..1f494400a18 100644 --- a/go/vt/proto/tabletmanagerservice/tabletmanagerservice.pb.go +++ b/go/vt/proto/tabletmanagerservice/tabletmanagerservice.pb.go @@ -6,11 +6,12 @@ package tabletmanagerservice import ( context "context" fmt "fmt" + math "math" + proto "github.com/golang/protobuf/proto" grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" status "google.golang.org/grpc/status" - math "math" tabletmanagerdata "vitess.io/vitess/go/vt/proto/tabletmanagerdata" ) diff --git a/go/vt/proto/throttlerdata/throttlerdata.pb.go b/go/vt/proto/throttlerdata/throttlerdata.pb.go index e80dc8c554c..fad03c327e7 100644 --- a/go/vt/proto/throttlerdata/throttlerdata.pb.go +++ b/go/vt/proto/throttlerdata/throttlerdata.pb.go @@ -5,8 +5,9 @@ package throttlerdata import ( fmt "fmt" - proto "github.com/golang/protobuf/proto" math "math" + + proto "github.com/golang/protobuf/proto" ) // Reference imports to suppress errors if they are not otherwise used. diff --git a/go/vt/proto/throttlerservice/throttlerservice.pb.go b/go/vt/proto/throttlerservice/throttlerservice.pb.go index a1fad7d2582..a9270a8df55 100644 --- a/go/vt/proto/throttlerservice/throttlerservice.pb.go +++ b/go/vt/proto/throttlerservice/throttlerservice.pb.go @@ -6,11 +6,12 @@ package throttlerservice import ( context "context" fmt "fmt" + math "math" + proto "github.com/golang/protobuf/proto" grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" status "google.golang.org/grpc/status" - math "math" throttlerdata "vitess.io/vitess/go/vt/proto/throttlerdata" ) diff --git a/go/vt/proto/topodata/topodata.pb.go b/go/vt/proto/topodata/topodata.pb.go index 85fbee2d1a4..b5a704f817b 100644 --- a/go/vt/proto/topodata/topodata.pb.go +++ b/go/vt/proto/topodata/topodata.pb.go @@ -5,8 +5,9 @@ package topodata import ( fmt "fmt" - proto "github.com/golang/protobuf/proto" math "math" + + proto "github.com/golang/protobuf/proto" vttime "vitess.io/vitess/go/vt/proto/vttime" ) diff --git a/go/vt/proto/vschema/vschema.pb.go b/go/vt/proto/vschema/vschema.pb.go index 2d24a06e8f2..05ba4a37e84 100644 --- a/go/vt/proto/vschema/vschema.pb.go +++ b/go/vt/proto/vschema/vschema.pb.go @@ -5,8 +5,9 @@ package vschema import ( fmt "fmt" - proto "github.com/golang/protobuf/proto" math "math" + + proto "github.com/golang/protobuf/proto" query "vitess.io/vitess/go/vt/proto/query" ) diff --git a/go/vt/proto/vtctldata/vtctldata.pb.go b/go/vt/proto/vtctldata/vtctldata.pb.go index 641589e4812..0b7171a412a 100644 --- a/go/vt/proto/vtctldata/vtctldata.pb.go +++ b/go/vt/proto/vtctldata/vtctldata.pb.go @@ -5,8 +5,9 @@ package vtctldata import ( fmt "fmt" - proto "github.com/golang/protobuf/proto" math "math" + + proto "github.com/golang/protobuf/proto" logutil "vitess.io/vitess/go/vt/proto/logutil" ) diff --git a/go/vt/proto/vtctlservice/vtctlservice.pb.go b/go/vt/proto/vtctlservice/vtctlservice.pb.go index 22e8102d6fe..d661e476922 100644 --- a/go/vt/proto/vtctlservice/vtctlservice.pb.go +++ b/go/vt/proto/vtctlservice/vtctlservice.pb.go @@ -6,11 +6,12 @@ package vtctlservice import ( context "context" fmt "fmt" + math "math" + proto "github.com/golang/protobuf/proto" grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" status "google.golang.org/grpc/status" - math "math" vtctldata "vitess.io/vitess/go/vt/proto/vtctldata" ) diff --git a/go/vt/proto/vtgate/vtgate.pb.go b/go/vt/proto/vtgate/vtgate.pb.go index 816d676f732..48186096d68 100644 --- a/go/vt/proto/vtgate/vtgate.pb.go +++ b/go/vt/proto/vtgate/vtgate.pb.go @@ -5,8 +5,9 @@ package vtgate import ( fmt "fmt" - proto "github.com/golang/protobuf/proto" math "math" + + proto "github.com/golang/protobuf/proto" binlogdata "vitess.io/vitess/go/vt/proto/binlogdata" query "vitess.io/vitess/go/vt/proto/query" topodata "vitess.io/vitess/go/vt/proto/topodata" diff --git a/go/vt/proto/vtgateservice/vtgateservice.pb.go b/go/vt/proto/vtgateservice/vtgateservice.pb.go index c020e282f58..2f82480284e 100644 --- a/go/vt/proto/vtgateservice/vtgateservice.pb.go +++ b/go/vt/proto/vtgateservice/vtgateservice.pb.go @@ -6,11 +6,12 @@ package vtgateservice import ( context "context" fmt "fmt" + math "math" + proto "github.com/golang/protobuf/proto" grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" status "google.golang.org/grpc/status" - math "math" query "vitess.io/vitess/go/vt/proto/query" vtgate "vitess.io/vitess/go/vt/proto/vtgate" ) diff --git a/go/vt/proto/vtrpc/vtrpc.pb.go b/go/vt/proto/vtrpc/vtrpc.pb.go index 560e65230da..8a673537d9a 100644 --- a/go/vt/proto/vtrpc/vtrpc.pb.go +++ b/go/vt/proto/vtrpc/vtrpc.pb.go @@ -5,8 +5,9 @@ package vtrpc import ( fmt "fmt" - proto "github.com/golang/protobuf/proto" math "math" + + proto "github.com/golang/protobuf/proto" ) // Reference imports to suppress errors if they are not otherwise used. diff --git a/go/vt/proto/vttest/vttest.pb.go b/go/vt/proto/vttest/vttest.pb.go index 9d72df30f44..7dad543bdfc 100644 --- a/go/vt/proto/vttest/vttest.pb.go +++ b/go/vt/proto/vttest/vttest.pb.go @@ -5,8 +5,9 @@ package vttest import ( fmt "fmt" - proto "github.com/golang/protobuf/proto" math "math" + + proto "github.com/golang/protobuf/proto" ) // Reference imports to suppress errors if they are not otherwise used. diff --git a/go/vt/proto/vttime/time.pb.go b/go/vt/proto/vttime/time.pb.go index 79cca2f65ee..449cd23197b 100644 --- a/go/vt/proto/vttime/time.pb.go +++ b/go/vt/proto/vttime/time.pb.go @@ -5,8 +5,9 @@ package vttime import ( fmt "fmt" - proto "github.com/golang/protobuf/proto" math "math" + + proto "github.com/golang/protobuf/proto" ) // Reference imports to suppress errors if they are not otherwise used. diff --git a/go/vt/proto/vtworkerdata/vtworkerdata.pb.go b/go/vt/proto/vtworkerdata/vtworkerdata.pb.go index 799436b8e1d..986447754bc 100644 --- a/go/vt/proto/vtworkerdata/vtworkerdata.pb.go +++ b/go/vt/proto/vtworkerdata/vtworkerdata.pb.go @@ -5,8 +5,9 @@ package vtworkerdata import ( fmt "fmt" - proto "github.com/golang/protobuf/proto" math "math" + + proto "github.com/golang/protobuf/proto" logutil "vitess.io/vitess/go/vt/proto/logutil" ) diff --git a/go/vt/proto/vtworkerservice/vtworkerservice.pb.go b/go/vt/proto/vtworkerservice/vtworkerservice.pb.go index 70e54ab3172..22a1a03e8aa 100644 --- a/go/vt/proto/vtworkerservice/vtworkerservice.pb.go +++ b/go/vt/proto/vtworkerservice/vtworkerservice.pb.go @@ -6,11 +6,12 @@ package vtworkerservice import ( context "context" fmt "fmt" + math "math" + proto "github.com/golang/protobuf/proto" grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" status "google.golang.org/grpc/status" - math "math" vtworkerdata "vitess.io/vitess/go/vt/proto/vtworkerdata" ) diff --git a/go/vt/proto/workflow/workflow.pb.go b/go/vt/proto/workflow/workflow.pb.go index ffd9d3b6957..44e4c451786 100644 --- a/go/vt/proto/workflow/workflow.pb.go +++ b/go/vt/proto/workflow/workflow.pb.go @@ -5,8 +5,9 @@ package workflow import ( fmt "fmt" - proto "github.com/golang/protobuf/proto" math "math" + + proto "github.com/golang/protobuf/proto" ) // Reference imports to suppress errors if they are not otherwise used. diff --git a/go/vt/vttablet/tabletmanager/vreplication/vplayer.go b/go/vt/vttablet/tabletmanager/vreplication/vplayer.go index 3485f44ae83..76460debbce 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vplayer.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vplayer.go @@ -53,6 +53,8 @@ type vplayer struct { lastTimestampNs int64 // timeOffsetNs keeps track of the clock difference with respect to source tablet. timeOffsetNs int64 + // canAcceptStmtEvents set to true if the current player can accept events in statement mode. Only true for filters that are match all. + canAcceptStmtEvents bool } func newVPlayer(vr *vreplicator, settings binlogplayer.VRSettings, copyState map[string]*sqltypes.Result, pausePos mysql.Position) *vplayer { @@ -88,6 +90,15 @@ func (vp *vplayer) play(ctx context.Context) error { } vp.replicatorPlan = plan + // We can't run in statement mode if there are filters defined. + vp.canAcceptStmtEvents = true + for _, rule := range vp.vr.source.Filter.Rules { + if rule.Filter != "" || rule.Match != "/.*" { + vp.canAcceptStmtEvents = false + break + } + } + if err := vp.fetchAndApply(ctx); err != nil { msg := err.Error() vp.vr.stats.History.Add(&binlogplayer.StatsHistoryRecord{ @@ -166,14 +177,11 @@ func (vp *vplayer) fetchAndApply(ctx context.Context) (err error) { } func (vp *vplayer) applyStmtEvent(ctx context.Context, event *binlogdatapb.VEvent) error { - for _, rule := range vp.vr.source.Filter.Rules { - if rule.Filter != "" || rule.Match != "/.*" { - return fmt.Errorf("Filter rules are not supported for SBR replication: %v", rule) - } - + if vp.canAcceptStmtEvents { + _, err := vp.vr.dbClient.ExecuteWithRetry(ctx, event.Dml) + return err } - _, err := vp.vr.dbClient.ExecuteWithRetry(ctx, event.Dml) - return err + return fmt.Errorf("Filter rules are not supported for SBR replication: %v", vp.vr.source.Filter.GetRules()) } func (vp *vplayer) applyRowEvent(ctx context.Context, rowEvent *binlogdatapb.RowEvent) error {