From ac42b98778b0731456b4da3a878034f7b338c5f3 Mon Sep 17 00:00:00 2001 From: Aaron Lehmann Date: Mon, 13 Mar 2017 15:20:59 -0700 Subject: [PATCH] Version service specs Adds fields to Service and Task that keep track of a version number for ServiceSpecs. This version number is different from the service object version (which is tied to the Raft index). Then change the scheduler to use this, instead of marshalling the specs to compare them (which didn't work reliably). Also make the orchestrator use SpecVersion as an optimization, when available. Signed-off-by: Aaron Lehmann --- api/objects.pb.go | 249 +++++++++++++++++++--------- api/objects.proto | 15 ++ manager/controlapi/service.go | 10 +- manager/orchestrator/task.go | 8 + manager/scheduler/scheduler.go | 40 +++-- manager/scheduler/scheduler_test.go | 34 +++- 6 files changed, 259 insertions(+), 97 deletions(-) diff --git a/api/objects.pb.go b/api/objects.pb.go index 74da298185..aea6330ff3 100644 --- a/api/objects.pb.go +++ b/api/objects.pb.go @@ -77,9 +77,18 @@ type Service struct { ID string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` Meta Meta `protobuf:"bytes,2,opt,name=meta" json:"meta"` Spec ServiceSpec `protobuf:"bytes,3,opt,name=spec" json:"spec"` + // SpecVersion versions Spec, to identify changes in the spec. This is + // not a Version because it uses a different versioning scheme from + // top-level objects and is not directly comparable to top-level object + // versions. + SpecVersion uint64 `protobuf:"varint,10,opt,name=spec_version,json=specVersion,proto3" json:"spec_version,omitempty"` // PreviousSpec is the previous service spec that was in place before // "Spec". PreviousSpec *ServiceSpec `protobuf:"bytes,6,opt,name=previous_spec,json=previousSpec" json:"previous_spec,omitempty"` + // PreviousSpecVersion versions PreviousSpec. This is not a Version + // because it uses a different versioning scheme from top-level + // objects and is not directly comparable to top-level object versions. + PreviousSpecVersion uint64 `protobuf:"varint,11,opt,name=previous_spec_version,json=previousSpecVersion,proto3" json:"previous_spec_version,omitempty"` // Runtime state of service endpoint. This may be different // from the spec version because the user may not have entered // the optional fields like node_port or virtual_ip and it @@ -140,6 +149,9 @@ type Task struct { // Spec defines the desired state of the task as specified by the user. // The system will honor this and will *never* modify it. Spec TaskSpec `protobuf:"bytes,3,opt,name=spec" json:"spec"` + // SpecVersion is copied from Service, to identify which version of the + // spec this task has. + SpecVersion uint64 `protobuf:"varint,14,opt,name=spec_version,json=specVersion,proto3" json:"spec_version,omitempty"` // ServiceID indicates the service under which this task is orchestrated. This // should almost always be set. ServiceID string `protobuf:"bytes,4,opt,name=service_id,json=serviceId,proto3" json:"service_id,omitempty"` @@ -772,6 +784,16 @@ func (m *Service) MarshalTo(dAtA []byte) (int, error) { } i += n15 } + if m.SpecVersion != 0 { + dAtA[i] = 0x50 + i++ + i = encodeVarintObjects(dAtA, i, uint64(m.SpecVersion)) + } + if m.PreviousSpecVersion != 0 { + dAtA[i] = 0x58 + i++ + i = encodeVarintObjects(dAtA, i, uint64(m.PreviousSpecVersion)) + } return i, nil } @@ -972,6 +994,11 @@ func (m *Task) MarshalTo(dAtA []byte) (int, error) { } i += n23 } + if m.SpecVersion != 0 { + dAtA[i] = 0x70 + i++ + i = encodeVarintObjects(dAtA, i, uint64(m.SpecVersion)) + } return i, nil } @@ -1348,6 +1375,12 @@ func (m *Service) Size() (n int) { l = m.PreviousSpec.Size() n += 1 + l + sovObjects(uint64(l)) } + if m.SpecVersion != 0 { + n += 1 + sovObjects(uint64(m.SpecVersion)) + } + if m.PreviousSpecVersion != 0 { + n += 1 + sovObjects(uint64(m.PreviousSpecVersion)) + } return n } @@ -1432,6 +1465,9 @@ func (m *Task) Size() (n int) { l = m.LogDriver.Size() n += 1 + l + sovObjects(uint64(l)) } + if m.SpecVersion != 0 { + n += 1 + sovObjects(uint64(m.SpecVersion)) + } return n } @@ -1594,6 +1630,8 @@ func (this *Service) String() string { `Endpoint:` + strings.Replace(fmt.Sprintf("%v", this.Endpoint), "Endpoint", "Endpoint", 1) + `,`, `UpdateStatus:` + strings.Replace(fmt.Sprintf("%v", this.UpdateStatus), "UpdateStatus", "UpdateStatus", 1) + `,`, `PreviousSpec:` + strings.Replace(fmt.Sprintf("%v", this.PreviousSpec), "ServiceSpec", "ServiceSpec", 1) + `,`, + `SpecVersion:` + fmt.Sprintf("%v", this.SpecVersion) + `,`, + `PreviousSpecVersion:` + fmt.Sprintf("%v", this.PreviousSpecVersion) + `,`, `}`, }, "") return s @@ -1639,6 +1677,7 @@ func (this *Task) String() string { `Networks:` + strings.Replace(fmt.Sprintf("%v", this.Networks), "NetworkAttachment", "NetworkAttachment", 1) + `,`, `Endpoint:` + strings.Replace(fmt.Sprintf("%v", this.Endpoint), "Endpoint", "Endpoint", 1) + `,`, `LogDriver:` + strings.Replace(fmt.Sprintf("%v", this.LogDriver), "Driver", "Driver", 1) + `,`, + `SpecVersion:` + fmt.Sprintf("%v", this.SpecVersion) + `,`, `}`, }, "") return s @@ -2397,6 +2436,44 @@ func (m *Service) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 10: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field SpecVersion", wireType) + } + m.SpecVersion = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowObjects + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.SpecVersion |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 11: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field PreviousSpecVersion", wireType) + } + m.PreviousSpecVersion = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowObjects + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.PreviousSpecVersion |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipObjects(dAtA[iNdEx:]) @@ -3072,6 +3149,25 @@ func (m *Task) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 14: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field SpecVersion", wireType) + } + m.SpecVersion = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowObjects + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.SpecVersion |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipObjects(dAtA[iNdEx:]) @@ -4077,79 +4173,82 @@ var ( func init() { proto.RegisterFile("objects.proto", fileDescriptorObjects) } var fileDescriptorObjects = []byte{ - // 1183 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xbc, 0x57, 0xcd, 0x8e, 0x1b, 0x45, - 0x17, 0x4d, 0x7b, 0x7a, 0x6c, 0xf7, 0xf5, 0x78, 0xf4, 0x7d, 0x45, 0x14, 0x9a, 0x61, 0xb0, 0x07, - 0x47, 0xa0, 0x08, 0x45, 0x0e, 0x84, 0x80, 0x26, 0x40, 0x04, 0xb6, 0x67, 0x04, 0x56, 0x08, 0x44, - 0x95, 0x90, 0x2c, 0xad, 0x9a, 0xee, 0x8a, 0x69, 0xdc, 0xee, 0x6a, 0x55, 0x95, 0x1d, 0x79, 0x87, - 0x78, 0x00, 0x5e, 0x00, 0x09, 0xb1, 0xe6, 0x0d, 0x78, 0x83, 0x2c, 0x58, 0xb0, 0x83, 0x95, 0x45, - 0xfc, 0x24, 0xa8, 0x7e, 0xda, 0xee, 0x91, 0xdb, 0x4e, 0x22, 0x45, 0xd9, 0xd5, 0x75, 0x9d, 0x73, - 0xff, 0xea, 0xd4, 0xed, 0x32, 0xd4, 0xd9, 0xd9, 0x0f, 0x34, 0x90, 0xa2, 0x9d, 0x72, 0x26, 0x19, - 0x42, 0x21, 0x0b, 0x46, 0x94, 0xb7, 0xc5, 0x63, 0xc2, 0xc7, 0xa3, 0x48, 0xb6, 0xa7, 0x1f, 0x1c, - 0xd4, 0xe4, 0x2c, 0xa5, 0x16, 0x70, 0x50, 0x13, 0x29, 0x0d, 0x32, 0xa3, 0x39, 0x64, 0x6c, 0x18, - 0xd3, 0x6b, 0xda, 0x3a, 0x9b, 0x3c, 0xba, 0x26, 0xa3, 0x31, 0x15, 0x92, 0x8c, 0x53, 0x0b, 0xb8, - 0x38, 0x64, 0x43, 0xa6, 0x97, 0xd7, 0xd4, 0xca, 0xfc, 0xda, 0xfa, 0xc3, 0x01, 0xf7, 0x0e, 0x95, - 0x04, 0x7d, 0x0a, 0x95, 0x29, 0xe5, 0x22, 0x62, 0x89, 0xef, 0x1c, 0x39, 0x57, 0x6a, 0xd7, 0xdf, - 0x6c, 0xaf, 0xc7, 0x6f, 0x3f, 0x30, 0x90, 0xae, 0xfb, 0x64, 0xde, 0xbc, 0x80, 0x33, 0x06, 0xba, - 0x09, 0x10, 0x70, 0x4a, 0x24, 0x0d, 0x07, 0x44, 0xfa, 0x25, 0xcd, 0x3f, 0x68, 0x9b, 0x8c, 0xda, - 0x59, 0x46, 0xed, 0xfb, 0x59, 0x46, 0xd8, 0xb3, 0xe8, 0x8e, 0x54, 0xd4, 0x49, 0x1a, 0x66, 0xd4, - 0x9d, 0x67, 0x53, 0x2d, 0xba, 0x23, 0x5b, 0xbf, 0xb8, 0xe0, 0x7e, 0xc3, 0x42, 0x8a, 0x2e, 0x41, - 0x29, 0x0a, 0x75, 0xda, 0x5e, 0xb7, 0xbc, 0x98, 0x37, 0x4b, 0xfd, 0x13, 0x5c, 0x8a, 0x42, 0x74, - 0x1d, 0xdc, 0x31, 0x95, 0xc4, 0x26, 0xe4, 0x17, 0x15, 0xa4, 0x6a, 0xb7, 0xd5, 0x68, 0x2c, 0xfa, - 0x18, 0x5c, 0xd5, 0x56, 0x9b, 0xc9, 0x61, 0x11, 0x47, 0xc5, 0xbc, 0x97, 0xd2, 0x20, 0xe3, 0x29, - 0x3c, 0x3a, 0x85, 0x5a, 0x48, 0x45, 0xc0, 0xa3, 0x54, 0xaa, 0x1e, 0xba, 0x9a, 0x7e, 0x79, 0x13, - 0xfd, 0x64, 0x05, 0xc5, 0x79, 0x1e, 0xfa, 0x0c, 0xca, 0x42, 0x12, 0x39, 0x11, 0xfe, 0xae, 0xf6, - 0xd0, 0xd8, 0x98, 0x80, 0x46, 0xd9, 0x14, 0x2c, 0x07, 0x7d, 0x05, 0xfb, 0x63, 0x92, 0x90, 0x21, - 0xe5, 0x03, 0xeb, 0xa5, 0xac, 0xbd, 0xbc, 0x5d, 0x58, 0xba, 0x41, 0x1a, 0x47, 0xb8, 0x3e, 0xce, - 0x9b, 0xe8, 0x14, 0x80, 0x48, 0x49, 0x82, 0xef, 0xc7, 0x34, 0x91, 0x7e, 0x45, 0x7b, 0x79, 0xa7, - 0x30, 0x17, 0x2a, 0x1f, 0x33, 0x3e, 0xea, 0x2c, 0xc1, 0x38, 0x47, 0x44, 0x5f, 0x42, 0x2d, 0xa0, - 0x5c, 0x46, 0x8f, 0xa2, 0x80, 0x48, 0xea, 0x57, 0xb5, 0x9f, 0x66, 0x91, 0x9f, 0xde, 0x0a, 0x66, - 0x8b, 0xca, 0x33, 0xd1, 0xfb, 0xe0, 0x72, 0x16, 0x53, 0xdf, 0x3b, 0x72, 0xae, 0xec, 0x6f, 0x3e, - 0x16, 0xcc, 0x62, 0x8a, 0x35, 0xb2, 0xf5, 0x77, 0x09, 0x2a, 0xf7, 0x28, 0x9f, 0x46, 0xc1, 0xcb, - 0x15, 0xc8, 0xcd, 0x73, 0x02, 0x29, 0xac, 0xc5, 0x86, 0x5d, 0xd3, 0xc8, 0x31, 0x54, 0x69, 0x12, - 0xa6, 0x2c, 0x4a, 0xa4, 0x15, 0x48, 0x61, 0x21, 0xa7, 0x16, 0x83, 0x97, 0x68, 0x74, 0x0a, 0x75, - 0xa3, 0xfb, 0xc1, 0x39, 0x75, 0x1c, 0x15, 0xd1, 0xbf, 0xd3, 0x40, 0x7b, 0xac, 0x7b, 0x93, 0x9c, - 0x85, 0x4e, 0xa0, 0x9e, 0x72, 0x3a, 0x8d, 0xd8, 0x44, 0x0c, 0x74, 0x11, 0xe5, 0xe7, 0x2a, 0x02, - 0xef, 0x65, 0x2c, 0x65, 0xb5, 0x7e, 0x2d, 0x41, 0x35, 0xcb, 0x11, 0xdd, 0xb0, 0xed, 0x70, 0x36, - 0x27, 0x94, 0x61, 0xb5, 0x2b, 0xd3, 0x89, 0x1b, 0xb0, 0x9b, 0x32, 0x2e, 0x85, 0x5f, 0x3a, 0xda, - 0xd9, 0xa4, 0xf2, 0xbb, 0x8c, 0xcb, 0x1e, 0x4b, 0x1e, 0x45, 0x43, 0x6c, 0xc0, 0xe8, 0x21, 0xd4, - 0xa6, 0x11, 0x97, 0x13, 0x12, 0x0f, 0xa2, 0x54, 0xf8, 0x3b, 0x9a, 0xfb, 0xee, 0xb6, 0x90, 0xed, - 0x07, 0x06, 0xdf, 0xbf, 0xdb, 0xdd, 0x5f, 0xcc, 0x9b, 0xb0, 0x34, 0x05, 0x06, 0xeb, 0xaa, 0x9f, - 0x8a, 0x83, 0x3b, 0xe0, 0x2d, 0x77, 0xd0, 0x55, 0x80, 0xc4, 0x88, 0x7a, 0xb0, 0x14, 0x4d, 0x7d, - 0x31, 0x6f, 0x7a, 0x56, 0xea, 0xfd, 0x13, 0xec, 0x59, 0x40, 0x3f, 0x44, 0x08, 0x5c, 0x12, 0x86, - 0x5c, 0x4b, 0xc8, 0xc3, 0x7a, 0xdd, 0xfa, 0x73, 0x17, 0xdc, 0xfb, 0x44, 0x8c, 0x5e, 0xf5, 0x60, - 0x52, 0x31, 0xd7, 0x44, 0x77, 0x15, 0x40, 0x98, 0xa3, 0x54, 0xe5, 0xb8, 0xab, 0x72, 0xec, 0x01, - 0xab, 0x72, 0x2c, 0xc0, 0x94, 0x23, 0x62, 0x26, 0xb5, 0xbe, 0x5c, 0xac, 0xd7, 0xe8, 0x32, 0x54, - 0x12, 0x16, 0x6a, 0x7a, 0x59, 0xd3, 0x61, 0x31, 0x6f, 0x96, 0xd5, 0x75, 0xeb, 0x9f, 0xe0, 0xb2, - 0xda, 0xea, 0x87, 0xea, 0xa6, 0x93, 0x24, 0x61, 0x92, 0xa8, 0x31, 0x26, 0xec, 0xc4, 0x28, 0x14, - 0x56, 0x67, 0x05, 0xcb, 0x6e, 0x7a, 0x8e, 0x89, 0x1e, 0xc0, 0x6b, 0x59, 0xbe, 0x79, 0x87, 0xd5, - 0x17, 0x71, 0x88, 0xac, 0x87, 0xdc, 0x4e, 0x6e, 0xb2, 0x7a, 0x9b, 0x27, 0xab, 0xee, 0x60, 0xd1, - 0x64, 0xed, 0x42, 0x3d, 0xa4, 0x22, 0xe2, 0x34, 0xd4, 0x37, 0x90, 0xfa, 0xa0, 0x07, 0xd1, 0x5b, - 0xdb, 0x9c, 0x50, 0xbc, 0x67, 0x39, 0xda, 0x42, 0x1d, 0xa8, 0x5a, 0xdd, 0x08, 0xbf, 0xa6, 0xb5, - 0xfb, 0x9c, 0x13, 0x75, 0x49, 0x3b, 0x37, 0x41, 0xf6, 0x5e, 0x68, 0x82, 0xdc, 0x04, 0x88, 0xd9, - 0x70, 0x10, 0xf2, 0x68, 0x4a, 0xb9, 0x5f, 0xb7, 0xdf, 0xd9, 0x02, 0xee, 0x89, 0x46, 0x60, 0x2f, - 0x66, 0x43, 0xb3, 0x6c, 0xfd, 0xe4, 0xc0, 0xff, 0xd7, 0x92, 0x42, 0x1f, 0x41, 0xc5, 0xa6, 0xb5, - 0xed, 0xc1, 0x60, 0x79, 0x38, 0xc3, 0xa2, 0x43, 0xf0, 0xd4, 0x1d, 0xa1, 0x42, 0x50, 0x73, 0xfb, - 0x3d, 0xbc, 0xfa, 0x01, 0xf9, 0x50, 0x21, 0x71, 0x44, 0xd4, 0xde, 0x8e, 0xde, 0xcb, 0xcc, 0xd6, - 0xcf, 0x25, 0xa8, 0x58, 0x67, 0xaf, 0x7a, 0x9c, 0xdb, 0xb0, 0x6b, 0x37, 0xeb, 0x16, 0xec, 0x99, - 0x76, 0x5a, 0x49, 0xb8, 0xcf, 0x6c, 0x6a, 0xcd, 0xe0, 0x8d, 0x1c, 0x6e, 0x81, 0x1b, 0xa5, 0x64, - 0x6c, 0x47, 0x79, 0x61, 0xe4, 0xfe, 0xdd, 0xce, 0x9d, 0x6f, 0x53, 0xa3, 0xec, 0xea, 0x62, 0xde, - 0x74, 0xd5, 0x0f, 0x58, 0xd3, 0x5a, 0xbf, 0xed, 0x42, 0xa5, 0x17, 0x4f, 0x84, 0xa4, 0xfc, 0x55, - 0x37, 0xc4, 0x86, 0x5d, 0x6b, 0x48, 0x0f, 0x2a, 0x9c, 0x31, 0x39, 0x08, 0xc8, 0xb6, 0x5e, 0x60, - 0xc6, 0x64, 0xaf, 0xd3, 0xdd, 0x57, 0x44, 0x35, 0x48, 0x8c, 0x8d, 0xcb, 0x8a, 0xda, 0x23, 0xe8, - 0x21, 0x5c, 0xca, 0xc6, 0xef, 0x19, 0x63, 0x52, 0x48, 0x4e, 0xd2, 0xc1, 0x88, 0xce, 0xd4, 0x37, - 0x6f, 0x67, 0xd3, 0x5b, 0xe6, 0x34, 0x09, 0xf8, 0x4c, 0x37, 0xea, 0x36, 0x9d, 0xe1, 0x8b, 0xd6, - 0x41, 0x37, 0xe3, 0xdf, 0xa6, 0x33, 0x81, 0x3e, 0x87, 0x43, 0xba, 0x84, 0x29, 0x8f, 0x83, 0x98, - 0x8c, 0xd5, 0x87, 0x65, 0x10, 0xc4, 0x2c, 0x18, 0xe9, 0xd9, 0xe6, 0xe2, 0x37, 0x68, 0xde, 0xd5, - 0xd7, 0x06, 0xd1, 0x53, 0x00, 0x24, 0xc0, 0x3f, 0x8b, 0x49, 0x30, 0x8a, 0x23, 0xa1, 0x9e, 0xab, - 0xb9, 0xe7, 0x89, 0x1a, 0x4f, 0x2a, 0xb7, 0xe3, 0x2d, 0xdd, 0x6a, 0x77, 0x57, 0xdc, 0xdc, 0x63, - 0x47, 0x9c, 0x26, 0x92, 0xcf, 0xf0, 0xeb, 0x67, 0xc5, 0xbb, 0xa8, 0x0b, 0xb5, 0x49, 0xa2, 0xc2, - 0x9b, 0x1e, 0x78, 0xcf, 0xdb, 0x03, 0x30, 0x2c, 0x55, 0xf9, 0xc1, 0x14, 0x0e, 0xb7, 0x05, 0x47, - 0xff, 0x83, 0x9d, 0x11, 0x9d, 0x19, 0xfd, 0x60, 0xb5, 0x44, 0x5f, 0xc0, 0xee, 0x94, 0xc4, 0x13, - 0x6a, 0x95, 0xf3, 0x5e, 0x51, 0xbc, 0x62, 0x97, 0xd8, 0x10, 0x3f, 0x29, 0x1d, 0x3b, 0xad, 0xdf, - 0x1d, 0x28, 0xdf, 0xa3, 0x01, 0xa7, 0xf2, 0xa5, 0x2a, 0xf4, 0xf8, 0x9c, 0x42, 0x1b, 0xc5, 0x8f, - 0x17, 0x15, 0x75, 0x4d, 0xa0, 0x07, 0x50, 0x8d, 0x12, 0x49, 0x79, 0x42, 0x62, 0xad, 0xd0, 0x2a, - 0x5e, 0xda, 0x5d, 0xff, 0xc9, 0xd3, 0xc6, 0x85, 0x7f, 0x9e, 0x36, 0x2e, 0xfc, 0xb8, 0x68, 0x38, - 0x4f, 0x16, 0x0d, 0xe7, 0xaf, 0x45, 0xc3, 0xf9, 0x77, 0xd1, 0x70, 0xce, 0xca, 0xfa, 0x6f, 0xc8, - 0x87, 0xff, 0x05, 0x00, 0x00, 0xff, 0xff, 0xf1, 0xbb, 0xa8, 0xb4, 0xa0, 0x0d, 0x00, 0x00, + // 1222 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xbc, 0x57, 0xcf, 0x6e, 0x1b, 0x45, + 0x1c, 0xee, 0xda, 0x1b, 0xff, 0xf9, 0x39, 0x8e, 0x60, 0x5a, 0xca, 0x12, 0x82, 0x9d, 0xba, 0x02, + 0x55, 0xa8, 0x72, 0x21, 0x14, 0x94, 0x02, 0x15, 0xd8, 0x4e, 0x04, 0x56, 0x29, 0x54, 0xd3, 0xd2, + 0x1e, 0x57, 0x93, 0xdd, 0xa9, 0x59, 0xbc, 0xde, 0x59, 0xcd, 0x8c, 0x5d, 0xf9, 0x86, 0x78, 0x00, + 0x5e, 0x00, 0x09, 0x71, 0xe6, 0xc0, 0x9d, 0x37, 0xe8, 0x91, 0x23, 0xa7, 0x88, 0xfa, 0x05, 0x78, + 0x05, 0x34, 0x7f, 0xd6, 0xd9, 0xc8, 0xeb, 0xb4, 0x95, 0xaa, 0xdc, 0x66, 0x3c, 0xdf, 0xf7, 0xfb, + 0x97, 0x6f, 0xbe, 0x9d, 0x40, 0x93, 0x1d, 0xfd, 0x48, 0x03, 0x29, 0xba, 0x29, 0x67, 0x92, 0x21, + 0x14, 0xb2, 0x60, 0x4c, 0x79, 0x57, 0x3c, 0x21, 0x7c, 0x32, 0x8e, 0x64, 0x77, 0xf6, 0xe1, 0x76, + 0x43, 0xce, 0x53, 0x6a, 0x01, 0xdb, 0x0d, 0x91, 0xd2, 0x20, 0xdb, 0xb4, 0x47, 0x8c, 0x8d, 0x62, + 0x7a, 0x43, 0xef, 0x8e, 0xa6, 0x8f, 0x6f, 0xc8, 0x68, 0x42, 0x85, 0x24, 0x93, 0xd4, 0x02, 0x2e, + 0x8d, 0xd8, 0x88, 0xe9, 0xe5, 0x0d, 0xb5, 0x32, 0xbf, 0x76, 0xfe, 0x72, 0xc0, 0xbd, 0x4b, 0x25, + 0x41, 0x9f, 0x41, 0x75, 0x46, 0xb9, 0x88, 0x58, 0xe2, 0x39, 0xbb, 0xce, 0xb5, 0xc6, 0xde, 0xdb, + 0xdd, 0xd5, 0xfc, 0xdd, 0x87, 0x06, 0xd2, 0x77, 0x9f, 0x1e, 0xb7, 0x2f, 0xe0, 0x8c, 0x81, 0x6e, + 0x01, 0x04, 0x9c, 0x12, 0x49, 0x43, 0x9f, 0x48, 0xaf, 0xa4, 0xf9, 0xdb, 0x5d, 0x53, 0x51, 0x37, + 0xab, 0xa8, 0xfb, 0x20, 0xab, 0x08, 0xd7, 0x2d, 0xba, 0x27, 0x15, 0x75, 0x9a, 0x86, 0x19, 0xb5, + 0xfc, 0x7c, 0xaa, 0x45, 0xf7, 0x64, 0xe7, 0x57, 0x17, 0xdc, 0x6f, 0x59, 0x48, 0xd1, 0x65, 0x28, + 0x45, 0xa1, 0x2e, 0xbb, 0xde, 0xaf, 0x2c, 0x8e, 0xdb, 0xa5, 0xe1, 0x01, 0x2e, 0x45, 0x21, 0xda, + 0x03, 0x77, 0x42, 0x25, 0xb1, 0x05, 0x79, 0x45, 0x0d, 0xa9, 0xde, 0x6d, 0x37, 0x1a, 0x8b, 0x3e, + 0x01, 0x57, 0x8d, 0xd5, 0x56, 0xb2, 0x53, 0xc4, 0x51, 0x39, 0xef, 0xa7, 0x34, 0xc8, 0x78, 0x0a, + 0x8f, 0x0e, 0xa1, 0x11, 0x52, 0x11, 0xf0, 0x28, 0x95, 0x6a, 0x86, 0xae, 0xa6, 0x5f, 0x5d, 0x47, + 0x3f, 0x38, 0x81, 0xe2, 0x3c, 0x0f, 0x7d, 0x0e, 0x15, 0x21, 0x89, 0x9c, 0x0a, 0x6f, 0x43, 0x47, + 0x68, 0xad, 0x2d, 0x40, 0xa3, 0x6c, 0x09, 0x96, 0x83, 0xbe, 0x86, 0xad, 0x09, 0x49, 0xc8, 0x88, + 0x72, 0xdf, 0x46, 0xa9, 0xe8, 0x28, 0x57, 0x0a, 0x5b, 0x37, 0x48, 0x13, 0x08, 0x37, 0x27, 0xf9, + 0x2d, 0x3a, 0x04, 0x20, 0x52, 0x92, 0xe0, 0x87, 0x09, 0x4d, 0xa4, 0x57, 0xd5, 0x51, 0xde, 0x2d, + 0xac, 0x85, 0xca, 0x27, 0x8c, 0x8f, 0x7b, 0x4b, 0x30, 0xce, 0x11, 0xd1, 0x57, 0xd0, 0x08, 0x28, + 0x97, 0xd1, 0xe3, 0x28, 0x20, 0x92, 0x7a, 0x35, 0x1d, 0xa7, 0x5d, 0x14, 0x67, 0x70, 0x02, 0xb3, + 0x4d, 0xe5, 0x99, 0xe8, 0x03, 0x70, 0x39, 0x8b, 0xa9, 0x57, 0xdf, 0x75, 0xae, 0x6d, 0xad, 0xff, + 0xb3, 0x60, 0x16, 0x53, 0xac, 0x91, 0x9d, 0x3f, 0xcb, 0x50, 0xbd, 0x4f, 0xf9, 0x2c, 0x0a, 0x5e, + 0xad, 0x40, 0x6e, 0x9d, 0x12, 0x48, 0x61, 0x2f, 0x36, 0xed, 0x8a, 0x46, 0xf6, 0xa1, 0x46, 0x93, + 0x30, 0x65, 0x51, 0x22, 0xad, 0x40, 0x0a, 0x1b, 0x39, 0xb4, 0x18, 0xbc, 0x44, 0xa3, 0x43, 0x68, + 0x1a, 0xdd, 0xfb, 0xa7, 0xd4, 0xb1, 0x5b, 0x44, 0xff, 0x5e, 0x03, 0xed, 0x9f, 0x75, 0x73, 0x9a, + 0xdb, 0xa1, 0x03, 0x68, 0xa6, 0x9c, 0xce, 0x22, 0x36, 0x15, 0xbe, 0x6e, 0xa2, 0xf2, 0x42, 0x4d, + 0xe0, 0xcd, 0x8c, 0xa5, 0x76, 0xe8, 0x0a, 0x6c, 0x2a, 0xb2, 0x9f, 0xf9, 0x05, 0xec, 0x3a, 0xd7, + 0x5c, 0xac, 0xdd, 0xc8, 0xfa, 0x03, 0xda, 0x83, 0x37, 0x4e, 0x25, 0x5a, 0x62, 0x1b, 0x1a, 0x7b, + 0x31, 0x1f, 0xcf, 0x72, 0x3a, 0xbf, 0x95, 0xa0, 0x96, 0xb5, 0x8e, 0x6e, 0xda, 0x29, 0x3b, 0xeb, + 0xfb, 0xcc, 0xb0, 0xba, 0x42, 0x33, 0xe0, 0x9b, 0xb0, 0x91, 0x32, 0x2e, 0x85, 0x57, 0xda, 0x2d, + 0xaf, 0xbb, 0x3c, 0xf7, 0x18, 0x97, 0x03, 0x96, 0x3c, 0x8e, 0x46, 0xd8, 0x80, 0xd1, 0x23, 0x68, + 0xcc, 0x22, 0x2e, 0xa7, 0x24, 0xf6, 0xa3, 0x54, 0x78, 0x65, 0xcd, 0x7d, 0xef, 0xac, 0x94, 0xdd, + 0x87, 0x06, 0x3f, 0xbc, 0xd7, 0xdf, 0x5a, 0x1c, 0xb7, 0x61, 0xb9, 0x15, 0x18, 0x6c, 0xa8, 0x61, + 0x2a, 0xb6, 0xef, 0x42, 0x7d, 0x79, 0x82, 0xae, 0x03, 0x24, 0xe6, 0xae, 0xf8, 0x4b, 0x2d, 0x36, + 0x17, 0xc7, 0xed, 0xba, 0xbd, 0x41, 0xc3, 0x03, 0x5c, 0xb7, 0x80, 0x61, 0x88, 0x10, 0xb8, 0x24, + 0x0c, 0xb9, 0x56, 0x66, 0x1d, 0xeb, 0x75, 0xe7, 0xbf, 0x0d, 0x70, 0x1f, 0x10, 0x31, 0x3e, 0x6f, + 0xbf, 0x53, 0x39, 0x57, 0xb4, 0x7c, 0x1d, 0x40, 0x18, 0x85, 0xa8, 0x76, 0xdc, 0x93, 0x76, 0xac, + 0x6e, 0x54, 0x3b, 0x16, 0x60, 0xda, 0x11, 0x31, 0x93, 0x5a, 0xb6, 0x2e, 0xd6, 0x6b, 0x74, 0x15, + 0xaa, 0x09, 0x0b, 0x35, 0xbd, 0xa2, 0xe9, 0xb0, 0x38, 0x6e, 0x57, 0xd4, 0x2d, 0x1e, 0x1e, 0xe0, + 0x8a, 0x3a, 0x1a, 0x86, 0xca, 0x40, 0x48, 0x92, 0x30, 0x49, 0x94, 0x3b, 0x0a, 0x6b, 0x44, 0x85, + 0x7a, 0xed, 0x9d, 0xc0, 0x32, 0x03, 0xc9, 0x31, 0xd1, 0x43, 0xb8, 0x98, 0xd5, 0x9b, 0x0f, 0x58, + 0x7b, 0x99, 0x80, 0xc8, 0x46, 0xc8, 0x9d, 0xe4, 0x0c, 0xbb, 0xbe, 0xde, 0xb0, 0xf5, 0x04, 0x8b, + 0x0c, 0xbb, 0x0f, 0xcd, 0x90, 0x8a, 0x88, 0xd3, 0x50, 0x5f, 0x6c, 0xaa, 0xef, 0xd2, 0xd6, 0xde, + 0x3b, 0x67, 0x05, 0xa1, 0x78, 0xd3, 0x72, 0xf4, 0x0e, 0xf5, 0xa0, 0x66, 0x75, 0x23, 0xbc, 0x86, + 0xd6, 0xee, 0x0b, 0x1a, 0xf5, 0x92, 0x76, 0xca, 0x98, 0x36, 0x5f, 0xca, 0x98, 0x6e, 0x01, 0xc4, + 0x6c, 0xe4, 0x87, 0x3c, 0x9a, 0x51, 0xee, 0x35, 0xed, 0xe7, 0xbb, 0x80, 0x7b, 0xa0, 0x11, 0xb8, + 0x1e, 0xb3, 0x91, 0x59, 0xae, 0xd8, 0xc8, 0xd6, 0x8a, 0x8d, 0x74, 0x7e, 0x76, 0xe0, 0xf5, 0x95, + 0xba, 0xd1, 0xc7, 0x50, 0xb5, 0x95, 0x9f, 0xf5, 0x54, 0xb1, 0x3c, 0x9c, 0x61, 0xd1, 0x0e, 0xd4, + 0xd5, 0x35, 0xa2, 0x42, 0x50, 0x63, 0x10, 0x75, 0x7c, 0xf2, 0x03, 0xf2, 0xa0, 0x4a, 0xe2, 0x88, + 0xa8, 0xb3, 0xb2, 0x3e, 0xcb, 0xb6, 0x9d, 0x5f, 0x4a, 0x50, 0xb5, 0xc1, 0xce, 0xfb, 0x43, 0x62, + 0xd3, 0xae, 0x5c, 0xbe, 0xdb, 0xb0, 0x69, 0x26, 0x6e, 0x55, 0xe3, 0x3e, 0x77, 0xee, 0x0d, 0x83, + 0x37, 0x8a, 0xb9, 0x0d, 0x6e, 0x94, 0x92, 0x89, 0xfd, 0x88, 0x14, 0x66, 0x1e, 0xde, 0xeb, 0xdd, + 0xfd, 0x2e, 0x35, 0xe2, 0xaf, 0x2d, 0x8e, 0xdb, 0xae, 0xfa, 0x01, 0x6b, 0x5a, 0xe7, 0xf7, 0x0d, + 0xa8, 0x0e, 0xe2, 0xa9, 0x90, 0x94, 0x9f, 0xf7, 0x40, 0x6c, 0xda, 0x95, 0x81, 0x0c, 0xa0, 0xca, + 0x19, 0x93, 0x7e, 0x40, 0xce, 0x9a, 0x05, 0x66, 0x4c, 0x0e, 0x7a, 0xfd, 0x2d, 0x45, 0x54, 0x5e, + 0x63, 0xf6, 0xb8, 0xa2, 0xa8, 0x03, 0x82, 0x1e, 0xc1, 0xe5, 0xcc, 0xa1, 0x8f, 0x18, 0x93, 0x42, + 0x72, 0x92, 0xfa, 0x63, 0x3a, 0x57, 0x5f, 0xdb, 0xf2, 0xba, 0x57, 0xd4, 0x61, 0x12, 0xf0, 0xb9, + 0x1e, 0xd4, 0x1d, 0x3a, 0xc7, 0x97, 0x6c, 0x80, 0x7e, 0xc6, 0xbf, 0x43, 0xe7, 0x02, 0x7d, 0x01, + 0x3b, 0x74, 0x09, 0x53, 0x11, 0xfd, 0x98, 0x4c, 0xd4, 0xb7, 0xc7, 0x0f, 0x62, 0x16, 0x8c, 0xb5, + 0xfd, 0xb9, 0xf8, 0x2d, 0x9a, 0x0f, 0xf5, 0x8d, 0x41, 0x0c, 0x14, 0x00, 0x09, 0xf0, 0x8e, 0x62, + 0x12, 0x8c, 0xe3, 0x48, 0xa8, 0x87, 0x72, 0xee, 0x61, 0xa4, 0x1c, 0x4c, 0xd5, 0xb6, 0x7f, 0xc6, + 0xb4, 0xba, 0xfd, 0x13, 0x6e, 0xee, 0x99, 0x25, 0x0e, 0x13, 0xc9, 0xe7, 0xf8, 0xcd, 0xa3, 0xe2, + 0x53, 0xd4, 0x87, 0xc6, 0x34, 0x51, 0xe9, 0xcd, 0x0c, 0xea, 0x2f, 0x3a, 0x03, 0x30, 0x2c, 0xd5, + 0xf9, 0xf6, 0x0c, 0x76, 0xce, 0x4a, 0x8e, 0x5e, 0x83, 0xf2, 0x98, 0xce, 0x8d, 0x7e, 0xb0, 0x5a, + 0xa2, 0x2f, 0x61, 0x63, 0x46, 0xe2, 0x29, 0xb5, 0xca, 0x79, 0xbf, 0x28, 0x5f, 0x71, 0x48, 0x6c, + 0x88, 0x9f, 0x96, 0xf6, 0x9d, 0xce, 0x1f, 0x0e, 0x54, 0xee, 0xd3, 0x80, 0x53, 0xf9, 0x4a, 0x15, + 0xba, 0x7f, 0x4a, 0xa1, 0xad, 0xe2, 0x67, 0x93, 0xca, 0xba, 0x22, 0xd0, 0x6d, 0xa8, 0x45, 0x89, + 0xa4, 0x3c, 0x21, 0xb1, 0x56, 0x68, 0x0d, 0x2f, 0xf7, 0x7d, 0xef, 0xe9, 0xb3, 0xd6, 0x85, 0x7f, + 0x9e, 0xb5, 0x2e, 0xfc, 0xb4, 0x68, 0x39, 0x4f, 0x17, 0x2d, 0xe7, 0xef, 0x45, 0xcb, 0xf9, 0x77, + 0xd1, 0x72, 0x8e, 0x2a, 0xfa, 0x1f, 0xa0, 0x8f, 0xfe, 0x0f, 0x00, 0x00, 0xff, 0xff, 0x56, 0x1e, + 0x34, 0x83, 0x1a, 0x0e, 0x00, 0x00, } diff --git a/api/objects.proto b/api/objects.proto index 36b7a4addb..c86f6b0619 100644 --- a/api/objects.proto +++ b/api/objects.proto @@ -69,10 +69,21 @@ message Service { ServiceSpec spec = 3 [(gogoproto.nullable) = false]; + // SpecVersion versions Spec, to identify changes in the spec. This is + // not a Version because it uses a different versioning scheme from + // top-level objects and is not directly comparable to top-level object + // versions. + uint64 spec_version = 10; + // PreviousSpec is the previous service spec that was in place before // "Spec". ServiceSpec previous_spec = 6; + // PreviousSpecVersion versions PreviousSpec. This is not a Version + // because it uses a different versioning scheme from top-level + // objects and is not directly comparable to top-level object versions. + uint64 previous_spec_version = 11; + // Runtime state of service endpoint. This may be different // from the spec version because the user may not have entered // the optional fields like node_port or virtual_ip and it @@ -131,6 +142,10 @@ message Task { // The system will honor this and will *never* modify it. TaskSpec spec = 3 [(gogoproto.nullable) = false]; + // SpecVersion is copied from Service, to identify which version of the + // spec this task has. + uint64 spec_version = 14; + // ServiceID indicates the service under which this task is orchestrated. This // should almost always be set. string service_id = 4; diff --git a/manager/controlapi/service.go b/manager/controlapi/service.go index b5e8710a4b..f510e5f842 100644 --- a/manager/controlapi/service.go +++ b/manager/controlapi/service.go @@ -476,8 +476,9 @@ func (s *Server) CreateService(ctx context.Context, request *api.CreateServiceRe // TODO(aluzzardi): Consider using `Name` as a primary key to handle // duplicate creations. See #65 service := &api.Service{ - ID: identity.NewID(), - Spec: *request.Spec, + ID: identity.NewID(), + Spec: *request.Spec, + SpecVersion: 1, } if doesServiceNeedIngress(service) { @@ -599,8 +600,11 @@ func (s *Server) UpdateService(ctx context.Context, request *api.UpdateServiceRe } curSpec := service.Spec.Copy() + curSpecVersion := service.SpecVersion service.Spec = *service.PreviousSpec.Copy() + service.SpecVersion = service.PreviousSpecVersion service.PreviousSpec = curSpec + service.PreviousSpecVersion = curSpecVersion service.UpdateStatus = &api.UpdateStatus{ State: api.UpdateStatus_ROLLBACK_STARTED, @@ -609,7 +613,9 @@ func (s *Server) UpdateService(ctx context.Context, request *api.UpdateServiceRe } } else { service.PreviousSpec = service.Spec.Copy() + service.PreviousSpecVersion = service.SpecVersion service.Spec = *request.Spec.Copy() + service.SpecVersion++ // Reset update status service.UpdateStatus = nil diff --git a/manager/orchestrator/task.go b/manager/orchestrator/task.go index 9197eee4c0..94cd898d26 100644 --- a/manager/orchestrator/task.go +++ b/manager/orchestrator/task.go @@ -29,6 +29,7 @@ func NewTask(cluster *api.Cluster, service *api.Service, slot uint64, nodeID str ID: taskID, ServiceAnnotations: service.Spec.Annotations, Spec: service.Spec.Task, + SpecVersion: service.SpecVersion, ServiceID: service.ID, Slot: slot, Status: api.TaskStatus{ @@ -62,6 +63,13 @@ func RestartCondition(task *api.Task) api.RestartPolicy_RestartCondition { // IsTaskDirty determines whether a task matches the given service's spec. func IsTaskDirty(s *api.Service, t *api.Task) bool { + // If the spec version matches, we know the task is not dirty. However, + // if it does not match, that doesn't mean the task is dirty, since + // only a portion of the spec is included in the comparison. + if t.SpecVersion != 0 && s.SpecVersion == t.SpecVersion { + return false + } + return !reflect.DeepEqual(s.Spec.Task, t.Spec) || (t.Endpoint != nil && !reflect.DeepEqual(s.Spec.Endpoint, t.Endpoint.Spec)) } diff --git a/manager/scheduler/scheduler.go b/manager/scheduler/scheduler.go index 7935187e93..67d1b8f2e6 100644 --- a/manager/scheduler/scheduler.go +++ b/manager/scheduler/scheduler.go @@ -331,7 +331,12 @@ func (s *Scheduler) processPreassignedTasks(ctx context.Context) { // tick attempts to schedule the queue. func (s *Scheduler) tick(ctx context.Context) { - tasksByCommonSpec := make(map[string]map[string]*api.Task) + type commonSpecKey struct { + serviceID string + specVersion uint64 + } + tasksByCommonSpec := make(map[commonSpecKey]map[string]*api.Task) + var oneOffTasks []*api.Task schedulingDecisions := make(map[string]schedulingDecision, len(s.unassignedTasks)) for taskID, t := range s.unassignedTasks { @@ -341,30 +346,31 @@ func (s *Scheduler) tick(ctx context.Context) { continue } - // Group common tasks with common specs by marshalling the spec - // into taskKey and using it as a map key. - // TODO(aaronl): Once specs are versioned, this will allow a - // much more efficient fast path. - fieldsToMarshal := api.Task{ - ServiceID: t.ServiceID, - Spec: t.Spec, - } - marshalled, err := fieldsToMarshal.Marshal() - if err != nil { - panic(err) - } - taskGroupKey := string(marshalled) + // Group tasks with common specs + if t.SpecVersion != 0 { + taskGroupKey := commonSpecKey{ + serviceID: t.ServiceID, + specVersion: t.SpecVersion, + } - if tasksByCommonSpec[taskGroupKey] == nil { - tasksByCommonSpec[taskGroupKey] = make(map[string]*api.Task) + if tasksByCommonSpec[taskGroupKey] == nil { + tasksByCommonSpec[taskGroupKey] = make(map[string]*api.Task) + } + tasksByCommonSpec[taskGroupKey][taskID] = t + } else { + // This task doesn't have a spec version. We have to + // schedule it as a one-off. + oneOffTasks = append(oneOffTasks, t) } - tasksByCommonSpec[taskGroupKey][taskID] = t delete(s.unassignedTasks, taskID) } for _, taskGroup := range tasksByCommonSpec { s.scheduleTaskGroup(ctx, taskGroup, schedulingDecisions) } + for _, t := range oneOffTasks { + s.scheduleTaskGroup(ctx, map[string]*api.Task{t.ID: t}, schedulingDecisions) + } _, failed := s.applySchedulingDecisions(ctx, schedulingDecisions) for _, decision := range failed { diff --git a/manager/scheduler/scheduler_test.go b/manager/scheduler/scheduler_test.go index 0d28632cce..e7bd7cf914 100644 --- a/manager/scheduler/scheduler_test.go +++ b/manager/scheduler/scheduler_test.go @@ -367,7 +367,7 @@ func TestScheduler(t *testing.T) { assert.NotEqual(t, "id6", assignment8.NodeID) } -func TestHA(t *testing.T) { +func testHA(t *testing.T, useSpecVersion bool) { ctx := context.Background() initialNodeSet := []*api.Node{ { @@ -432,6 +432,11 @@ func TestHA(t *testing.T) { }, } + if useSpecVersion { + taskTemplate1.SpecVersion = 1 + taskTemplate2.SpecVersion = 1 + } + s := store.NewMemoryStore(nil) assert.NotNil(t, s) defer s.Close() @@ -641,7 +646,12 @@ func TestHA(t *testing.T) { assert.Equal(t, 1, t2Assignments["id1"]) } -func TestPreferences(t *testing.T) { +func TestHA(t *testing.T) { + t.Run("useSpecVersion=false", func(t *testing.T) { testHA(t, false) }) + t.Run("useSpecVersion=true", func(t *testing.T) { testHA(t, true) }) +} + +func testPreferences(t *testing.T, useSpecVersion bool) { ctx := context.Background() initialNodeSet := []*api.Node{ { @@ -737,6 +747,10 @@ func TestPreferences(t *testing.T) { }, } + if useSpecVersion { + taskTemplate1.SpecVersion = 1 + } + s := store.NewMemoryStore(nil) assert.NotNil(t, s) defer s.Close() @@ -785,7 +799,12 @@ func TestPreferences(t *testing.T) { assert.Equal(t, 1, t1Assignments["id5"]) } -func TestMultiplePreferences(t *testing.T) { +func TestPreferences(t *testing.T) { + t.Run("useSpecVersion=false", func(t *testing.T) { testPreferences(t, false) }) + t.Run("useSpecVersion=true", func(t *testing.T) { testPreferences(t, true) }) +} + +func testMultiplePreferences(t *testing.T, useSpecVersion bool) { ctx := context.Background() initialNodeSet := []*api.Node{ { @@ -968,6 +987,10 @@ func TestMultiplePreferences(t *testing.T) { }, } + if useSpecVersion { + taskTemplate1.SpecVersion = 1 + } + s := store.NewMemoryStore(nil) assert.NotNil(t, s) defer s.Close() @@ -1057,6 +1080,11 @@ func TestMultiplePreferences(t *testing.T) { } } +func TestMultiplePreferences(t *testing.T) { + t.Run("useSpecVersion=false", func(t *testing.T) { testMultiplePreferences(t, false) }) + t.Run("useSpecVersion=true", func(t *testing.T) { testMultiplePreferences(t, true) }) +} + func TestSchedulerNoReadyNodes(t *testing.T) { ctx := context.Background() initialTask := &api.Task{