From 1231441065c69e088b401c74c0b2dd2247393d23 Mon Sep 17 00:00:00 2001 From: Chris Martin Date: Thu, 2 Jan 2025 12:59:57 +0000 Subject: [PATCH 01/17] Replace github.com/golang/mock with go.uber.org/mock (#4084) * remove mocks * update gomock * update tools * lint --- go.mod | 2 +- go.sum | 11 +- .../controlplaneevents/mock_publisher.go | 67 -------- internal/common/mocks/generate.go | 3 +- .../mocks/jobsetevents/mock_publisher.go | 67 -------- internal/common/mocks/mock_executorapi.go | 51 +++--- internal/common/mocks/mock_publisher.go | 73 ++++++++ internal/common/mocks/mock_pulsar.go | 46 ++--- internal/common/pulsarutils/publisher_test.go | 2 +- .../executor/service/lease_requester_test.go | 2 +- internal/scheduler/api_test.go | 12 +- internal/scheduler/leader/leader_test.go | 2 +- internal/scheduler/metrics_test.go | 2 +- internal/scheduler/mocks/api.go | 159 +++++++++--------- .../scheduler/mocks/executor_repository.go | 40 +++-- internal/scheduler/mocks/grpc.go | 36 ++-- internal/scheduler/mocks/job_repository.go | 56 +++--- internal/scheduler/mocks/leases_getter.go | 91 +++++----- internal/scheduler/mocks/queue_cache.go | 16 +- internal/scheduler/publisher_test.go | 2 +- internal/scheduler/queue/queue_cache_test.go | 2 +- .../scheduling/scheduling_algo_test.go | 2 +- internal/scheduler/submitcheck_test.go | 2 +- internal/server/mocks/generate.go | 5 +- internal/server/mocks/mock_authorizer.go | 26 +-- internal/server/mocks/mock_deduplicator.go | 26 +-- internal/server/mocks/mock_repository.go | 80 +++++---- internal/server/queue/queue_cache_test.go | 2 +- internal/server/submit/submit_test.go | 8 +- tools.yaml | 2 +- 30 files changed, 449 insertions(+), 446 deletions(-) delete mode 100644 internal/common/mocks/controlplaneevents/mock_publisher.go delete mode 100644 internal/common/mocks/jobsetevents/mock_publisher.go create mode 100644 internal/common/mocks/mock_publisher.go diff --git a/go.mod b/go.mod index c01b5c50f49..7ee53d24406 100644 --- a/go.mod +++ b/go.mod @@ -71,7 +71,6 @@ require ( github.com/go-openapi/validate v0.22.6 github.com/go-playground/validator/v10 v10.15.4 github.com/gogo/status v1.1.1 - github.com/golang/mock v1.6.0 github.com/goreleaser/goreleaser v1.24.0 github.com/jackc/pgx/v5 v5.5.4 github.com/jessevdk/go-flags v1.5.0 @@ -83,6 +82,7 @@ require ( github.com/redis/go-redis/v9 v9.7.0 github.com/segmentio/fasthash v1.0.3 github.com/xitongsys/parquet-go v1.6.2 + go.uber.org/mock v0.5.0 golang.org/x/term v0.25.0 golang.org/x/time v0.5.0 google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 diff --git a/go.sum b/go.sum index f63c61de172..0b215ee5d93 100644 --- a/go.sum +++ b/go.sum @@ -202,8 +202,6 @@ github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= -github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -526,7 +524,6 @@ github.com/xlab/treeprint v1.1.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.3.7/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/goldmark v1.5.4 h1:2uY/xC0roWy8IBEGLgB1ywIoEJFGmRrX21YQcvGZzjU= @@ -545,6 +542,8 @@ go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU= +go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= @@ -592,7 +591,6 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0= golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= @@ -617,7 +615,6 @@ golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= @@ -636,7 +633,6 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= @@ -664,9 +660,7 @@ golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210819135213-f52c844e1c1c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -728,7 +722,6 @@ golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/internal/common/mocks/controlplaneevents/mock_publisher.go b/internal/common/mocks/controlplaneevents/mock_publisher.go deleted file mode 100644 index 9b7e4fa41a5..00000000000 --- a/internal/common/mocks/controlplaneevents/mock_publisher.go +++ /dev/null @@ -1,67 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: github.com/armadaproject/armada/internal/common/pulsarutils/controlplaneevents (interfaces: Publisher) - -// Package controlplaneevents is a generated GoMock package. -package controlplaneevents - -import ( - reflect "reflect" - - armadacontext "github.com/armadaproject/armada/internal/common/armadacontext" - controlplaneevents "github.com/armadaproject/armada/pkg/controlplaneevents" - gomock "github.com/golang/mock/gomock" -) - -// MockPublisher is a mock of Publisher interface. -type MockPublisher struct { - ctrl *gomock.Controller - recorder *MockPublisherMockRecorder -} - -// MockPublisherMockRecorder is the mock recorder for MockPublisher. -type MockPublisherMockRecorder struct { - mock *MockPublisher -} - -// NewMockPublisher creates a new mock instance. -func NewMockPublisher(ctrl *gomock.Controller) *MockPublisher { - mock := &MockPublisher{ctrl: ctrl} - mock.recorder = &MockPublisherMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockPublisher) EXPECT() *MockPublisherMockRecorder { - return m.recorder -} - -// Close mocks base method. -func (m *MockPublisher) Close() { - m.ctrl.T.Helper() - m.ctrl.Call(m, "Close") -} - -// Close indicates an expected call of Close. -func (mr *MockPublisherMockRecorder) Close() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockPublisher)(nil).Close)) -} - -// PublishMessages mocks base method. -func (m *MockPublisher) PublishMessages(arg0 *armadacontext.Context, arg1 ...*controlplaneevents.Event) error { - m.ctrl.T.Helper() - varargs := []interface{}{arg0} - for _, a := range arg1 { - varargs = append(varargs, a) - } - ret := m.ctrl.Call(m, "PublishMessages", varargs...) - ret0, _ := ret[0].(error) - return ret0 -} - -// PublishMessages indicates an expected call of PublishMessages. -func (mr *MockPublisherMockRecorder) PublishMessages(arg0 interface{}, arg1 ...interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - varargs := append([]interface{}{arg0}, arg1...) - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PublishMessages", reflect.TypeOf((*MockPublisher)(nil).PublishMessages), varargs...) -} diff --git a/internal/common/mocks/generate.go b/internal/common/mocks/generate.go index 9f09f3ad54e..953b0532ffe 100644 --- a/internal/common/mocks/generate.go +++ b/internal/common/mocks/generate.go @@ -3,5 +3,4 @@ package mocks // Mock implementations used by tests //go:generate mockgen -destination=./mock_pulsar.go -package=mocks "github.com/apache/pulsar-client-go/pulsar" Client,Producer,Message //go:generate mockgen -destination=./mock_executorapi.go -package=mocks "github.com/armadaproject/armada/pkg/executorapi" ExecutorApiClient,ExecutorApi_LeaseJobRunsClient -//go:generate mockgen -destination=./controlplaneevents/mock_publisher.go -package=controlplaneevents "github.com/armadaproject/armada/internal/common/pulsarutils/controlplaneevents" Publisher -//go:generate mockgen -destination=./jobsetevents/mock_publisher.go -package=jobsetevents "github.com/armadaproject/armada/internal/common/pulsarutils/jobsetevents" Publisher +//go:generate mockgen -destination=./mock_publisher.go -package=mocks "github.com/armadaproject/armada/internal/common/pulsarutils" Publisher diff --git a/internal/common/mocks/jobsetevents/mock_publisher.go b/internal/common/mocks/jobsetevents/mock_publisher.go deleted file mode 100644 index 319ccb561a8..00000000000 --- a/internal/common/mocks/jobsetevents/mock_publisher.go +++ /dev/null @@ -1,67 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: github.com/armadaproject/armada/internal/common/pulsarutils/jobsetevents (interfaces: Publisher) - -// Package jobsetevents is a generated GoMock package. -package jobsetevents - -import ( - reflect "reflect" - - armadacontext "github.com/armadaproject/armada/internal/common/armadacontext" - armadaevents "github.com/armadaproject/armada/pkg/armadaevents" - gomock "github.com/golang/mock/gomock" -) - -// MockPublisher is a mock of Publisher interface. -type MockPublisher struct { - ctrl *gomock.Controller - recorder *MockPublisherMockRecorder -} - -// MockPublisherMockRecorder is the mock recorder for MockPublisher. -type MockPublisherMockRecorder struct { - mock *MockPublisher -} - -// NewMockPublisher creates a new mock instance. -func NewMockPublisher(ctrl *gomock.Controller) *MockPublisher { - mock := &MockPublisher{ctrl: ctrl} - mock.recorder = &MockPublisherMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockPublisher) EXPECT() *MockPublisherMockRecorder { - return m.recorder -} - -// Close mocks base method. -func (m *MockPublisher) Close() { - m.ctrl.T.Helper() - m.ctrl.Call(m, "Close") -} - -// Close indicates an expected call of Close. -func (mr *MockPublisherMockRecorder) Close() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockPublisher)(nil).Close)) -} - -// PublishMessages mocks base method. -func (m *MockPublisher) PublishMessages(arg0 *armadacontext.Context, arg1 ...*armadaevents.EventSequence) error { - m.ctrl.T.Helper() - varargs := []interface{}{arg0} - for _, a := range arg1 { - varargs = append(varargs, a) - } - ret := m.ctrl.Call(m, "PublishMessages", varargs...) - ret0, _ := ret[0].(error) - return ret0 -} - -// PublishMessages indicates an expected call of PublishMessages. -func (mr *MockPublisherMockRecorder) PublishMessages(arg0 interface{}, arg1 ...interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - varargs := append([]interface{}{arg0}, arg1...) - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PublishMessages", reflect.TypeOf((*MockPublisher)(nil).PublishMessages), varargs...) -} diff --git a/internal/common/mocks/mock_executorapi.go b/internal/common/mocks/mock_executorapi.go index 3d6081ead6d..b136f106371 100644 --- a/internal/common/mocks/mock_executorapi.go +++ b/internal/common/mocks/mock_executorapi.go @@ -1,5 +1,10 @@ // Code generated by MockGen. DO NOT EDIT. // Source: github.com/armadaproject/armada/pkg/executorapi (interfaces: ExecutorApiClient,ExecutorApi_LeaseJobRunsClient) +// +// Generated by this command: +// +// mockgen -destination=./mock_executorapi.go -package=mocks github.com/armadaproject/armada/pkg/executorapi ExecutorApiClient,ExecutorApi_LeaseJobRunsClient +// // Package mocks is a generated GoMock package. package mocks @@ -10,7 +15,7 @@ import ( executorapi "github.com/armadaproject/armada/pkg/executorapi" types "github.com/gogo/protobuf/types" - gomock "github.com/golang/mock/gomock" + gomock "go.uber.org/mock/gomock" grpc "google.golang.org/grpc" metadata "google.golang.org/grpc/metadata" ) @@ -19,6 +24,7 @@ import ( type MockExecutorApiClient struct { ctrl *gomock.Controller recorder *MockExecutorApiClientMockRecorder + isgomock struct{} } // MockExecutorApiClientMockRecorder is the mock recorder for MockExecutorApiClient. @@ -39,10 +45,10 @@ func (m *MockExecutorApiClient) EXPECT() *MockExecutorApiClientMockRecorder { } // LeaseJobRuns mocks base method. -func (m *MockExecutorApiClient) LeaseJobRuns(arg0 context.Context, arg1 ...grpc.CallOption) (executorapi.ExecutorApi_LeaseJobRunsClient, error) { +func (m *MockExecutorApiClient) LeaseJobRuns(ctx context.Context, opts ...grpc.CallOption) (executorapi.ExecutorApi_LeaseJobRunsClient, error) { m.ctrl.T.Helper() - varargs := []interface{}{arg0} - for _, a := range arg1 { + varargs := []any{ctx} + for _, a := range opts { varargs = append(varargs, a) } ret := m.ctrl.Call(m, "LeaseJobRuns", varargs...) @@ -52,17 +58,17 @@ func (m *MockExecutorApiClient) LeaseJobRuns(arg0 context.Context, arg1 ...grpc. } // LeaseJobRuns indicates an expected call of LeaseJobRuns. -func (mr *MockExecutorApiClientMockRecorder) LeaseJobRuns(arg0 interface{}, arg1 ...interface{}) *gomock.Call { +func (mr *MockExecutorApiClientMockRecorder) LeaseJobRuns(ctx any, opts ...any) *gomock.Call { mr.mock.ctrl.T.Helper() - varargs := append([]interface{}{arg0}, arg1...) + varargs := append([]any{ctx}, opts...) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LeaseJobRuns", reflect.TypeOf((*MockExecutorApiClient)(nil).LeaseJobRuns), varargs...) } // ReportEvents mocks base method. -func (m *MockExecutorApiClient) ReportEvents(arg0 context.Context, arg1 *executorapi.EventList, arg2 ...grpc.CallOption) (*types.Empty, error) { +func (m *MockExecutorApiClient) ReportEvents(ctx context.Context, in *executorapi.EventList, opts ...grpc.CallOption) (*types.Empty, error) { m.ctrl.T.Helper() - varargs := []interface{}{arg0, arg1} - for _, a := range arg2 { + varargs := []any{ctx, in} + for _, a := range opts { varargs = append(varargs, a) } ret := m.ctrl.Call(m, "ReportEvents", varargs...) @@ -72,9 +78,9 @@ func (m *MockExecutorApiClient) ReportEvents(arg0 context.Context, arg1 *executo } // ReportEvents indicates an expected call of ReportEvents. -func (mr *MockExecutorApiClientMockRecorder) ReportEvents(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { +func (mr *MockExecutorApiClientMockRecorder) ReportEvents(ctx, in any, opts ...any) *gomock.Call { mr.mock.ctrl.T.Helper() - varargs := append([]interface{}{arg0, arg1}, arg2...) + varargs := append([]any{ctx, in}, opts...) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReportEvents", reflect.TypeOf((*MockExecutorApiClient)(nil).ReportEvents), varargs...) } @@ -82,6 +88,7 @@ func (mr *MockExecutorApiClientMockRecorder) ReportEvents(arg0, arg1 interface{} type MockExecutorApi_LeaseJobRunsClient struct { ctrl *gomock.Controller recorder *MockExecutorApi_LeaseJobRunsClientMockRecorder + isgomock struct{} } // MockExecutorApi_LeaseJobRunsClientMockRecorder is the mock recorder for MockExecutorApi_LeaseJobRunsClient. @@ -160,17 +167,17 @@ func (mr *MockExecutorApi_LeaseJobRunsClientMockRecorder) Recv() *gomock.Call { } // RecvMsg mocks base method. -func (m *MockExecutorApi_LeaseJobRunsClient) RecvMsg(arg0 interface{}) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "RecvMsg", arg0) +func (m_2 *MockExecutorApi_LeaseJobRunsClient) RecvMsg(m any) error { + m_2.ctrl.T.Helper() + ret := m_2.ctrl.Call(m_2, "RecvMsg", m) ret0, _ := ret[0].(error) return ret0 } // RecvMsg indicates an expected call of RecvMsg. -func (mr *MockExecutorApi_LeaseJobRunsClientMockRecorder) RecvMsg(arg0 interface{}) *gomock.Call { +func (mr *MockExecutorApi_LeaseJobRunsClientMockRecorder) RecvMsg(m any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RecvMsg", reflect.TypeOf((*MockExecutorApi_LeaseJobRunsClient)(nil).RecvMsg), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RecvMsg", reflect.TypeOf((*MockExecutorApi_LeaseJobRunsClient)(nil).RecvMsg), m) } // Send mocks base method. @@ -182,23 +189,23 @@ func (m *MockExecutorApi_LeaseJobRunsClient) Send(arg0 *executorapi.LeaseRequest } // Send indicates an expected call of Send. -func (mr *MockExecutorApi_LeaseJobRunsClientMockRecorder) Send(arg0 interface{}) *gomock.Call { +func (mr *MockExecutorApi_LeaseJobRunsClientMockRecorder) Send(arg0 any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Send", reflect.TypeOf((*MockExecutorApi_LeaseJobRunsClient)(nil).Send), arg0) } // SendMsg mocks base method. -func (m *MockExecutorApi_LeaseJobRunsClient) SendMsg(arg0 interface{}) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SendMsg", arg0) +func (m_2 *MockExecutorApi_LeaseJobRunsClient) SendMsg(m any) error { + m_2.ctrl.T.Helper() + ret := m_2.ctrl.Call(m_2, "SendMsg", m) ret0, _ := ret[0].(error) return ret0 } // SendMsg indicates an expected call of SendMsg. -func (mr *MockExecutorApi_LeaseJobRunsClientMockRecorder) SendMsg(arg0 interface{}) *gomock.Call { +func (mr *MockExecutorApi_LeaseJobRunsClientMockRecorder) SendMsg(m any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendMsg", reflect.TypeOf((*MockExecutorApi_LeaseJobRunsClient)(nil).SendMsg), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendMsg", reflect.TypeOf((*MockExecutorApi_LeaseJobRunsClient)(nil).SendMsg), m) } // Trailer mocks base method. diff --git a/internal/common/mocks/mock_publisher.go b/internal/common/mocks/mock_publisher.go new file mode 100644 index 00000000000..4d17f29b723 --- /dev/null +++ b/internal/common/mocks/mock_publisher.go @@ -0,0 +1,73 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/armadaproject/armada/internal/common/pulsarutils (interfaces: Publisher) +// +// Generated by this command: +// +// mockgen -destination=./mock_publisher.go -package=mocks github.com/armadaproject/armada/internal/common/pulsarutils Publisher +// + +// Package mocks is a generated GoMock package. +package mocks + +import ( + reflect "reflect" + + armadacontext "github.com/armadaproject/armada/internal/common/armadacontext" + utils "github.com/armadaproject/armada/internal/common/ingest/utils" + gomock "go.uber.org/mock/gomock" +) + +// MockPublisher is a mock of Publisher interface. +type MockPublisher[T utils.ArmadaEvent] struct { + ctrl *gomock.Controller + recorder *MockPublisherMockRecorder[T] + isgomock struct{} +} + +// MockPublisherMockRecorder is the mock recorder for MockPublisher. +type MockPublisherMockRecorder[T utils.ArmadaEvent] struct { + mock *MockPublisher[T] +} + +// NewMockPublisher creates a new mock instance. +func NewMockPublisher[T utils.ArmadaEvent](ctrl *gomock.Controller) *MockPublisher[T] { + mock := &MockPublisher[T]{ctrl: ctrl} + mock.recorder = &MockPublisherMockRecorder[T]{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockPublisher[T]) EXPECT() *MockPublisherMockRecorder[T] { + return m.recorder +} + +// Close mocks base method. +func (m *MockPublisher[T]) Close() { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Close") +} + +// Close indicates an expected call of Close. +func (mr *MockPublisherMockRecorder[T]) Close() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockPublisher[T])(nil).Close)) +} + +// PublishMessages mocks base method. +func (m *MockPublisher[T]) PublishMessages(ctx *armadacontext.Context, events ...T) error { + m.ctrl.T.Helper() + varargs := []any{ctx} + for _, a := range events { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "PublishMessages", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// PublishMessages indicates an expected call of PublishMessages. +func (mr *MockPublisherMockRecorder[T]) PublishMessages(ctx any, events ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{ctx}, events...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PublishMessages", reflect.TypeOf((*MockPublisher[T])(nil).PublishMessages), varargs...) +} diff --git a/internal/common/mocks/mock_pulsar.go b/internal/common/mocks/mock_pulsar.go index 50de787b91a..910d9d315b1 100644 --- a/internal/common/mocks/mock_pulsar.go +++ b/internal/common/mocks/mock_pulsar.go @@ -1,5 +1,10 @@ // Code generated by MockGen. DO NOT EDIT. // Source: github.com/apache/pulsar-client-go/pulsar (interfaces: Client,Producer,Message) +// +// Generated by this command: +// +// mockgen -destination=./mock_pulsar.go -package=mocks github.com/apache/pulsar-client-go/pulsar Client,Producer,Message +// // Package mocks is a generated GoMock package. package mocks @@ -10,13 +15,14 @@ import ( time "time" pulsar "github.com/apache/pulsar-client-go/pulsar" - gomock "github.com/golang/mock/gomock" + gomock "go.uber.org/mock/gomock" ) // MockClient is a mock of Client interface. type MockClient struct { ctrl *gomock.Controller recorder *MockClientMockRecorder + isgomock struct{} } // MockClientMockRecorder is the mock recorder for MockClient. @@ -58,7 +64,7 @@ func (m *MockClient) CreateProducer(arg0 pulsar.ProducerOptions) (pulsar.Produce } // CreateProducer indicates an expected call of CreateProducer. -func (mr *MockClientMockRecorder) CreateProducer(arg0 interface{}) *gomock.Call { +func (mr *MockClientMockRecorder) CreateProducer(arg0 any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateProducer", reflect.TypeOf((*MockClient)(nil).CreateProducer), arg0) } @@ -73,7 +79,7 @@ func (m *MockClient) CreateReader(arg0 pulsar.ReaderOptions) (pulsar.Reader, err } // CreateReader indicates an expected call of CreateReader. -func (mr *MockClientMockRecorder) CreateReader(arg0 interface{}) *gomock.Call { +func (mr *MockClientMockRecorder) CreateReader(arg0 any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateReader", reflect.TypeOf((*MockClient)(nil).CreateReader), arg0) } @@ -88,24 +94,24 @@ func (m *MockClient) CreateTableView(arg0 pulsar.TableViewOptions) (pulsar.Table } // CreateTableView indicates an expected call of CreateTableView. -func (mr *MockClientMockRecorder) CreateTableView(arg0 interface{}) *gomock.Call { +func (mr *MockClientMockRecorder) CreateTableView(arg0 any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateTableView", reflect.TypeOf((*MockClient)(nil).CreateTableView), arg0) } // NewTransaction mocks base method. -func (m *MockClient) NewTransaction(arg0 time.Duration) (pulsar.Transaction, error) { +func (m *MockClient) NewTransaction(duration time.Duration) (pulsar.Transaction, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "NewTransaction", arg0) + ret := m.ctrl.Call(m, "NewTransaction", duration) ret0, _ := ret[0].(pulsar.Transaction) ret1, _ := ret[1].(error) return ret0, ret1 } // NewTransaction indicates an expected call of NewTransaction. -func (mr *MockClientMockRecorder) NewTransaction(arg0 interface{}) *gomock.Call { +func (mr *MockClientMockRecorder) NewTransaction(duration any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NewTransaction", reflect.TypeOf((*MockClient)(nil).NewTransaction), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NewTransaction", reflect.TypeOf((*MockClient)(nil).NewTransaction), duration) } // Subscribe mocks base method. @@ -118,30 +124,31 @@ func (m *MockClient) Subscribe(arg0 pulsar.ConsumerOptions) (pulsar.Consumer, er } // Subscribe indicates an expected call of Subscribe. -func (mr *MockClientMockRecorder) Subscribe(arg0 interface{}) *gomock.Call { +func (mr *MockClientMockRecorder) Subscribe(arg0 any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Subscribe", reflect.TypeOf((*MockClient)(nil).Subscribe), arg0) } // TopicPartitions mocks base method. -func (m *MockClient) TopicPartitions(arg0 string) ([]string, error) { +func (m *MockClient) TopicPartitions(topic string) ([]string, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "TopicPartitions", arg0) + ret := m.ctrl.Call(m, "TopicPartitions", topic) ret0, _ := ret[0].([]string) ret1, _ := ret[1].(error) return ret0, ret1 } // TopicPartitions indicates an expected call of TopicPartitions. -func (mr *MockClientMockRecorder) TopicPartitions(arg0 interface{}) *gomock.Call { +func (mr *MockClientMockRecorder) TopicPartitions(topic any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TopicPartitions", reflect.TypeOf((*MockClient)(nil).TopicPartitions), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TopicPartitions", reflect.TypeOf((*MockClient)(nil).TopicPartitions), topic) } // MockProducer is a mock of Producer interface. type MockProducer struct { ctrl *gomock.Controller recorder *MockProducerMockRecorder + isgomock struct{} } // MockProducerMockRecorder is the mock recorder for MockProducer. @@ -225,7 +232,7 @@ func (m *MockProducer) Send(arg0 context.Context, arg1 *pulsar.ProducerMessage) } // Send indicates an expected call of Send. -func (mr *MockProducerMockRecorder) Send(arg0, arg1 interface{}) *gomock.Call { +func (mr *MockProducerMockRecorder) Send(arg0, arg1 any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Send", reflect.TypeOf((*MockProducer)(nil).Send), arg0, arg1) } @@ -237,7 +244,7 @@ func (m *MockProducer) SendAsync(arg0 context.Context, arg1 *pulsar.ProducerMess } // SendAsync indicates an expected call of SendAsync. -func (mr *MockProducerMockRecorder) SendAsync(arg0, arg1, arg2 interface{}) *gomock.Call { +func (mr *MockProducerMockRecorder) SendAsync(arg0, arg1, arg2 any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendAsync", reflect.TypeOf((*MockProducer)(nil).SendAsync), arg0, arg1, arg2) } @@ -260,6 +267,7 @@ func (mr *MockProducerMockRecorder) Topic() *gomock.Call { type MockMessage struct { ctrl *gomock.Controller recorder *MockMessageMockRecorder + isgomock struct{} } // MockMessageMockRecorder is the mock recorder for MockMessage. @@ -336,17 +344,17 @@ func (mr *MockMessageMockRecorder) GetReplicatedFrom() *gomock.Call { } // GetSchemaValue mocks base method. -func (m *MockMessage) GetSchemaValue(arg0 interface{}) error { +func (m *MockMessage) GetSchemaValue(v any) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetSchemaValue", arg0) + ret := m.ctrl.Call(m, "GetSchemaValue", v) ret0, _ := ret[0].(error) return ret0 } // GetSchemaValue indicates an expected call of GetSchemaValue. -func (mr *MockMessageMockRecorder) GetSchemaValue(arg0 interface{}) *gomock.Call { +func (mr *MockMessageMockRecorder) GetSchemaValue(v any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSchemaValue", reflect.TypeOf((*MockMessage)(nil).GetSchemaValue), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSchemaValue", reflect.TypeOf((*MockMessage)(nil).GetSchemaValue), v) } // ID mocks base method. diff --git a/internal/common/pulsarutils/publisher_test.go b/internal/common/pulsarutils/publisher_test.go index be438930a4f..c5deae6cb5f 100644 --- a/internal/common/pulsarutils/publisher_test.go +++ b/internal/common/pulsarutils/publisher_test.go @@ -8,10 +8,10 @@ import ( "github.com/apache/pulsar-client-go/pulsar" "github.com/gogo/protobuf/proto" - "github.com/golang/mock/gomock" "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" "github.com/armadaproject/armada/internal/common/armadacontext" "github.com/armadaproject/armada/internal/common/armadaerrors" diff --git a/internal/executor/service/lease_requester_test.go b/internal/executor/service/lease_requester_test.go index 42be14db476..f0175b69821 100644 --- a/internal/executor/service/lease_requester_test.go +++ b/internal/executor/service/lease_requester_test.go @@ -6,9 +6,9 @@ import ( "testing" "time" - "github.com/golang/mock/gomock" "github.com/google/uuid" "github.com/stretchr/testify/assert" + "go.uber.org/mock/gomock" "k8s.io/apimachinery/pkg/api/resource" "github.com/armadaproject/armada/internal/common/armadacontext" diff --git a/internal/scheduler/api_test.go b/internal/scheduler/api_test.go index 913a950e08c..9094a48f55c 100644 --- a/internal/scheduler/api_test.go +++ b/internal/scheduler/api_test.go @@ -5,10 +5,10 @@ import ( "time" "github.com/gogo/protobuf/proto" - "github.com/golang/mock/gomock" "github.com/google/uuid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" v1 "k8s.io/api/core/v1" @@ -18,7 +18,7 @@ import ( "github.com/armadaproject/armada/internal/common/armadaerrors" "github.com/armadaproject/armada/internal/common/auth/permission" "github.com/armadaproject/armada/internal/common/compress" - mocks "github.com/armadaproject/armada/internal/common/mocks/jobsetevents" + mocks "github.com/armadaproject/armada/internal/common/mocks" protoutil "github.com/armadaproject/armada/internal/common/proto" "github.com/armadaproject/armada/internal/common/slices" "github.com/armadaproject/armada/internal/common/types" @@ -306,7 +306,7 @@ func TestExecutorApi_LeaseJobRuns(t *testing.T) { t.Run(name, func(t *testing.T) { ctx, cancel := armadacontext.WithTimeout(armadacontext.Background(), 5*time.Second) ctrl := gomock.NewController(t) - mockPulsarPublisher := mocks.NewMockPublisher(ctrl) + mockPulsarPublisher := mocks.NewMockPublisher[*armadaevents.EventSequence](ctrl) mockJobRepository := schedulermocks.NewMockJobRepository(ctrl) mockExecutorRepository := schedulermocks.NewMockExecutorRepository(ctrl) mockStream := schedulermocks.NewMockExecutorApi_LeaseJobRunsServer(ctrl) @@ -373,7 +373,7 @@ func TestExecutorApi_LeaseJobRuns_Unauthorised(t *testing.T) { ctx, cancel := armadacontext.WithTimeout(armadacontext.Background(), 5*time.Second) defer cancel() ctrl := gomock.NewController(t) - mockPulsarPublisher := mocks.NewMockPublisher(ctrl) + mockPulsarPublisher := mocks.NewMockPublisher[*armadaevents.EventSequence](ctrl) mockJobRepository := schedulermocks.NewMockJobRepository(ctrl) mockExecutorRepository := schedulermocks.NewMockExecutorRepository(ctrl) mockStream := schedulermocks.NewMockExecutorApi_LeaseJobRunsServer(ctrl) @@ -501,7 +501,7 @@ func TestExecutorApi_Publish(t *testing.T) { t.Run(name, func(t *testing.T) { ctx, cancel := armadacontext.WithTimeout(armadacontext.Background(), 5*time.Second) ctrl := gomock.NewController(t) - mockPulsarPublisher := mocks.NewMockPublisher(ctrl) + mockPulsarPublisher := mocks.NewMockPublisher[*armadaevents.EventSequence](ctrl) mockJobRepository := schedulermocks.NewMockJobRepository(ctrl) mockExecutorRepository := schedulermocks.NewMockExecutorRepository(ctrl) mockAuthorizer := mocks2.NewMockActionAuthorizer(ctrl) @@ -546,7 +546,7 @@ func TestExecutorApi_Publish_Unauthorised(t *testing.T) { ctx, cancel := armadacontext.WithTimeout(armadacontext.Background(), 5*time.Second) defer cancel() ctrl := gomock.NewController(t) - mockPulsarPublisher := mocks.NewMockPublisher(ctrl) + mockPulsarPublisher := mocks.NewMockPublisher[*armadaevents.EventSequence](ctrl) mockJobRepository := schedulermocks.NewMockJobRepository(ctrl) mockExecutorRepository := schedulermocks.NewMockExecutorRepository(ctrl) mockAuthorizer := mocks2.NewMockActionAuthorizer(ctrl) diff --git a/internal/scheduler/leader/leader_test.go b/internal/scheduler/leader/leader_test.go index f5f65d40c0a..ba02e9dcc67 100644 --- a/internal/scheduler/leader/leader_test.go +++ b/internal/scheduler/leader/leader_test.go @@ -6,9 +6,9 @@ import ( "testing" "time" - "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" v1 "k8s.io/api/coordination/v1" "k8s.io/utils/pointer" diff --git a/internal/scheduler/metrics_test.go b/internal/scheduler/metrics_test.go index c2f7d77c117..b67c29598ab 100644 --- a/internal/scheduler/metrics_test.go +++ b/internal/scheduler/metrics_test.go @@ -4,11 +4,11 @@ import ( "testing" "time" - "github.com/golang/mock/gomock" "github.com/google/uuid" "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" "k8s.io/apimachinery/pkg/api/resource" clock "k8s.io/utils/clock/testing" diff --git a/internal/scheduler/mocks/api.go b/internal/scheduler/mocks/api.go index 7b846a53c83..1e1ec8d5192 100644 --- a/internal/scheduler/mocks/api.go +++ b/internal/scheduler/mocks/api.go @@ -1,5 +1,10 @@ // Code generated by MockGen. DO NOT EDIT. // Source: github.com/armadaproject/armada/pkg/api (interfaces: SubmitClient,Submit_GetQueuesClient) +// +// Generated by this command: +// +// mockgen -destination=./api.go -package=schedulermocks github.com/armadaproject/armada/pkg/api SubmitClient,Submit_GetQueuesClient +// // Package schedulermocks is a generated GoMock package. package schedulermocks @@ -10,7 +15,7 @@ import ( api "github.com/armadaproject/armada/pkg/api" types "github.com/gogo/protobuf/types" - gomock "github.com/golang/mock/gomock" + gomock "go.uber.org/mock/gomock" grpc "google.golang.org/grpc" metadata "google.golang.org/grpc/metadata" ) @@ -19,6 +24,7 @@ import ( type MockSubmitClient struct { ctrl *gomock.Controller recorder *MockSubmitClientMockRecorder + isgomock struct{} } // MockSubmitClientMockRecorder is the mock recorder for MockSubmitClient. @@ -39,10 +45,10 @@ func (m *MockSubmitClient) EXPECT() *MockSubmitClientMockRecorder { } // CancelJobSet mocks base method. -func (m *MockSubmitClient) CancelJobSet(arg0 context.Context, arg1 *api.JobSetCancelRequest, arg2 ...grpc.CallOption) (*types.Empty, error) { +func (m *MockSubmitClient) CancelJobSet(ctx context.Context, in *api.JobSetCancelRequest, opts ...grpc.CallOption) (*types.Empty, error) { m.ctrl.T.Helper() - varargs := []interface{}{arg0, arg1} - for _, a := range arg2 { + varargs := []any{ctx, in} + for _, a := range opts { varargs = append(varargs, a) } ret := m.ctrl.Call(m, "CancelJobSet", varargs...) @@ -52,17 +58,17 @@ func (m *MockSubmitClient) CancelJobSet(arg0 context.Context, arg1 *api.JobSetCa } // CancelJobSet indicates an expected call of CancelJobSet. -func (mr *MockSubmitClientMockRecorder) CancelJobSet(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { +func (mr *MockSubmitClientMockRecorder) CancelJobSet(ctx, in any, opts ...any) *gomock.Call { mr.mock.ctrl.T.Helper() - varargs := append([]interface{}{arg0, arg1}, arg2...) + varargs := append([]any{ctx, in}, opts...) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CancelJobSet", reflect.TypeOf((*MockSubmitClient)(nil).CancelJobSet), varargs...) } // CancelJobs mocks base method. -func (m *MockSubmitClient) CancelJobs(arg0 context.Context, arg1 *api.JobCancelRequest, arg2 ...grpc.CallOption) (*api.CancellationResult, error) { +func (m *MockSubmitClient) CancelJobs(ctx context.Context, in *api.JobCancelRequest, opts ...grpc.CallOption) (*api.CancellationResult, error) { m.ctrl.T.Helper() - varargs := []interface{}{arg0, arg1} - for _, a := range arg2 { + varargs := []any{ctx, in} + for _, a := range opts { varargs = append(varargs, a) } ret := m.ctrl.Call(m, "CancelJobs", varargs...) @@ -72,17 +78,17 @@ func (m *MockSubmitClient) CancelJobs(arg0 context.Context, arg1 *api.JobCancelR } // CancelJobs indicates an expected call of CancelJobs. -func (mr *MockSubmitClientMockRecorder) CancelJobs(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { +func (mr *MockSubmitClientMockRecorder) CancelJobs(ctx, in any, opts ...any) *gomock.Call { mr.mock.ctrl.T.Helper() - varargs := append([]interface{}{arg0, arg1}, arg2...) + varargs := append([]any{ctx, in}, opts...) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CancelJobs", reflect.TypeOf((*MockSubmitClient)(nil).CancelJobs), varargs...) } // CreateQueue mocks base method. -func (m *MockSubmitClient) CreateQueue(arg0 context.Context, arg1 *api.Queue, arg2 ...grpc.CallOption) (*types.Empty, error) { +func (m *MockSubmitClient) CreateQueue(ctx context.Context, in *api.Queue, opts ...grpc.CallOption) (*types.Empty, error) { m.ctrl.T.Helper() - varargs := []interface{}{arg0, arg1} - for _, a := range arg2 { + varargs := []any{ctx, in} + for _, a := range opts { varargs = append(varargs, a) } ret := m.ctrl.Call(m, "CreateQueue", varargs...) @@ -92,17 +98,17 @@ func (m *MockSubmitClient) CreateQueue(arg0 context.Context, arg1 *api.Queue, ar } // CreateQueue indicates an expected call of CreateQueue. -func (mr *MockSubmitClientMockRecorder) CreateQueue(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { +func (mr *MockSubmitClientMockRecorder) CreateQueue(ctx, in any, opts ...any) *gomock.Call { mr.mock.ctrl.T.Helper() - varargs := append([]interface{}{arg0, arg1}, arg2...) + varargs := append([]any{ctx, in}, opts...) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateQueue", reflect.TypeOf((*MockSubmitClient)(nil).CreateQueue), varargs...) } // CreateQueues mocks base method. -func (m *MockSubmitClient) CreateQueues(arg0 context.Context, arg1 *api.QueueList, arg2 ...grpc.CallOption) (*api.BatchQueueCreateResponse, error) { +func (m *MockSubmitClient) CreateQueues(ctx context.Context, in *api.QueueList, opts ...grpc.CallOption) (*api.BatchQueueCreateResponse, error) { m.ctrl.T.Helper() - varargs := []interface{}{arg0, arg1} - for _, a := range arg2 { + varargs := []any{ctx, in} + for _, a := range opts { varargs = append(varargs, a) } ret := m.ctrl.Call(m, "CreateQueues", varargs...) @@ -112,17 +118,17 @@ func (m *MockSubmitClient) CreateQueues(arg0 context.Context, arg1 *api.QueueLis } // CreateQueues indicates an expected call of CreateQueues. -func (mr *MockSubmitClientMockRecorder) CreateQueues(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { +func (mr *MockSubmitClientMockRecorder) CreateQueues(ctx, in any, opts ...any) *gomock.Call { mr.mock.ctrl.T.Helper() - varargs := append([]interface{}{arg0, arg1}, arg2...) + varargs := append([]any{ctx, in}, opts...) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateQueues", reflect.TypeOf((*MockSubmitClient)(nil).CreateQueues), varargs...) } // DeleteQueue mocks base method. -func (m *MockSubmitClient) DeleteQueue(arg0 context.Context, arg1 *api.QueueDeleteRequest, arg2 ...grpc.CallOption) (*types.Empty, error) { +func (m *MockSubmitClient) DeleteQueue(ctx context.Context, in *api.QueueDeleteRequest, opts ...grpc.CallOption) (*types.Empty, error) { m.ctrl.T.Helper() - varargs := []interface{}{arg0, arg1} - for _, a := range arg2 { + varargs := []any{ctx, in} + for _, a := range opts { varargs = append(varargs, a) } ret := m.ctrl.Call(m, "DeleteQueue", varargs...) @@ -132,17 +138,17 @@ func (m *MockSubmitClient) DeleteQueue(arg0 context.Context, arg1 *api.QueueDele } // DeleteQueue indicates an expected call of DeleteQueue. -func (mr *MockSubmitClientMockRecorder) DeleteQueue(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { +func (mr *MockSubmitClientMockRecorder) DeleteQueue(ctx, in any, opts ...any) *gomock.Call { mr.mock.ctrl.T.Helper() - varargs := append([]interface{}{arg0, arg1}, arg2...) + varargs := append([]any{ctx, in}, opts...) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteQueue", reflect.TypeOf((*MockSubmitClient)(nil).DeleteQueue), varargs...) } // GetQueue mocks base method. -func (m *MockSubmitClient) GetQueue(arg0 context.Context, arg1 *api.QueueGetRequest, arg2 ...grpc.CallOption) (*api.Queue, error) { +func (m *MockSubmitClient) GetQueue(ctx context.Context, in *api.QueueGetRequest, opts ...grpc.CallOption) (*api.Queue, error) { m.ctrl.T.Helper() - varargs := []interface{}{arg0, arg1} - for _, a := range arg2 { + varargs := []any{ctx, in} + for _, a := range opts { varargs = append(varargs, a) } ret := m.ctrl.Call(m, "GetQueue", varargs...) @@ -152,17 +158,17 @@ func (m *MockSubmitClient) GetQueue(arg0 context.Context, arg1 *api.QueueGetRequ } // GetQueue indicates an expected call of GetQueue. -func (mr *MockSubmitClientMockRecorder) GetQueue(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { +func (mr *MockSubmitClientMockRecorder) GetQueue(ctx, in any, opts ...any) *gomock.Call { mr.mock.ctrl.T.Helper() - varargs := append([]interface{}{arg0, arg1}, arg2...) + varargs := append([]any{ctx, in}, opts...) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetQueue", reflect.TypeOf((*MockSubmitClient)(nil).GetQueue), varargs...) } // GetQueues mocks base method. -func (m *MockSubmitClient) GetQueues(arg0 context.Context, arg1 *api.StreamingQueueGetRequest, arg2 ...grpc.CallOption) (api.Submit_GetQueuesClient, error) { +func (m *MockSubmitClient) GetQueues(ctx context.Context, in *api.StreamingQueueGetRequest, opts ...grpc.CallOption) (api.Submit_GetQueuesClient, error) { m.ctrl.T.Helper() - varargs := []interface{}{arg0, arg1} - for _, a := range arg2 { + varargs := []any{ctx, in} + for _, a := range opts { varargs = append(varargs, a) } ret := m.ctrl.Call(m, "GetQueues", varargs...) @@ -172,17 +178,17 @@ func (m *MockSubmitClient) GetQueues(arg0 context.Context, arg1 *api.StreamingQu } // GetQueues indicates an expected call of GetQueues. -func (mr *MockSubmitClientMockRecorder) GetQueues(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { +func (mr *MockSubmitClientMockRecorder) GetQueues(ctx, in any, opts ...any) *gomock.Call { mr.mock.ctrl.T.Helper() - varargs := append([]interface{}{arg0, arg1}, arg2...) + varargs := append([]any{ctx, in}, opts...) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetQueues", reflect.TypeOf((*MockSubmitClient)(nil).GetQueues), varargs...) } // Health mocks base method. -func (m *MockSubmitClient) Health(arg0 context.Context, arg1 *types.Empty, arg2 ...grpc.CallOption) (*api.HealthCheckResponse, error) { +func (m *MockSubmitClient) Health(ctx context.Context, in *types.Empty, opts ...grpc.CallOption) (*api.HealthCheckResponse, error) { m.ctrl.T.Helper() - varargs := []interface{}{arg0, arg1} - for _, a := range arg2 { + varargs := []any{ctx, in} + for _, a := range opts { varargs = append(varargs, a) } ret := m.ctrl.Call(m, "Health", varargs...) @@ -192,17 +198,17 @@ func (m *MockSubmitClient) Health(arg0 context.Context, arg1 *types.Empty, arg2 } // Health indicates an expected call of Health. -func (mr *MockSubmitClientMockRecorder) Health(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { +func (mr *MockSubmitClientMockRecorder) Health(ctx, in any, opts ...any) *gomock.Call { mr.mock.ctrl.T.Helper() - varargs := append([]interface{}{arg0, arg1}, arg2...) + varargs := append([]any{ctx, in}, opts...) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Health", reflect.TypeOf((*MockSubmitClient)(nil).Health), varargs...) } // PreemptJobs mocks base method. -func (m *MockSubmitClient) PreemptJobs(arg0 context.Context, arg1 *api.JobPreemptRequest, arg2 ...grpc.CallOption) (*types.Empty, error) { +func (m *MockSubmitClient) PreemptJobs(ctx context.Context, in *api.JobPreemptRequest, opts ...grpc.CallOption) (*types.Empty, error) { m.ctrl.T.Helper() - varargs := []interface{}{arg0, arg1} - for _, a := range arg2 { + varargs := []any{ctx, in} + for _, a := range opts { varargs = append(varargs, a) } ret := m.ctrl.Call(m, "PreemptJobs", varargs...) @@ -212,17 +218,17 @@ func (m *MockSubmitClient) PreemptJobs(arg0 context.Context, arg1 *api.JobPreemp } // PreemptJobs indicates an expected call of PreemptJobs. -func (mr *MockSubmitClientMockRecorder) PreemptJobs(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { +func (mr *MockSubmitClientMockRecorder) PreemptJobs(ctx, in any, opts ...any) *gomock.Call { mr.mock.ctrl.T.Helper() - varargs := append([]interface{}{arg0, arg1}, arg2...) + varargs := append([]any{ctx, in}, opts...) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PreemptJobs", reflect.TypeOf((*MockSubmitClient)(nil).PreemptJobs), varargs...) } // ReprioritizeJobs mocks base method. -func (m *MockSubmitClient) ReprioritizeJobs(arg0 context.Context, arg1 *api.JobReprioritizeRequest, arg2 ...grpc.CallOption) (*api.JobReprioritizeResponse, error) { +func (m *MockSubmitClient) ReprioritizeJobs(ctx context.Context, in *api.JobReprioritizeRequest, opts ...grpc.CallOption) (*api.JobReprioritizeResponse, error) { m.ctrl.T.Helper() - varargs := []interface{}{arg0, arg1} - for _, a := range arg2 { + varargs := []any{ctx, in} + for _, a := range opts { varargs = append(varargs, a) } ret := m.ctrl.Call(m, "ReprioritizeJobs", varargs...) @@ -232,17 +238,17 @@ func (m *MockSubmitClient) ReprioritizeJobs(arg0 context.Context, arg1 *api.JobR } // ReprioritizeJobs indicates an expected call of ReprioritizeJobs. -func (mr *MockSubmitClientMockRecorder) ReprioritizeJobs(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { +func (mr *MockSubmitClientMockRecorder) ReprioritizeJobs(ctx, in any, opts ...any) *gomock.Call { mr.mock.ctrl.T.Helper() - varargs := append([]interface{}{arg0, arg1}, arg2...) + varargs := append([]any{ctx, in}, opts...) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReprioritizeJobs", reflect.TypeOf((*MockSubmitClient)(nil).ReprioritizeJobs), varargs...) } // SubmitJobs mocks base method. -func (m *MockSubmitClient) SubmitJobs(arg0 context.Context, arg1 *api.JobSubmitRequest, arg2 ...grpc.CallOption) (*api.JobSubmitResponse, error) { +func (m *MockSubmitClient) SubmitJobs(ctx context.Context, in *api.JobSubmitRequest, opts ...grpc.CallOption) (*api.JobSubmitResponse, error) { m.ctrl.T.Helper() - varargs := []interface{}{arg0, arg1} - for _, a := range arg2 { + varargs := []any{ctx, in} + for _, a := range opts { varargs = append(varargs, a) } ret := m.ctrl.Call(m, "SubmitJobs", varargs...) @@ -252,17 +258,17 @@ func (m *MockSubmitClient) SubmitJobs(arg0 context.Context, arg1 *api.JobSubmitR } // SubmitJobs indicates an expected call of SubmitJobs. -func (mr *MockSubmitClientMockRecorder) SubmitJobs(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { +func (mr *MockSubmitClientMockRecorder) SubmitJobs(ctx, in any, opts ...any) *gomock.Call { mr.mock.ctrl.T.Helper() - varargs := append([]interface{}{arg0, arg1}, arg2...) + varargs := append([]any{ctx, in}, opts...) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SubmitJobs", reflect.TypeOf((*MockSubmitClient)(nil).SubmitJobs), varargs...) } // UpdateQueue mocks base method. -func (m *MockSubmitClient) UpdateQueue(arg0 context.Context, arg1 *api.Queue, arg2 ...grpc.CallOption) (*types.Empty, error) { +func (m *MockSubmitClient) UpdateQueue(ctx context.Context, in *api.Queue, opts ...grpc.CallOption) (*types.Empty, error) { m.ctrl.T.Helper() - varargs := []interface{}{arg0, arg1} - for _, a := range arg2 { + varargs := []any{ctx, in} + for _, a := range opts { varargs = append(varargs, a) } ret := m.ctrl.Call(m, "UpdateQueue", varargs...) @@ -272,17 +278,17 @@ func (m *MockSubmitClient) UpdateQueue(arg0 context.Context, arg1 *api.Queue, ar } // UpdateQueue indicates an expected call of UpdateQueue. -func (mr *MockSubmitClientMockRecorder) UpdateQueue(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { +func (mr *MockSubmitClientMockRecorder) UpdateQueue(ctx, in any, opts ...any) *gomock.Call { mr.mock.ctrl.T.Helper() - varargs := append([]interface{}{arg0, arg1}, arg2...) + varargs := append([]any{ctx, in}, opts...) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateQueue", reflect.TypeOf((*MockSubmitClient)(nil).UpdateQueue), varargs...) } // UpdateQueues mocks base method. -func (m *MockSubmitClient) UpdateQueues(arg0 context.Context, arg1 *api.QueueList, arg2 ...grpc.CallOption) (*api.BatchQueueUpdateResponse, error) { +func (m *MockSubmitClient) UpdateQueues(ctx context.Context, in *api.QueueList, opts ...grpc.CallOption) (*api.BatchQueueUpdateResponse, error) { m.ctrl.T.Helper() - varargs := []interface{}{arg0, arg1} - for _, a := range arg2 { + varargs := []any{ctx, in} + for _, a := range opts { varargs = append(varargs, a) } ret := m.ctrl.Call(m, "UpdateQueues", varargs...) @@ -292,9 +298,9 @@ func (m *MockSubmitClient) UpdateQueues(arg0 context.Context, arg1 *api.QueueLis } // UpdateQueues indicates an expected call of UpdateQueues. -func (mr *MockSubmitClientMockRecorder) UpdateQueues(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { +func (mr *MockSubmitClientMockRecorder) UpdateQueues(ctx, in any, opts ...any) *gomock.Call { mr.mock.ctrl.T.Helper() - varargs := append([]interface{}{arg0, arg1}, arg2...) + varargs := append([]any{ctx, in}, opts...) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateQueues", reflect.TypeOf((*MockSubmitClient)(nil).UpdateQueues), varargs...) } @@ -302,6 +308,7 @@ func (mr *MockSubmitClientMockRecorder) UpdateQueues(arg0, arg1 interface{}, arg type MockSubmit_GetQueuesClient struct { ctrl *gomock.Controller recorder *MockSubmit_GetQueuesClientMockRecorder + isgomock struct{} } // MockSubmit_GetQueuesClientMockRecorder is the mock recorder for MockSubmit_GetQueuesClient. @@ -380,31 +387,31 @@ func (mr *MockSubmit_GetQueuesClientMockRecorder) Recv() *gomock.Call { } // RecvMsg mocks base method. -func (m *MockSubmit_GetQueuesClient) RecvMsg(arg0 interface{}) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "RecvMsg", arg0) +func (m_2 *MockSubmit_GetQueuesClient) RecvMsg(m any) error { + m_2.ctrl.T.Helper() + ret := m_2.ctrl.Call(m_2, "RecvMsg", m) ret0, _ := ret[0].(error) return ret0 } // RecvMsg indicates an expected call of RecvMsg. -func (mr *MockSubmit_GetQueuesClientMockRecorder) RecvMsg(arg0 interface{}) *gomock.Call { +func (mr *MockSubmit_GetQueuesClientMockRecorder) RecvMsg(m any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RecvMsg", reflect.TypeOf((*MockSubmit_GetQueuesClient)(nil).RecvMsg), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RecvMsg", reflect.TypeOf((*MockSubmit_GetQueuesClient)(nil).RecvMsg), m) } // SendMsg mocks base method. -func (m *MockSubmit_GetQueuesClient) SendMsg(arg0 interface{}) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SendMsg", arg0) +func (m_2 *MockSubmit_GetQueuesClient) SendMsg(m any) error { + m_2.ctrl.T.Helper() + ret := m_2.ctrl.Call(m_2, "SendMsg", m) ret0, _ := ret[0].(error) return ret0 } // SendMsg indicates an expected call of SendMsg. -func (mr *MockSubmit_GetQueuesClientMockRecorder) SendMsg(arg0 interface{}) *gomock.Call { +func (mr *MockSubmit_GetQueuesClientMockRecorder) SendMsg(m any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendMsg", reflect.TypeOf((*MockSubmit_GetQueuesClient)(nil).SendMsg), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendMsg", reflect.TypeOf((*MockSubmit_GetQueuesClient)(nil).SendMsg), m) } // Trailer mocks base method. diff --git a/internal/scheduler/mocks/executor_repository.go b/internal/scheduler/mocks/executor_repository.go index b780c1dbe0c..f7022a24c4f 100644 --- a/internal/scheduler/mocks/executor_repository.go +++ b/internal/scheduler/mocks/executor_repository.go @@ -1,5 +1,10 @@ // Code generated by MockGen. DO NOT EDIT. // Source: github.com/armadaproject/armada/internal/scheduler/database (interfaces: ExecutorRepository) +// +// Generated by this command: +// +// mockgen -destination=./executor_repository.go -package=schedulermocks github.com/armadaproject/armada/internal/scheduler/database ExecutorRepository +// // Package schedulermocks is a generated GoMock package. package schedulermocks @@ -10,13 +15,14 @@ import ( armadacontext "github.com/armadaproject/armada/internal/common/armadacontext" schedulerobjects "github.com/armadaproject/armada/internal/scheduler/schedulerobjects" - gomock "github.com/golang/mock/gomock" + gomock "go.uber.org/mock/gomock" ) // MockExecutorRepository is a mock of ExecutorRepository interface. type MockExecutorRepository struct { ctrl *gomock.Controller recorder *MockExecutorRepositoryMockRecorder + isgomock struct{} } // MockExecutorRepositoryMockRecorder is the mock recorder for MockExecutorRepository. @@ -37,60 +43,60 @@ func (m *MockExecutorRepository) EXPECT() *MockExecutorRepositoryMockRecorder { } // GetExecutorSettings mocks base method. -func (m *MockExecutorRepository) GetExecutorSettings(arg0 *armadacontext.Context) ([]*schedulerobjects.ExecutorSettings, error) { +func (m *MockExecutorRepository) GetExecutorSettings(ctx *armadacontext.Context) ([]*schedulerobjects.ExecutorSettings, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetExecutorSettings", arg0) + ret := m.ctrl.Call(m, "GetExecutorSettings", ctx) ret0, _ := ret[0].([]*schedulerobjects.ExecutorSettings) ret1, _ := ret[1].(error) return ret0, ret1 } // GetExecutorSettings indicates an expected call of GetExecutorSettings. -func (mr *MockExecutorRepositoryMockRecorder) GetExecutorSettings(arg0 interface{}) *gomock.Call { +func (mr *MockExecutorRepositoryMockRecorder) GetExecutorSettings(ctx any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetExecutorSettings", reflect.TypeOf((*MockExecutorRepository)(nil).GetExecutorSettings), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetExecutorSettings", reflect.TypeOf((*MockExecutorRepository)(nil).GetExecutorSettings), ctx) } // GetExecutors mocks base method. -func (m *MockExecutorRepository) GetExecutors(arg0 *armadacontext.Context) ([]*schedulerobjects.Executor, error) { +func (m *MockExecutorRepository) GetExecutors(ctx *armadacontext.Context) ([]*schedulerobjects.Executor, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetExecutors", arg0) + ret := m.ctrl.Call(m, "GetExecutors", ctx) ret0, _ := ret[0].([]*schedulerobjects.Executor) ret1, _ := ret[1].(error) return ret0, ret1 } // GetExecutors indicates an expected call of GetExecutors. -func (mr *MockExecutorRepositoryMockRecorder) GetExecutors(arg0 interface{}) *gomock.Call { +func (mr *MockExecutorRepositoryMockRecorder) GetExecutors(ctx any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetExecutors", reflect.TypeOf((*MockExecutorRepository)(nil).GetExecutors), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetExecutors", reflect.TypeOf((*MockExecutorRepository)(nil).GetExecutors), ctx) } // GetLastUpdateTimes mocks base method. -func (m *MockExecutorRepository) GetLastUpdateTimes(arg0 *armadacontext.Context) (map[string]time.Time, error) { +func (m *MockExecutorRepository) GetLastUpdateTimes(ctx *armadacontext.Context) (map[string]time.Time, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetLastUpdateTimes", arg0) + ret := m.ctrl.Call(m, "GetLastUpdateTimes", ctx) ret0, _ := ret[0].(map[string]time.Time) ret1, _ := ret[1].(error) return ret0, ret1 } // GetLastUpdateTimes indicates an expected call of GetLastUpdateTimes. -func (mr *MockExecutorRepositoryMockRecorder) GetLastUpdateTimes(arg0 interface{}) *gomock.Call { +func (mr *MockExecutorRepositoryMockRecorder) GetLastUpdateTimes(ctx any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLastUpdateTimes", reflect.TypeOf((*MockExecutorRepository)(nil).GetLastUpdateTimes), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLastUpdateTimes", reflect.TypeOf((*MockExecutorRepository)(nil).GetLastUpdateTimes), ctx) } // StoreExecutor mocks base method. -func (m *MockExecutorRepository) StoreExecutor(arg0 *armadacontext.Context, arg1 *schedulerobjects.Executor) error { +func (m *MockExecutorRepository) StoreExecutor(ctx *armadacontext.Context, executor *schedulerobjects.Executor) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "StoreExecutor", arg0, arg1) + ret := m.ctrl.Call(m, "StoreExecutor", ctx, executor) ret0, _ := ret[0].(error) return ret0 } // StoreExecutor indicates an expected call of StoreExecutor. -func (mr *MockExecutorRepositoryMockRecorder) StoreExecutor(arg0, arg1 interface{}) *gomock.Call { +func (mr *MockExecutorRepositoryMockRecorder) StoreExecutor(ctx, executor any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StoreExecutor", reflect.TypeOf((*MockExecutorRepository)(nil).StoreExecutor), arg0, arg1) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StoreExecutor", reflect.TypeOf((*MockExecutorRepository)(nil).StoreExecutor), ctx, executor) } diff --git a/internal/scheduler/mocks/grpc.go b/internal/scheduler/mocks/grpc.go index 618f4b7debe..0fd7e6881c2 100644 --- a/internal/scheduler/mocks/grpc.go +++ b/internal/scheduler/mocks/grpc.go @@ -1,5 +1,10 @@ // Code generated by MockGen. DO NOT EDIT. // Source: github.com/armadaproject/armada/pkg/executorapi (interfaces: ExecutorApi_LeaseJobRunsServer) +// +// Generated by this command: +// +// mockgen -destination=./grpc.go -package=schedulermocks github.com/armadaproject/armada/pkg/executorapi ExecutorApi_LeaseJobRunsServer +// // Package schedulermocks is a generated GoMock package. package schedulermocks @@ -9,7 +14,7 @@ import ( reflect "reflect" executorapi "github.com/armadaproject/armada/pkg/executorapi" - gomock "github.com/golang/mock/gomock" + gomock "go.uber.org/mock/gomock" metadata "google.golang.org/grpc/metadata" ) @@ -17,6 +22,7 @@ import ( type MockExecutorApi_LeaseJobRunsServer struct { ctrl *gomock.Controller recorder *MockExecutorApi_LeaseJobRunsServerMockRecorder + isgomock struct{} } // MockExecutorApi_LeaseJobRunsServerMockRecorder is the mock recorder for MockExecutorApi_LeaseJobRunsServer. @@ -66,17 +72,17 @@ func (mr *MockExecutorApi_LeaseJobRunsServerMockRecorder) Recv() *gomock.Call { } // RecvMsg mocks base method. -func (m *MockExecutorApi_LeaseJobRunsServer) RecvMsg(arg0 interface{}) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "RecvMsg", arg0) +func (m_2 *MockExecutorApi_LeaseJobRunsServer) RecvMsg(m any) error { + m_2.ctrl.T.Helper() + ret := m_2.ctrl.Call(m_2, "RecvMsg", m) ret0, _ := ret[0].(error) return ret0 } // RecvMsg indicates an expected call of RecvMsg. -func (mr *MockExecutorApi_LeaseJobRunsServerMockRecorder) RecvMsg(arg0 interface{}) *gomock.Call { +func (mr *MockExecutorApi_LeaseJobRunsServerMockRecorder) RecvMsg(m any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RecvMsg", reflect.TypeOf((*MockExecutorApi_LeaseJobRunsServer)(nil).RecvMsg), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RecvMsg", reflect.TypeOf((*MockExecutorApi_LeaseJobRunsServer)(nil).RecvMsg), m) } // Send mocks base method. @@ -88,7 +94,7 @@ func (m *MockExecutorApi_LeaseJobRunsServer) Send(arg0 *executorapi.LeaseStreamM } // Send indicates an expected call of Send. -func (mr *MockExecutorApi_LeaseJobRunsServerMockRecorder) Send(arg0 interface{}) *gomock.Call { +func (mr *MockExecutorApi_LeaseJobRunsServerMockRecorder) Send(arg0 any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Send", reflect.TypeOf((*MockExecutorApi_LeaseJobRunsServer)(nil).Send), arg0) } @@ -102,23 +108,23 @@ func (m *MockExecutorApi_LeaseJobRunsServer) SendHeader(arg0 metadata.MD) error } // SendHeader indicates an expected call of SendHeader. -func (mr *MockExecutorApi_LeaseJobRunsServerMockRecorder) SendHeader(arg0 interface{}) *gomock.Call { +func (mr *MockExecutorApi_LeaseJobRunsServerMockRecorder) SendHeader(arg0 any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendHeader", reflect.TypeOf((*MockExecutorApi_LeaseJobRunsServer)(nil).SendHeader), arg0) } // SendMsg mocks base method. -func (m *MockExecutorApi_LeaseJobRunsServer) SendMsg(arg0 interface{}) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SendMsg", arg0) +func (m_2 *MockExecutorApi_LeaseJobRunsServer) SendMsg(m any) error { + m_2.ctrl.T.Helper() + ret := m_2.ctrl.Call(m_2, "SendMsg", m) ret0, _ := ret[0].(error) return ret0 } // SendMsg indicates an expected call of SendMsg. -func (mr *MockExecutorApi_LeaseJobRunsServerMockRecorder) SendMsg(arg0 interface{}) *gomock.Call { +func (mr *MockExecutorApi_LeaseJobRunsServerMockRecorder) SendMsg(m any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendMsg", reflect.TypeOf((*MockExecutorApi_LeaseJobRunsServer)(nil).SendMsg), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendMsg", reflect.TypeOf((*MockExecutorApi_LeaseJobRunsServer)(nil).SendMsg), m) } // SetHeader mocks base method. @@ -130,7 +136,7 @@ func (m *MockExecutorApi_LeaseJobRunsServer) SetHeader(arg0 metadata.MD) error { } // SetHeader indicates an expected call of SetHeader. -func (mr *MockExecutorApi_LeaseJobRunsServerMockRecorder) SetHeader(arg0 interface{}) *gomock.Call { +func (mr *MockExecutorApi_LeaseJobRunsServerMockRecorder) SetHeader(arg0 any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetHeader", reflect.TypeOf((*MockExecutorApi_LeaseJobRunsServer)(nil).SetHeader), arg0) } @@ -142,7 +148,7 @@ func (m *MockExecutorApi_LeaseJobRunsServer) SetTrailer(arg0 metadata.MD) { } // SetTrailer indicates an expected call of SetTrailer. -func (mr *MockExecutorApi_LeaseJobRunsServerMockRecorder) SetTrailer(arg0 interface{}) *gomock.Call { +func (mr *MockExecutorApi_LeaseJobRunsServerMockRecorder) SetTrailer(arg0 any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetTrailer", reflect.TypeOf((*MockExecutorApi_LeaseJobRunsServer)(nil).SetTrailer), arg0) } diff --git a/internal/scheduler/mocks/job_repository.go b/internal/scheduler/mocks/job_repository.go index 21dba092880..0b5a18d2489 100644 --- a/internal/scheduler/mocks/job_repository.go +++ b/internal/scheduler/mocks/job_repository.go @@ -1,5 +1,10 @@ // Code generated by MockGen. DO NOT EDIT. // Source: github.com/armadaproject/armada/internal/scheduler/database (interfaces: JobRepository) +// +// Generated by this command: +// +// mockgen -destination=./job_repository.go -package=schedulermocks github.com/armadaproject/armada/internal/scheduler/database JobRepository +// // Package schedulermocks is a generated GoMock package. package schedulermocks @@ -10,14 +15,15 @@ import ( armadacontext "github.com/armadaproject/armada/internal/common/armadacontext" database "github.com/armadaproject/armada/internal/scheduler/database" armadaevents "github.com/armadaproject/armada/pkg/armadaevents" - gomock "github.com/golang/mock/gomock" uuid "github.com/google/uuid" + gomock "go.uber.org/mock/gomock" ) // MockJobRepository is a mock of JobRepository interface. type MockJobRepository struct { ctrl *gomock.Controller recorder *MockJobRepositoryMockRecorder + isgomock struct{} } // MockJobRepositoryMockRecorder is the mock recorder for MockJobRepository. @@ -38,24 +44,24 @@ func (m *MockJobRepository) EXPECT() *MockJobRepositoryMockRecorder { } // CountReceivedPartitions mocks base method. -func (m *MockJobRepository) CountReceivedPartitions(arg0 *armadacontext.Context, arg1 uuid.UUID) (uint32, error) { +func (m *MockJobRepository) CountReceivedPartitions(ctx *armadacontext.Context, groupId uuid.UUID) (uint32, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CountReceivedPartitions", arg0, arg1) + ret := m.ctrl.Call(m, "CountReceivedPartitions", ctx, groupId) ret0, _ := ret[0].(uint32) ret1, _ := ret[1].(error) return ret0, ret1 } // CountReceivedPartitions indicates an expected call of CountReceivedPartitions. -func (mr *MockJobRepositoryMockRecorder) CountReceivedPartitions(arg0, arg1 interface{}) *gomock.Call { +func (mr *MockJobRepositoryMockRecorder) CountReceivedPartitions(ctx, groupId any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CountReceivedPartitions", reflect.TypeOf((*MockJobRepository)(nil).CountReceivedPartitions), arg0, arg1) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CountReceivedPartitions", reflect.TypeOf((*MockJobRepository)(nil).CountReceivedPartitions), ctx, groupId) } // FetchInitialJobs mocks base method. -func (m *MockJobRepository) FetchInitialJobs(arg0 *armadacontext.Context) ([]database.Job, []database.Run, error) { +func (m *MockJobRepository) FetchInitialJobs(ctx *armadacontext.Context) ([]database.Job, []database.Run, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "FetchInitialJobs", arg0) + ret := m.ctrl.Call(m, "FetchInitialJobs", ctx) ret0, _ := ret[0].([]database.Job) ret1, _ := ret[1].([]database.Run) ret2, _ := ret[2].(error) @@ -63,45 +69,45 @@ func (m *MockJobRepository) FetchInitialJobs(arg0 *armadacontext.Context) ([]dat } // FetchInitialJobs indicates an expected call of FetchInitialJobs. -func (mr *MockJobRepositoryMockRecorder) FetchInitialJobs(arg0 interface{}) *gomock.Call { +func (mr *MockJobRepositoryMockRecorder) FetchInitialJobs(ctx any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FetchInitialJobs", reflect.TypeOf((*MockJobRepository)(nil).FetchInitialJobs), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FetchInitialJobs", reflect.TypeOf((*MockJobRepository)(nil).FetchInitialJobs), ctx) } // FetchJobRunErrors mocks base method. -func (m *MockJobRepository) FetchJobRunErrors(arg0 *armadacontext.Context, arg1 []string) (map[string]*armadaevents.Error, error) { +func (m *MockJobRepository) FetchJobRunErrors(ctx *armadacontext.Context, runIds []string) (map[string]*armadaevents.Error, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "FetchJobRunErrors", arg0, arg1) + ret := m.ctrl.Call(m, "FetchJobRunErrors", ctx, runIds) ret0, _ := ret[0].(map[string]*armadaevents.Error) ret1, _ := ret[1].(error) return ret0, ret1 } // FetchJobRunErrors indicates an expected call of FetchJobRunErrors. -func (mr *MockJobRepositoryMockRecorder) FetchJobRunErrors(arg0, arg1 interface{}) *gomock.Call { +func (mr *MockJobRepositoryMockRecorder) FetchJobRunErrors(ctx, runIds any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FetchJobRunErrors", reflect.TypeOf((*MockJobRepository)(nil).FetchJobRunErrors), arg0, arg1) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FetchJobRunErrors", reflect.TypeOf((*MockJobRepository)(nil).FetchJobRunErrors), ctx, runIds) } // FetchJobRunLeases mocks base method. -func (m *MockJobRepository) FetchJobRunLeases(arg0 *armadacontext.Context, arg1 string, arg2 uint, arg3 []string) ([]*database.JobRunLease, error) { +func (m *MockJobRepository) FetchJobRunLeases(ctx *armadacontext.Context, executor string, maxResults uint, excludedRunIds []string) ([]*database.JobRunLease, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "FetchJobRunLeases", arg0, arg1, arg2, arg3) + ret := m.ctrl.Call(m, "FetchJobRunLeases", ctx, executor, maxResults, excludedRunIds) ret0, _ := ret[0].([]*database.JobRunLease) ret1, _ := ret[1].(error) return ret0, ret1 } // FetchJobRunLeases indicates an expected call of FetchJobRunLeases. -func (mr *MockJobRepositoryMockRecorder) FetchJobRunLeases(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { +func (mr *MockJobRepositoryMockRecorder) FetchJobRunLeases(ctx, executor, maxResults, excludedRunIds any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FetchJobRunLeases", reflect.TypeOf((*MockJobRepository)(nil).FetchJobRunLeases), arg0, arg1, arg2, arg3) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FetchJobRunLeases", reflect.TypeOf((*MockJobRepository)(nil).FetchJobRunLeases), ctx, executor, maxResults, excludedRunIds) } // FetchJobUpdates mocks base method. -func (m *MockJobRepository) FetchJobUpdates(arg0 *armadacontext.Context, arg1, arg2 int64) ([]database.Job, []database.Run, error) { +func (m *MockJobRepository) FetchJobUpdates(ctx *armadacontext.Context, jobSerial, jobRunSerial int64) ([]database.Job, []database.Run, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "FetchJobUpdates", arg0, arg1, arg2) + ret := m.ctrl.Call(m, "FetchJobUpdates", ctx, jobSerial, jobRunSerial) ret0, _ := ret[0].([]database.Job) ret1, _ := ret[1].([]database.Run) ret2, _ := ret[2].(error) @@ -109,22 +115,22 @@ func (m *MockJobRepository) FetchJobUpdates(arg0 *armadacontext.Context, arg1, a } // FetchJobUpdates indicates an expected call of FetchJobUpdates. -func (mr *MockJobRepositoryMockRecorder) FetchJobUpdates(arg0, arg1, arg2 interface{}) *gomock.Call { +func (mr *MockJobRepositoryMockRecorder) FetchJobUpdates(ctx, jobSerial, jobRunSerial any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FetchJobUpdates", reflect.TypeOf((*MockJobRepository)(nil).FetchJobUpdates), arg0, arg1, arg2) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FetchJobUpdates", reflect.TypeOf((*MockJobRepository)(nil).FetchJobUpdates), ctx, jobSerial, jobRunSerial) } // FindInactiveRuns mocks base method. -func (m *MockJobRepository) FindInactiveRuns(arg0 *armadacontext.Context, arg1 []string) ([]string, error) { +func (m *MockJobRepository) FindInactiveRuns(ctx *armadacontext.Context, runIds []string) ([]string, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "FindInactiveRuns", arg0, arg1) + ret := m.ctrl.Call(m, "FindInactiveRuns", ctx, runIds) ret0, _ := ret[0].([]string) ret1, _ := ret[1].(error) return ret0, ret1 } // FindInactiveRuns indicates an expected call of FindInactiveRuns. -func (mr *MockJobRepositoryMockRecorder) FindInactiveRuns(arg0, arg1 interface{}) *gomock.Call { +func (mr *MockJobRepositoryMockRecorder) FindInactiveRuns(ctx, runIds any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FindInactiveRuns", reflect.TypeOf((*MockJobRepository)(nil).FindInactiveRuns), arg0, arg1) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FindInactiveRuns", reflect.TypeOf((*MockJobRepository)(nil).FindInactiveRuns), ctx, runIds) } diff --git a/internal/scheduler/mocks/leases_getter.go b/internal/scheduler/mocks/leases_getter.go index 012c2bfffd5..364ea7b792e 100644 --- a/internal/scheduler/mocks/leases_getter.go +++ b/internal/scheduler/mocks/leases_getter.go @@ -1,5 +1,10 @@ // Code generated by MockGen. DO NOT EDIT. // Source: k8s.io/client-go/kubernetes/typed/coordination/v1 (interfaces: LeasesGetter,LeaseInterface) +// +// Generated by this command: +// +// mockgen -destination=./leases_getter.go -package=schedulermocks k8s.io/client-go/kubernetes/typed/coordination/v1 LeasesGetter,LeaseInterface +// // Package schedulermocks is a generated GoMock package. package schedulermocks @@ -8,7 +13,7 @@ import ( context "context" reflect "reflect" - gomock "github.com/golang/mock/gomock" + gomock "go.uber.org/mock/gomock" v1 "k8s.io/api/coordination/v1" v10 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -21,6 +26,7 @@ import ( type MockLeasesGetter struct { ctrl *gomock.Controller recorder *MockLeasesGetterMockRecorder + isgomock struct{} } // MockLeasesGetterMockRecorder is the mock recorder for MockLeasesGetter. @@ -41,23 +47,24 @@ func (m *MockLeasesGetter) EXPECT() *MockLeasesGetterMockRecorder { } // Leases mocks base method. -func (m *MockLeasesGetter) Leases(arg0 string) v12.LeaseInterface { +func (m *MockLeasesGetter) Leases(namespace string) v12.LeaseInterface { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Leases", arg0) + ret := m.ctrl.Call(m, "Leases", namespace) ret0, _ := ret[0].(v12.LeaseInterface) return ret0 } // Leases indicates an expected call of Leases. -func (mr *MockLeasesGetterMockRecorder) Leases(arg0 interface{}) *gomock.Call { +func (mr *MockLeasesGetterMockRecorder) Leases(namespace any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Leases", reflect.TypeOf((*MockLeasesGetter)(nil).Leases), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Leases", reflect.TypeOf((*MockLeasesGetter)(nil).Leases), namespace) } // MockLeaseInterface is a mock of LeaseInterface interface. type MockLeaseInterface struct { ctrl *gomock.Controller recorder *MockLeaseInterfaceMockRecorder + isgomock struct{} } // MockLeaseInterfaceMockRecorder is the mock recorder for MockLeaseInterface. @@ -78,98 +85,98 @@ func (m *MockLeaseInterface) EXPECT() *MockLeaseInterfaceMockRecorder { } // Apply mocks base method. -func (m *MockLeaseInterface) Apply(arg0 context.Context, arg1 *v11.LeaseApplyConfiguration, arg2 v10.ApplyOptions) (*v1.Lease, error) { +func (m *MockLeaseInterface) Apply(ctx context.Context, lease *v11.LeaseApplyConfiguration, opts v10.ApplyOptions) (*v1.Lease, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Apply", arg0, arg1, arg2) + ret := m.ctrl.Call(m, "Apply", ctx, lease, opts) ret0, _ := ret[0].(*v1.Lease) ret1, _ := ret[1].(error) return ret0, ret1 } // Apply indicates an expected call of Apply. -func (mr *MockLeaseInterfaceMockRecorder) Apply(arg0, arg1, arg2 interface{}) *gomock.Call { +func (mr *MockLeaseInterfaceMockRecorder) Apply(ctx, lease, opts any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Apply", reflect.TypeOf((*MockLeaseInterface)(nil).Apply), arg0, arg1, arg2) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Apply", reflect.TypeOf((*MockLeaseInterface)(nil).Apply), ctx, lease, opts) } // Create mocks base method. -func (m *MockLeaseInterface) Create(arg0 context.Context, arg1 *v1.Lease, arg2 v10.CreateOptions) (*v1.Lease, error) { +func (m *MockLeaseInterface) Create(ctx context.Context, lease *v1.Lease, opts v10.CreateOptions) (*v1.Lease, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Create", arg0, arg1, arg2) + ret := m.ctrl.Call(m, "Create", ctx, lease, opts) ret0, _ := ret[0].(*v1.Lease) ret1, _ := ret[1].(error) return ret0, ret1 } // Create indicates an expected call of Create. -func (mr *MockLeaseInterfaceMockRecorder) Create(arg0, arg1, arg2 interface{}) *gomock.Call { +func (mr *MockLeaseInterfaceMockRecorder) Create(ctx, lease, opts any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Create", reflect.TypeOf((*MockLeaseInterface)(nil).Create), arg0, arg1, arg2) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Create", reflect.TypeOf((*MockLeaseInterface)(nil).Create), ctx, lease, opts) } // Delete mocks base method. -func (m *MockLeaseInterface) Delete(arg0 context.Context, arg1 string, arg2 v10.DeleteOptions) error { +func (m *MockLeaseInterface) Delete(ctx context.Context, name string, opts v10.DeleteOptions) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Delete", arg0, arg1, arg2) + ret := m.ctrl.Call(m, "Delete", ctx, name, opts) ret0, _ := ret[0].(error) return ret0 } // Delete indicates an expected call of Delete. -func (mr *MockLeaseInterfaceMockRecorder) Delete(arg0, arg1, arg2 interface{}) *gomock.Call { +func (mr *MockLeaseInterfaceMockRecorder) Delete(ctx, name, opts any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockLeaseInterface)(nil).Delete), arg0, arg1, arg2) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockLeaseInterface)(nil).Delete), ctx, name, opts) } // DeleteCollection mocks base method. -func (m *MockLeaseInterface) DeleteCollection(arg0 context.Context, arg1 v10.DeleteOptions, arg2 v10.ListOptions) error { +func (m *MockLeaseInterface) DeleteCollection(ctx context.Context, opts v10.DeleteOptions, listOpts v10.ListOptions) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DeleteCollection", arg0, arg1, arg2) + ret := m.ctrl.Call(m, "DeleteCollection", ctx, opts, listOpts) ret0, _ := ret[0].(error) return ret0 } // DeleteCollection indicates an expected call of DeleteCollection. -func (mr *MockLeaseInterfaceMockRecorder) DeleteCollection(arg0, arg1, arg2 interface{}) *gomock.Call { +func (mr *MockLeaseInterfaceMockRecorder) DeleteCollection(ctx, opts, listOpts any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteCollection", reflect.TypeOf((*MockLeaseInterface)(nil).DeleteCollection), arg0, arg1, arg2) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteCollection", reflect.TypeOf((*MockLeaseInterface)(nil).DeleteCollection), ctx, opts, listOpts) } // Get mocks base method. -func (m *MockLeaseInterface) Get(arg0 context.Context, arg1 string, arg2 v10.GetOptions) (*v1.Lease, error) { +func (m *MockLeaseInterface) Get(ctx context.Context, name string, opts v10.GetOptions) (*v1.Lease, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Get", arg0, arg1, arg2) + ret := m.ctrl.Call(m, "Get", ctx, name, opts) ret0, _ := ret[0].(*v1.Lease) ret1, _ := ret[1].(error) return ret0, ret1 } // Get indicates an expected call of Get. -func (mr *MockLeaseInterfaceMockRecorder) Get(arg0, arg1, arg2 interface{}) *gomock.Call { +func (mr *MockLeaseInterfaceMockRecorder) Get(ctx, name, opts any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockLeaseInterface)(nil).Get), arg0, arg1, arg2) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockLeaseInterface)(nil).Get), ctx, name, opts) } // List mocks base method. -func (m *MockLeaseInterface) List(arg0 context.Context, arg1 v10.ListOptions) (*v1.LeaseList, error) { +func (m *MockLeaseInterface) List(ctx context.Context, opts v10.ListOptions) (*v1.LeaseList, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "List", arg0, arg1) + ret := m.ctrl.Call(m, "List", ctx, opts) ret0, _ := ret[0].(*v1.LeaseList) ret1, _ := ret[1].(error) return ret0, ret1 } // List indicates an expected call of List. -func (mr *MockLeaseInterfaceMockRecorder) List(arg0, arg1 interface{}) *gomock.Call { +func (mr *MockLeaseInterfaceMockRecorder) List(ctx, opts any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockLeaseInterface)(nil).List), arg0, arg1) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockLeaseInterface)(nil).List), ctx, opts) } // Patch mocks base method. -func (m *MockLeaseInterface) Patch(arg0 context.Context, arg1 string, arg2 types.PatchType, arg3 []byte, arg4 v10.PatchOptions, arg5 ...string) (*v1.Lease, error) { +func (m *MockLeaseInterface) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v10.PatchOptions, subresources ...string) (*v1.Lease, error) { m.ctrl.T.Helper() - varargs := []interface{}{arg0, arg1, arg2, arg3, arg4} - for _, a := range arg5 { + varargs := []any{ctx, name, pt, data, opts} + for _, a := range subresources { varargs = append(varargs, a) } ret := m.ctrl.Call(m, "Patch", varargs...) @@ -179,38 +186,38 @@ func (m *MockLeaseInterface) Patch(arg0 context.Context, arg1 string, arg2 types } // Patch indicates an expected call of Patch. -func (mr *MockLeaseInterfaceMockRecorder) Patch(arg0, arg1, arg2, arg3, arg4 interface{}, arg5 ...interface{}) *gomock.Call { +func (mr *MockLeaseInterfaceMockRecorder) Patch(ctx, name, pt, data, opts any, subresources ...any) *gomock.Call { mr.mock.ctrl.T.Helper() - varargs := append([]interface{}{arg0, arg1, arg2, arg3, arg4}, arg5...) + varargs := append([]any{ctx, name, pt, data, opts}, subresources...) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Patch", reflect.TypeOf((*MockLeaseInterface)(nil).Patch), varargs...) } // Update mocks base method. -func (m *MockLeaseInterface) Update(arg0 context.Context, arg1 *v1.Lease, arg2 v10.UpdateOptions) (*v1.Lease, error) { +func (m *MockLeaseInterface) Update(ctx context.Context, lease *v1.Lease, opts v10.UpdateOptions) (*v1.Lease, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Update", arg0, arg1, arg2) + ret := m.ctrl.Call(m, "Update", ctx, lease, opts) ret0, _ := ret[0].(*v1.Lease) ret1, _ := ret[1].(error) return ret0, ret1 } // Update indicates an expected call of Update. -func (mr *MockLeaseInterfaceMockRecorder) Update(arg0, arg1, arg2 interface{}) *gomock.Call { +func (mr *MockLeaseInterfaceMockRecorder) Update(ctx, lease, opts any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Update", reflect.TypeOf((*MockLeaseInterface)(nil).Update), arg0, arg1, arg2) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Update", reflect.TypeOf((*MockLeaseInterface)(nil).Update), ctx, lease, opts) } // Watch mocks base method. -func (m *MockLeaseInterface) Watch(arg0 context.Context, arg1 v10.ListOptions) (watch.Interface, error) { +func (m *MockLeaseInterface) Watch(ctx context.Context, opts v10.ListOptions) (watch.Interface, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Watch", arg0, arg1) + ret := m.ctrl.Call(m, "Watch", ctx, opts) ret0, _ := ret[0].(watch.Interface) ret1, _ := ret[1].(error) return ret0, ret1 } // Watch indicates an expected call of Watch. -func (mr *MockLeaseInterfaceMockRecorder) Watch(arg0, arg1 interface{}) *gomock.Call { +func (mr *MockLeaseInterfaceMockRecorder) Watch(ctx, opts any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Watch", reflect.TypeOf((*MockLeaseInterface)(nil).Watch), arg0, arg1) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Watch", reflect.TypeOf((*MockLeaseInterface)(nil).Watch), ctx, opts) } diff --git a/internal/scheduler/mocks/queue_cache.go b/internal/scheduler/mocks/queue_cache.go index 1970a5fa010..1a24387c10f 100644 --- a/internal/scheduler/mocks/queue_cache.go +++ b/internal/scheduler/mocks/queue_cache.go @@ -1,5 +1,10 @@ // Code generated by MockGen. DO NOT EDIT. // Source: github.com/armadaproject/armada/internal/scheduler/queue (interfaces: QueueCache) +// +// Generated by this command: +// +// mockgen -destination=./queue_cache.go -package=schedulermocks github.com/armadaproject/armada/internal/scheduler/queue QueueCache +// // Package schedulermocks is a generated GoMock package. package schedulermocks @@ -9,13 +14,14 @@ import ( armadacontext "github.com/armadaproject/armada/internal/common/armadacontext" api "github.com/armadaproject/armada/pkg/api" - gomock "github.com/golang/mock/gomock" + gomock "go.uber.org/mock/gomock" ) // MockQueueCache is a mock of QueueCache interface. type MockQueueCache struct { ctrl *gomock.Controller recorder *MockQueueCacheMockRecorder + isgomock struct{} } // MockQueueCacheMockRecorder is the mock recorder for MockQueueCache. @@ -36,16 +42,16 @@ func (m *MockQueueCache) EXPECT() *MockQueueCacheMockRecorder { } // GetAll mocks base method. -func (m *MockQueueCache) GetAll(arg0 *armadacontext.Context) ([]*api.Queue, error) { +func (m *MockQueueCache) GetAll(ctx *armadacontext.Context) ([]*api.Queue, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetAll", arg0) + ret := m.ctrl.Call(m, "GetAll", ctx) ret0, _ := ret[0].([]*api.Queue) ret1, _ := ret[1].(error) return ret0, ret1 } // GetAll indicates an expected call of GetAll. -func (mr *MockQueueCacheMockRecorder) GetAll(arg0 interface{}) *gomock.Call { +func (mr *MockQueueCacheMockRecorder) GetAll(ctx any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAll", reflect.TypeOf((*MockQueueCache)(nil).GetAll), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAll", reflect.TypeOf((*MockQueueCache)(nil).GetAll), ctx) } diff --git a/internal/scheduler/publisher_test.go b/internal/scheduler/publisher_test.go index 66a42a20453..8ccd2d3f68d 100644 --- a/internal/scheduler/publisher_test.go +++ b/internal/scheduler/publisher_test.go @@ -8,11 +8,11 @@ import ( "github.com/apache/pulsar-client-go/pulsar" "github.com/gogo/protobuf/proto" - "github.com/golang/mock/gomock" "github.com/google/uuid" "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" "github.com/armadaproject/armada/internal/common/armadacontext" "github.com/armadaproject/armada/internal/common/mocks" diff --git a/internal/scheduler/queue/queue_cache_test.go b/internal/scheduler/queue/queue_cache_test.go index 7687d5772ea..d57c65afef1 100644 --- a/internal/scheduler/queue/queue_cache_test.go +++ b/internal/scheduler/queue/queue_cache_test.go @@ -5,8 +5,8 @@ import ( "testing" "time" - "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" + "go.uber.org/mock/gomock" "github.com/armadaproject/armada/internal/common/armadacontext" schedulermocks "github.com/armadaproject/armada/internal/scheduler/mocks" diff --git a/internal/scheduler/scheduling/scheduling_algo_test.go b/internal/scheduler/scheduling/scheduling_algo_test.go index 18e69ffd7f2..b69ab010244 100644 --- a/internal/scheduler/scheduling/scheduling_algo_test.go +++ b/internal/scheduler/scheduling/scheduling_algo_test.go @@ -6,9 +6,9 @@ import ( "testing" "time" - "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" "golang.org/x/exp/slices" k8sResource "k8s.io/apimachinery/pkg/api/resource" clock "k8s.io/utils/clock/testing" diff --git a/internal/scheduler/submitcheck_test.go b/internal/scheduler/submitcheck_test.go index f87dac1cdc8..d5d355a9e5d 100644 --- a/internal/scheduler/submitcheck_test.go +++ b/internal/scheduler/submitcheck_test.go @@ -4,10 +4,10 @@ import ( "testing" "time" - "github.com/golang/mock/gomock" "github.com/google/uuid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" "golang.org/x/exp/slices" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" diff --git a/internal/server/mocks/generate.go b/internal/server/mocks/generate.go index 0fc5f59e161..2efeadc2e72 100644 --- a/internal/server/mocks/generate.go +++ b/internal/server/mocks/generate.go @@ -1,5 +1,6 @@ package mocks // Mock implementations used by tests -//go:generate mockgen -destination=./mock_deduplicator.go -package=mocks "github.com/armadaproject/armada/internal/armada/submit" Deduplicator -//go:generate mockgen -destination=./mock_authorizer.go -package=mocks "github.com/armadaproject/armada/internal/armada/server" ActionAuthorizer +//go:generate mockgen -destination=./mock_deduplicator.go -package=mocks "github.com/armadaproject/armada/internal/server/submit" Deduplicator +//go:generate mockgen -destination=./mock_authorizer.go -package=mocks "github.com/armadaproject/armada/internal/common/auth" ActionAuthorizer +//go:generate mockgen -destination=./mock_repository.go -package=mocks "github.com/armadaproject/armada/internal/server/queue" QueueRepository diff --git a/internal/server/mocks/mock_authorizer.go b/internal/server/mocks/mock_authorizer.go index c4bab20654e..57b7f85584a 100644 --- a/internal/server/mocks/mock_authorizer.go +++ b/internal/server/mocks/mock_authorizer.go @@ -1,5 +1,10 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/armadaproject/armada/internal/armada/server (interfaces: ActionAuthorizer) +// Source: github.com/armadaproject/armada/internal/common/auth (interfaces: ActionAuthorizer) +// +// Generated by this command: +// +// mockgen -destination=./mock_authorizer.go -package=mocks github.com/armadaproject/armada/internal/common/auth ActionAuthorizer +// // Package mocks is a generated GoMock package. package mocks @@ -10,13 +15,14 @@ import ( armadacontext "github.com/armadaproject/armada/internal/common/armadacontext" permission "github.com/armadaproject/armada/internal/common/auth/permission" queue "github.com/armadaproject/armada/pkg/client/queue" - gomock "github.com/golang/mock/gomock" + gomock "go.uber.org/mock/gomock" ) // MockActionAuthorizer is a mock of ActionAuthorizer interface. type MockActionAuthorizer struct { ctrl *gomock.Controller recorder *MockActionAuthorizerMockRecorder + isgomock struct{} } // MockActionAuthorizerMockRecorder is the mock recorder for MockActionAuthorizer. @@ -37,29 +43,29 @@ func (m *MockActionAuthorizer) EXPECT() *MockActionAuthorizerMockRecorder { } // AuthorizeAction mocks base method. -func (m *MockActionAuthorizer) AuthorizeAction(arg0 *armadacontext.Context, arg1 permission.Permission) error { +func (m *MockActionAuthorizer) AuthorizeAction(ctx *armadacontext.Context, perm permission.Permission) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "AuthorizeAction", arg0, arg1) + ret := m.ctrl.Call(m, "AuthorizeAction", ctx, perm) ret0, _ := ret[0].(error) return ret0 } // AuthorizeAction indicates an expected call of AuthorizeAction. -func (mr *MockActionAuthorizerMockRecorder) AuthorizeAction(arg0, arg1 interface{}) *gomock.Call { +func (mr *MockActionAuthorizerMockRecorder) AuthorizeAction(ctx, perm any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AuthorizeAction", reflect.TypeOf((*MockActionAuthorizer)(nil).AuthorizeAction), arg0, arg1) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AuthorizeAction", reflect.TypeOf((*MockActionAuthorizer)(nil).AuthorizeAction), ctx, perm) } // AuthorizeQueueAction mocks base method. -func (m *MockActionAuthorizer) AuthorizeQueueAction(arg0 *armadacontext.Context, arg1 queue.Queue, arg2 permission.Permission, arg3 queue.PermissionVerb) error { +func (m *MockActionAuthorizer) AuthorizeQueueAction(ctx *armadacontext.Context, queue queue.Queue, anyPerm permission.Permission, perm queue.PermissionVerb) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "AuthorizeQueueAction", arg0, arg1, arg2, arg3) + ret := m.ctrl.Call(m, "AuthorizeQueueAction", ctx, queue, anyPerm, perm) ret0, _ := ret[0].(error) return ret0 } // AuthorizeQueueAction indicates an expected call of AuthorizeQueueAction. -func (mr *MockActionAuthorizerMockRecorder) AuthorizeQueueAction(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { +func (mr *MockActionAuthorizerMockRecorder) AuthorizeQueueAction(ctx, queue, anyPerm, perm any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AuthorizeQueueAction", reflect.TypeOf((*MockActionAuthorizer)(nil).AuthorizeQueueAction), arg0, arg1, arg2, arg3) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AuthorizeQueueAction", reflect.TypeOf((*MockActionAuthorizer)(nil).AuthorizeQueueAction), ctx, queue, anyPerm, perm) } diff --git a/internal/server/mocks/mock_deduplicator.go b/internal/server/mocks/mock_deduplicator.go index edb4ef413c2..a5dc0d1db3d 100644 --- a/internal/server/mocks/mock_deduplicator.go +++ b/internal/server/mocks/mock_deduplicator.go @@ -1,5 +1,10 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/armadaproject/armada/internal/armada/submit (interfaces: Deduplicator) +// Source: github.com/armadaproject/armada/internal/server/submit (interfaces: Deduplicator) +// +// Generated by this command: +// +// mockgen -destination=./mock_deduplicator.go -package=mocks github.com/armadaproject/armada/internal/server/submit Deduplicator +// // Package mocks is a generated GoMock package. package mocks @@ -9,13 +14,14 @@ import ( armadacontext "github.com/armadaproject/armada/internal/common/armadacontext" api "github.com/armadaproject/armada/pkg/api" - gomock "github.com/golang/mock/gomock" + gomock "go.uber.org/mock/gomock" ) // MockDeduplicator is a mock of Deduplicator interface. type MockDeduplicator struct { ctrl *gomock.Controller recorder *MockDeduplicatorMockRecorder + isgomock struct{} } // MockDeduplicatorMockRecorder is the mock recorder for MockDeduplicator. @@ -36,30 +42,30 @@ func (m *MockDeduplicator) EXPECT() *MockDeduplicatorMockRecorder { } // GetOriginalJobIds mocks base method. -func (m *MockDeduplicator) GetOriginalJobIds(arg0 *armadacontext.Context, arg1 string, arg2 []*api.JobSubmitRequestItem) (map[string]string, error) { +func (m *MockDeduplicator) GetOriginalJobIds(ctx *armadacontext.Context, queue string, jobRequests []*api.JobSubmitRequestItem) (map[string]string, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetOriginalJobIds", arg0, arg1, arg2) + ret := m.ctrl.Call(m, "GetOriginalJobIds", ctx, queue, jobRequests) ret0, _ := ret[0].(map[string]string) ret1, _ := ret[1].(error) return ret0, ret1 } // GetOriginalJobIds indicates an expected call of GetOriginalJobIds. -func (mr *MockDeduplicatorMockRecorder) GetOriginalJobIds(arg0, arg1, arg2 interface{}) *gomock.Call { +func (mr *MockDeduplicatorMockRecorder) GetOriginalJobIds(ctx, queue, jobRequests any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetOriginalJobIds", reflect.TypeOf((*MockDeduplicator)(nil).GetOriginalJobIds), arg0, arg1, arg2) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetOriginalJobIds", reflect.TypeOf((*MockDeduplicator)(nil).GetOriginalJobIds), ctx, queue, jobRequests) } // StoreOriginalJobIds mocks base method. -func (m *MockDeduplicator) StoreOriginalJobIds(arg0 *armadacontext.Context, arg1 string, arg2 map[string]string) error { +func (m *MockDeduplicator) StoreOriginalJobIds(ctx *armadacontext.Context, queue string, mappings map[string]string) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "StoreOriginalJobIds", arg0, arg1, arg2) + ret := m.ctrl.Call(m, "StoreOriginalJobIds", ctx, queue, mappings) ret0, _ := ret[0].(error) return ret0 } // StoreOriginalJobIds indicates an expected call of StoreOriginalJobIds. -func (mr *MockDeduplicatorMockRecorder) StoreOriginalJobIds(arg0, arg1, arg2 interface{}) *gomock.Call { +func (mr *MockDeduplicatorMockRecorder) StoreOriginalJobIds(ctx, queue, mappings any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StoreOriginalJobIds", reflect.TypeOf((*MockDeduplicator)(nil).StoreOriginalJobIds), arg0, arg1, arg2) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StoreOriginalJobIds", reflect.TypeOf((*MockDeduplicator)(nil).StoreOriginalJobIds), ctx, queue, mappings) } diff --git a/internal/server/mocks/mock_repository.go b/internal/server/mocks/mock_repository.go index a9f79e2702b..c5b6ebc8560 100644 --- a/internal/server/mocks/mock_repository.go +++ b/internal/server/mocks/mock_repository.go @@ -1,5 +1,10 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/armadaproject/armada/internal/armada/queue (interfaces: QueueRepository) +// Source: github.com/armadaproject/armada/internal/server/queue (interfaces: QueueRepository) +// +// Generated by this command: +// +// mockgen -destination=./mock_repository.go -package=mocks github.com/armadaproject/armada/internal/server/queue QueueRepository +// // Package mocks is a generated GoMock package. package mocks @@ -9,13 +14,14 @@ import ( armadacontext "github.com/armadaproject/armada/internal/common/armadacontext" queue "github.com/armadaproject/armada/pkg/client/queue" - gomock "github.com/golang/mock/gomock" + gomock "go.uber.org/mock/gomock" ) // MockQueueRepository is a mock of QueueRepository interface. type MockQueueRepository struct { ctrl *gomock.Controller recorder *MockQueueRepositoryMockRecorder + isgomock struct{} } // MockQueueRepositoryMockRecorder is the mock recorder for MockQueueRepository. @@ -35,6 +41,20 @@ func (m *MockQueueRepository) EXPECT() *MockQueueRepositoryMockRecorder { return m.recorder } +// CordonQueue mocks base method. +func (m *MockQueueRepository) CordonQueue(ctx *armadacontext.Context, name string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CordonQueue", ctx, name) + ret0, _ := ret[0].(error) + return ret0 +} + +// CordonQueue indicates an expected call of CordonQueue. +func (mr *MockQueueRepositoryMockRecorder) CordonQueue(ctx, name any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CordonQueue", reflect.TypeOf((*MockQueueRepository)(nil).CordonQueue), ctx, name) +} + // CreateQueue mocks base method. func (m *MockQueueRepository) CreateQueue(arg0 *armadacontext.Context, arg1 queue.Queue) error { m.ctrl.T.Helper() @@ -44,69 +64,67 @@ func (m *MockQueueRepository) CreateQueue(arg0 *armadacontext.Context, arg1 queu } // CreateQueue indicates an expected call of CreateQueue. -func (mr *MockQueueRepositoryMockRecorder) CreateQueue(arg0, arg1 interface{}) *gomock.Call { +func (mr *MockQueueRepositoryMockRecorder) CreateQueue(arg0, arg1 any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateQueue", reflect.TypeOf((*MockQueueRepository)(nil).CreateQueue), arg0, arg1) } // DeleteQueue mocks base method. -func (m *MockQueueRepository) DeleteQueue(arg0 *armadacontext.Context, arg1 string) error { +func (m *MockQueueRepository) DeleteQueue(ctx *armadacontext.Context, name string) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DeleteQueue", arg0, arg1) + ret := m.ctrl.Call(m, "DeleteQueue", ctx, name) ret0, _ := ret[0].(error) return ret0 } // DeleteQueue indicates an expected call of DeleteQueue. -func (mr *MockQueueRepositoryMockRecorder) DeleteQueue(arg0, arg1 interface{}) *gomock.Call { +func (mr *MockQueueRepositoryMockRecorder) DeleteQueue(ctx, name any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteQueue", reflect.TypeOf((*MockQueueRepository)(nil).DeleteQueue), arg0, arg1) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteQueue", reflect.TypeOf((*MockQueueRepository)(nil).DeleteQueue), ctx, name) } // GetAllQueues mocks base method. -func (m *MockQueueRepository) GetAllQueues(arg0 *armadacontext.Context) ([]queue.Queue, error) { +func (m *MockQueueRepository) GetAllQueues(ctx *armadacontext.Context) ([]queue.Queue, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetAllQueues", arg0) + ret := m.ctrl.Call(m, "GetAllQueues", ctx) ret0, _ := ret[0].([]queue.Queue) ret1, _ := ret[1].(error) return ret0, ret1 } // GetAllQueues indicates an expected call of GetAllQueues. -func (mr *MockQueueRepositoryMockRecorder) GetAllQueues(arg0 interface{}) *gomock.Call { +func (mr *MockQueueRepositoryMockRecorder) GetAllQueues(ctx any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAllQueues", reflect.TypeOf((*MockQueueRepository)(nil).GetAllQueues), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAllQueues", reflect.TypeOf((*MockQueueRepository)(nil).GetAllQueues), ctx) } // GetQueue mocks base method. -func (m *MockQueueRepository) GetQueue(arg0 *armadacontext.Context, arg1 string) (queue.Queue, error) { +func (m *MockQueueRepository) GetQueue(ctx *armadacontext.Context, name string) (queue.Queue, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetQueue", arg0, arg1) + ret := m.ctrl.Call(m, "GetQueue", ctx, name) ret0, _ := ret[0].(queue.Queue) ret1, _ := ret[1].(error) return ret0, ret1 } -// CordonQueue mocks base method. -func (m *MockQueueRepository) CordonQueue(arg0 *armadacontext.Context, arg1 string) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CordonQueue", arg0, arg1) - ret0, _ := ret[0].(error) - return ret0 +// GetQueue indicates an expected call of GetQueue. +func (mr *MockQueueRepositoryMockRecorder) GetQueue(ctx, name any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetQueue", reflect.TypeOf((*MockQueueRepository)(nil).GetQueue), ctx, name) } // UncordonQueue mocks base method. -func (m *MockQueueRepository) UncordonQueue(arg0 *armadacontext.Context, arg1 string) error { +func (m *MockQueueRepository) UncordonQueue(ctx *armadacontext.Context, name string) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "UncordonQueue", arg0, arg1) + ret := m.ctrl.Call(m, "UncordonQueue", ctx, name) ret0, _ := ret[0].(error) return ret0 } -// GetQueue indicates an expected call of GetQueue. -func (mr *MockQueueRepositoryMockRecorder) GetQueue(arg0, arg1 interface{}) *gomock.Call { +// UncordonQueue indicates an expected call of UncordonQueue. +func (mr *MockQueueRepositoryMockRecorder) UncordonQueue(ctx, name any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetQueue", reflect.TypeOf((*MockQueueRepository)(nil).GetQueue), arg0, arg1) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UncordonQueue", reflect.TypeOf((*MockQueueRepository)(nil).UncordonQueue), ctx, name) } // UpdateQueue mocks base method. @@ -118,19 +136,7 @@ func (m *MockQueueRepository) UpdateQueue(arg0 *armadacontext.Context, arg1 queu } // UpdateQueue indicates an expected call of UpdateQueue. -func (mr *MockQueueRepositoryMockRecorder) UpdateQueue(arg0, arg1 interface{}) *gomock.Call { +func (mr *MockQueueRepositoryMockRecorder) UpdateQueue(arg0, arg1 any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateQueue", reflect.TypeOf((*MockQueueRepository)(nil).UpdateQueue), arg0, arg1) } - -// CordonQueue indicates an expected call of CordonQueue. -func (mr *MockQueueRepositoryMockRecorder) CordonQueue(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CordonQueue", reflect.TypeOf((*MockQueueRepository)(nil).CordonQueue), arg0, arg1) -} - -// UncordonQueue indicates an expected call of UncordonQueue. -func (mr *MockQueueRepositoryMockRecorder) UncordonQueue(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UncordonQueue", reflect.TypeOf((*MockQueueRepository)(nil).UncordonQueue), arg0, arg1) -} diff --git a/internal/server/queue/queue_cache_test.go b/internal/server/queue/queue_cache_test.go index f6443466fa7..ecdbe03e950 100644 --- a/internal/server/queue/queue_cache_test.go +++ b/internal/server/queue/queue_cache_test.go @@ -5,9 +5,9 @@ import ( "testing" "time" - "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" "github.com/armadaproject/armada/internal/common/armadacontext" "github.com/armadaproject/armada/internal/server/mocks" diff --git a/internal/server/submit/submit_test.go b/internal/server/submit/submit_test.go index 3a67b2e7209..68bc3736694 100644 --- a/internal/server/submit/submit_test.go +++ b/internal/server/submit/submit_test.go @@ -4,8 +4,8 @@ import ( "testing" "time" - "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" + "go.uber.org/mock/gomock" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" clock "k8s.io/utils/clock/testing" @@ -13,7 +13,7 @@ import ( "github.com/armadaproject/armada/internal/common/armadacontext" "github.com/armadaproject/armada/internal/common/auth/permission" - commonMocks "github.com/armadaproject/armada/internal/common/mocks/jobsetevents" + commonMocks "github.com/armadaproject/armada/internal/common/mocks" "github.com/armadaproject/armada/internal/common/util" "github.com/armadaproject/armada/internal/server/mocks" "github.com/armadaproject/armada/internal/server/permissions" @@ -24,7 +24,7 @@ import ( ) type mockObjects struct { - publisher *commonMocks.MockPublisher + publisher *commonMocks.MockPublisher[*armadaevents.EventSequence] queueRepo *mocks.MockQueueRepository deduplicator *mocks.MockDeduplicator authorizer *mocks.MockActionAuthorizer @@ -33,7 +33,7 @@ type mockObjects struct { func createMocks(t *testing.T) *mockObjects { ctrl := gomock.NewController(t) return &mockObjects{ - publisher: commonMocks.NewMockPublisher(ctrl), + publisher: commonMocks.NewMockPublisher[*armadaevents.EventSequence](ctrl), queueRepo: mocks.NewMockQueueRepository(ctrl), deduplicator: mocks.NewMockDeduplicator(ctrl), authorizer: mocks.NewMockActionAuthorizer(ctrl), diff --git a/tools.yaml b/tools.yaml index 35bc73c2f02..3483cb3e2c7 100644 --- a/tools.yaml +++ b/tools.yaml @@ -2,7 +2,7 @@ # TODO: Use latest goreleaser. After upgrading k8s.io packages. tools: - github.com/go-swagger/go-swagger/cmd/swagger@v0.31.0 -- github.com/golang/mock/mockgen@v1.6.0 +- go.uber.org/mock/mockgen@v0.5.0 - github.com/gordonklaus/ineffassign@v0.0.0-20210914165742-4cc7213b9bc8 - github.com/goreleaser/goreleaser@v1.20.0 - github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway@v1.16.0 From 24f16fca1a273e5e275e3735168eeab67dacab23 Mon Sep 17 00:00:00 2001 From: Chris Martin Date: Thu, 2 Jan 2025 14:18:52 +0000 Subject: [PATCH 02/17] Update Pulsar client to version 0.14 (#4089) * wip * remove mocks * update gomock * update tools * lint * update pulsar mock --- go.mod | 8 +-- go.sum | 85 +++++++++++++++++++++++++--- internal/common/mocks/mock_pulsar.go | 14 +++++ 3 files changed, 96 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index 7ee53d24406..a650830b832 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ toolchain go1.23.1 replace github.com/AthenZ/athenz v1.10.39 => github.com/AthenZ/athenz v1.10.4 require ( - github.com/apache/pulsar-client-go v0.11.0 + github.com/apache/pulsar-client-go v0.14.0 github.com/coreos/go-oidc v2.2.1+incompatible github.com/go-openapi/analysis v0.22.2 github.com/go-openapi/jsonreference v0.20.4 @@ -129,8 +129,8 @@ require ( github.com/gobwas/glob v0.2.3 // indirect github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a // indirect - github.com/golang-jwt/jwt v3.2.2+incompatible // indirect - github.com/golang/snappy v0.0.3 // indirect + github.com/golang-jwt/jwt/v5 v5.2.1 // indirect + github.com/golang/snappy v0.0.4 // indirect github.com/google/btree v1.0.1 // indirect github.com/google/gnostic v0.5.7-v3refs // indirect github.com/google/gofuzz v1.1.0 // indirect @@ -140,6 +140,7 @@ require ( github.com/gorilla/css v1.0.0 // indirect github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect + github.com/hamba/avro/v2 v2.22.2-0.20240625062549-66aad10411d9 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-immutable-radix v1.3.1 // indirect github.com/hashicorp/go-uuid v1.0.3 // indirect @@ -156,7 +157,6 @@ require ( github.com/kylelemons/godebug v1.1.0 // indirect github.com/leodido/go-urn v1.2.4 // indirect github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect - github.com/linkedin/goavro/v2 v2.9.8 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect diff --git a/go.sum b/go.sum index 0b215ee5d93..ac75f1140ad 100644 --- a/go.sum +++ b/go.sum @@ -23,6 +23,8 @@ cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIA cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= +dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 h1:/vQbFIOMbk2FiG/kXiLl8BRyzTWDw7gX/Hz7Dd5eDMs= github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4/go.mod h1:hN7oaIRCjzsZ2dE+yG5k+rsdt3qcwykqK6HVGcKwsw4= @@ -30,6 +32,8 @@ github.com/99designs/keyring v1.2.1 h1:tYLp1ULvO7i3fI5vE21ReQuj99QFSs7lGm0xWyJo8 github.com/99designs/keyring v1.2.1/go.mod h1:fc+wB5KTk9wQ9sDx0kFXB3A0MaeGHM9AwRStKOQ5vOA= github.com/AthenZ/athenz v1.10.4 h1:EhCptJxuPU2BNU0ZUTJRLrNwAFv06zMx0viN+PrV9YA= github.com/AthenZ/athenz v1.10.4/go.mod h1:ZKAbcckIMkqD2UKqBU2amZoynztPrgYcsmZ934LTDH4= +github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= +github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DataDog/zstd v1.5.0 h1:+K/VEwIAaPcHiMtQvpLD4lqW7f0Gk3xdYZmI1hD+CXo= @@ -38,6 +42,10 @@ github.com/IBM/pgxpoolprometheus v1.1.1 h1:xkWNUe87TIuBj/ypdSiDgNYktsuM7MoZCT8a+ github.com/IBM/pgxpoolprometheus v1.1.1/go.mod h1:GFJDkHbidFfB2APbhBTSy2X4PKH3bLWsEMBhmzK1ipo= github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0= github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/Microsoft/hcsshim v0.11.5 h1:haEcLNpj9Ka1gd3B3tAEs9CpE0c+1IhoL59w/exYU38= +github.com/Microsoft/hcsshim v0.11.5/go.mod h1:MV8xMfmECjl5HdO7U/3/hFVnkmSBjAjmA09d4bExKcU= github.com/alecthomas/assert/v2 v2.2.1 h1:XivOgYcduV98QCahG8T5XTezV5bylXe+lBxLG2K2ink= github.com/alecthomas/assert/v2 v2.2.1/go.mod h1:pXcQ2Asjp247dahGEmsZ6ru0UVwnkhktn7S0bBDLxvQ= github.com/alecthomas/chroma/v2 v2.8.0 h1:w9WJUjFFmHHB2e8mRpL9jjy3alYDlU0QLDezj1xE264= @@ -47,8 +55,8 @@ github.com/alecthomas/repr v0.2.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW5 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apache/arrow/go/arrow v0.0.0-20200730104253-651201b0f516 h1:byKBBF2CKWBjjA4J1ZL2JXttJULvWSl50LegTyRZ728= github.com/apache/arrow/go/arrow v0.0.0-20200730104253-651201b0f516/go.mod h1:QNYViu/X0HXDHw7m3KXzWSVXIbfUvJqBFe6Gj8/pYA0= -github.com/apache/pulsar-client-go v0.11.0 h1:fniyVbewAOcMSMLwxzhdrCFmFTorCW40jfnmQVcsrJw= -github.com/apache/pulsar-client-go v0.11.0/go.mod h1:FoijqJwgjroSKptIWp1vvK1CXs8dXnQiL8I+MHOri4A= +github.com/apache/pulsar-client-go v0.14.0 h1:P7yfAQhQ52OCAu8yVmtdbNQ81vV8bF54S2MLmCPJC9w= +github.com/apache/pulsar-client-go v0.14.0/go.mod h1:PNUE29x9G1EHMvm41Bs2vcqwgv7N8AEjeej+nEVYbX8= github.com/apache/thrift v0.0.0-20181112125854-24918abba929/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.14.2 h1:hY4rAyg7Eqbb27GB6gkhUKrRAuc8xRjlNtJq+LseKeY= github.com/apache/thrift v0.14.2/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= @@ -84,6 +92,8 @@ github.com/caarlos0/log v0.4.4/go.mod h1:+AmCI9Liv5LKXmzFmFI1htuHdTTj/0R3KuoP9DM github.com/caarlos0/testfs v0.4.4 h1:3PHvzHi5Lt+g332CiShwS8ogTgS3HjrmzZxCm6JCDr8= github.com/caarlos0/testfs v0.4.4/go.mod h1:bRN55zgG4XCUVVHZCeU+/Tz1Q6AxEJOEJTliBy+1DMk= github.com/cenkalti/backoff/v4 v4.0.0/go.mod h1:eEew/i+1Q6OrCDZh3WiXYv3+nJwBASZ8Bog/87DQnVg= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -97,8 +107,16 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/colinmarc/hdfs/v2 v2.1.1/go.mod h1:M3x+k8UKKmxtFu++uAZ0OtDU8jR3jnaZIAc6yK4Ue0c= +github.com/containerd/containerd v1.7.18 h1:jqjZTQNfXGoEaZdW1WwPU0RqSn1Bm2Ay/KJPUuO8nao= +github.com/containerd/containerd v1.7.18/go.mod h1:IYEk9/IO6wAPUz2bCMVUbsfXjzw5UNP5fLz4PsUygQ4= +github.com/containerd/errdefs v0.1.0 h1:m0wCRBiu1WJT/Fr+iOoQHMQS/eP5myQ8lCv4Dz5ZURM= +github.com/containerd/errdefs v0.1.0/go.mod h1:YgWiiHtLmSeBrvpw+UfPijzbLaB77mEG1WwJTDETIV0= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= github.com/coreos/go-oidc v2.2.1+incompatible h1:mh48q/BqXqgjVHpy2ZY7WnWAbenxRjsz9N1i1YxjHAk= github.com/coreos/go-oidc v2.2.1+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= +github.com/cpuguy83/dockercfg v0.3.1 h1:/FpZ+JaygUR/lZP2NlFI2DVfrOEMAIKP5wWEJdoYe9E= +github.com/cpuguy83/dockercfg v0.3.1/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/danieljoos/wincred v1.1.2 h1:QLdCxFs1/Yl4zduvBdcHB8goaYk9RARS2SgLLRuAyr0= github.com/danieljoos/wincred v1.1.2/go.mod h1:GijpziifJoIBfYh+S7BbkdUTU4LfM+QnGqR5Vl2tAx0= @@ -111,8 +129,14 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/dimfeld/httptreemux v5.0.1+incompatible h1:Qj3gVcDNoOthBAqftuD596rm4wg/adLLz5xh5CmpiCA= github.com/dimfeld/httptreemux v5.0.1+incompatible/go.mod h1:rbUlSV+CCpv/SuqUTP/8Bk2O3LyUV436/yaRGkhP6Z0= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/dlclark/regexp2 v1.4.0 h1:F1rxgk7p4uKjwIQxBs9oAXe5CqrXlCduYEJvrF4u93E= github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= +github.com/docker/docker v27.1.1+incompatible h1:hO/M4MtV36kzKldqnA37IWhebRA+LnqqcqDja6kVaKY= +github.com/docker/docker v27.1.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= +github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= @@ -130,6 +154,8 @@ github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/camelcase v1.0.0 h1:hxNvNX/xYBp0ovncs8WyWZrOrpBNub/JfaMvbURyft8= github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= @@ -149,6 +175,10 @@ github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-openapi/analysis v0.22.2 h1:ZBmNoP2h5omLKr/srIC9bfqrUGzT6g6gNv03HE9Vpj0= github.com/go-openapi/analysis v0.22.2/go.mod h1:pDF4UbZsQTo/oNuRfAWWd4dAh4yuYf//LYorPTjrpvo= github.com/go-openapi/errors v0.21.0 h1:FhChC/duCnfoLj1gZ0BgaBmzhJC2SL/sJr8a2vAobSY= @@ -189,8 +219,8 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/gogo/status v1.1.1 h1:DuHXlSFHNKqTQ+/ACf5Vs6r4X/dH2EgIzR9Vr+H65kg= github.com/gogo/status v1.1.1/go.mod h1:jpG3dM5QPcqu19Hg8lkUhBFBa3TcLs1DG7+2Jqci7oU= -github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= -github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= +github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= +github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -219,8 +249,9 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= @@ -275,6 +306,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c h1:6rhixN/i8ZofjG1Y75iExal34USq5p+wiN1tpie8IrU= github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c/go.mod h1:NMPJylDgVpX0MLRlPy15sqSwOFv/U1GZ2m21JhFfek0= +github.com/hamba/avro/v2 v2.22.2-0.20240625062549-66aad10411d9 h1:NEoabXt33PDWK4fXryK4e+XX+fSKDmmu9vg3yb9YI2M= +github.com/hamba/avro/v2 v2.22.2-0.20240625062549-66aad10411d9/go.mod h1:fQVdB2mFZBhPW1D5Abej41LMvrErARGrrdjOnKbm5yw= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -351,10 +384,10 @@ github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= -github.com/linkedin/goavro/v2 v2.9.8 h1:jN50elxBsGBDGVDEKqUlDuU1cFwJ11K/yrJCBMe/7Wg= -github.com/linkedin/goavro/v2 v2.9.8/go.mod h1:UgQUb2N/pmueQYH9bfqFioWxzYCZXSfF8Jw03O5sjqA= github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/magefile/mage v1.14.0 h1:6QDX3g6z1YvJ4olPhT1wksUcSa/V0a1B+pJb73fBjyo= github.com/magefile/mage v1.14.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= @@ -379,6 +412,16 @@ github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= +github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= +github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc= +github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo= +github.com/moby/sys/user v0.1.0 h1:WmZ93f5Ux6het5iituh9x2zAG7NFY9Aqi49jjE1PaQg= +github.com/moby/sys/user v0.1.0/go.mod h1:fKJhFOnsCN6xZ5gSfbM6zaHGgDJMrqt9/reuj4T7MmU= +github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= +github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -387,6 +430,8 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/mtibben/percent v0.2.1 h1:5gssi8Nqo8QU/r2pynCm+hBQHpkB/uNK7BJCFogWdzs= github.com/mtibben/percent v0.2.1/go.mod h1:KG9uO+SZkUp+VkRHsCdYQV3XSZrrSpR3O9ibNBTZrns= github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= @@ -413,6 +458,10 @@ github.com/openconfig/goyang v0.0.0-20200115183954-d0a48929f0ea/go.mod h1:dhXaV0 github.com/openconfig/goyang v1.2.0 h1:mChUZvp1kCWq6Q00wVCtOToddFzEsGlMGG+V+wNXva8= github.com/openconfig/goyang v1.2.0/go.mod h1:vX61x01Q46AzbZUzG617vWqh/cB+aisc+RrNkXRd3W8= github.com/openconfig/ygot v0.6.0/go.mod h1:o30svNf7O0xK+R35tlx95odkDmZWS9JyWWQSmIhqwAs= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= +github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= @@ -432,6 +481,8 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/pquerna/cachecontrol v0.2.0 h1:vBXSNuE5MYP9IJ5kjsdo8uq+w41jSPgvba2DEnkRx9k= github.com/pquerna/cachecontrol v0.2.0/go.mod h1:NrUG3Z7Rdu85UNR3vm7SOsl1nFIeSiQnrHV5K9mBcUI= github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= @@ -466,6 +517,10 @@ github.com/segmentio/fasthash v1.0.3 h1:EI9+KE1EwvMLBWwjpRDc+fEM+prwxDYbslddQGtr github.com/segmentio/fasthash v1.0.3/go.mod h1:waKX8l2N8yckOgmSsXJi7x1ZfdKZ4x7KRMzBtS3oedY= github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/shirou/gopsutil/v3 v3.23.12 h1:z90NtUkp3bMtmICZKpC4+WaknU1eXtp5vtbQ11DgpE4= +github.com/shirou/gopsutil/v3 v3.23.12/go.mod h1:1FrWgea594Jp7qmjHUUPlJDTPgcsb9mGnXDxavtikzM= +github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= +github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= @@ -506,6 +561,12 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +github.com/testcontainers/testcontainers-go v0.32.0 h1:ug1aK08L3gCHdhknlTTwWjPHPS+/alvLJU/DRxTD/ME= +github.com/testcontainers/testcontainers-go v0.32.0/go.mod h1:CRHrzHLQhlXUsa5gXjTOfqIEJcrK5+xMDmBr/WMI88E= +github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= +github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= +github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= +github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= github.com/weaveworks/promrus v1.2.0 h1:jOLf6pe6/vss4qGHjXmGz4oDJQA+AOCqEL3FvvZGz7M= github.com/weaveworks/promrus v1.2.0/go.mod h1:SaE82+OJ91yqjrE1rsvBWVzNZKcHYFtMUyS1+Ogs/KA= github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= @@ -530,12 +591,22 @@ github.com/yuin/goldmark v1.5.4 h1:2uY/xC0roWy8IBEGLgB1ywIoEJFGmRrX21YQcvGZzjU= github.com/yuin/goldmark v1.5.4/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/goldmark-emoji v1.0.2 h1:c/RgTShNgHTtc6xdz2KKI74jJr6rWi7FPgnP9GAsO5s= github.com/yuin/goldmark-emoji v1.0.2/go.mod h1:RhP/RWpexdp+KHs7ghKnifRoIs/Bq4nDS7tRbCkOwKY= +github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= +github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.mongodb.org/mongo-driver v1.13.1 h1:YIc7HTYsKndGK4RFzJ3covLz1byri52x0IoMB0Pt/vk= go.mongodb.org/mongo-driver v1.13.1/go.mod h1:wcDf1JBCXy2mOW0bWHwO/IOYqdca1MPCwDtFu/Z9+eo= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= +go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= +go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= +go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= +go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= +go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= +go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 h1:+FNtrFTmVw0YZGpBGX56XDee331t6JAXeK2bcyhLOOc= go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5/go.mod h1:nmDLcffg48OtT/PSW0Hg7FvpRQsQh5OSqIylirxKC7o= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= diff --git a/internal/common/mocks/mock_pulsar.go b/internal/common/mocks/mock_pulsar.go index 910d9d315b1..9ed8371d20b 100644 --- a/internal/common/mocks/mock_pulsar.go +++ b/internal/common/mocks/mock_pulsar.go @@ -194,6 +194,20 @@ func (mr *MockProducerMockRecorder) Flush() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Flush", reflect.TypeOf((*MockProducer)(nil).Flush)) } +// FlushWithCtx mocks base method. +func (m *MockProducer) FlushWithCtx(ctx context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "FlushWithCtx", ctx) + ret0, _ := ret[0].(error) + return ret0 +} + +// FlushWithCtx indicates an expected call of FlushWithCtx. +func (mr *MockProducerMockRecorder) FlushWithCtx(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FlushWithCtx", reflect.TypeOf((*MockProducer)(nil).FlushWithCtx), ctx) +} + // LastSequenceID mocks base method. func (m *MockProducer) LastSequenceID() int64 { m.ctrl.T.Helper() From df5401c23e2454bca8642faaec7bd547455d74a0 Mon Sep 17 00:00:00 2001 From: Chris Martin Date: Fri, 3 Jan 2025 09:25:43 +0000 Subject: [PATCH 03/17] update alpine to 3.21.0 (#4111) --- .goreleaser.yml | 2 +- build/armada-load-tester/Dockerfile | 2 +- build/armadactl/Dockerfile | 2 +- build/binoculars/Dockerfile | 2 +- build/bundles/armada/Dockerfile | 2 +- build/bundles/full/Dockerfile | 2 +- build/bundles/lookout/Dockerfile | 2 +- build/eventingester/Dockerfile | 2 +- build/executor/Dockerfile | 2 +- build/fakeexecutor/Dockerfile | 2 +- build/loadtester/Dockerfile | 2 +- build/lookoutingesterv2/Dockerfile | 2 +- build/lookoutv2/Dockerfile | 2 +- build/scheduler/Dockerfile | 2 +- build/scheduleringester/Dockerfile | 2 +- build/server/Dockerfile | 2 +- build/testsuite/Dockerfile | 2 +- cmd/armada-load-tester/cmd/loadtest.go | 2 +- e2e/armadactl_test/armadactl_test.go | 2 +- third_party/airflow/README.md | 2 +- 20 files changed, 20 insertions(+), 20 deletions(-) diff --git a/.goreleaser.yml b/.goreleaser.yml index cdab09fda9c..9172b64c6b0 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -186,7 +186,7 @@ dockers: - --label=org.opencontainers.image.version={{ .Version }} - --label=org.opencontainers.image.created={{ time "2006-01-02T15:04:05Z07:00" }} - --label=org.opencontainers.image.revision={{ .FullCommit }} - - --label=org.opencontainers.image.base.name=alpine:3.20.3 + - --label=org.opencontainers.image.base.name=alpine:3.21.0 - --label=org.opencontainers.image.licenses=Apache-2.0 - --label=org.opencontainers.image.vendor=G-Research ids: diff --git a/build/armada-load-tester/Dockerfile b/build/armada-load-tester/Dockerfile index 7d18149b427..31db4074dab 100644 --- a/build/armada-load-tester/Dockerfile +++ b/build/armada-load-tester/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.20.3 +FROM alpine:3.21.0 RUN addgroup -S -g 2000 armada && adduser -S -u 1000 armada -G armada diff --git a/build/armadactl/Dockerfile b/build/armadactl/Dockerfile index 1ca42d48402..36166cddb6d 100644 --- a/build/armadactl/Dockerfile +++ b/build/armadactl/Dockerfile @@ -1,4 +1,4 @@ -ARG BASE_IMAGE=alpine:3.20.3 +ARG BASE_IMAGE=alpine:3.21.0 FROM ${BASE_IMAGE} LABEL org.opencontainers.image.title=armadactl diff --git a/build/binoculars/Dockerfile b/build/binoculars/Dockerfile index 44dd4e15b7c..8d58cc1d228 100644 --- a/build/binoculars/Dockerfile +++ b/build/binoculars/Dockerfile @@ -1,4 +1,4 @@ -ARG BASE_IMAGE=alpine:3.20.3 +ARG BASE_IMAGE=alpine:3.21.0 FROM ${BASE_IMAGE} LABEL org.opencontainers.image.title=binoculars LABEL org.opencontainers.image.description="binoculars" diff --git a/build/bundles/armada/Dockerfile b/build/bundles/armada/Dockerfile index 3b989d0b0fc..0955bd18d75 100644 --- a/build/bundles/armada/Dockerfile +++ b/build/bundles/armada/Dockerfile @@ -1,4 +1,4 @@ -ARG BASE_IMAGE=alpine:3.20.3 +ARG BASE_IMAGE=alpine:3.21.0 FROM ${BASE_IMAGE} LABEL org.opencontainers.image.title=armada LABEL org.opencontainers.image.description="Armada Bundle" diff --git a/build/bundles/full/Dockerfile b/build/bundles/full/Dockerfile index 26fb977d92e..5199e21239b 100644 --- a/build/bundles/full/Dockerfile +++ b/build/bundles/full/Dockerfile @@ -1,6 +1,6 @@ ARG NODE_BUILD_IMAGE=node:22.12-bullseye ARG OPENAPI_BUILD_IMAGE=openapitools/openapi-generator-cli:v5.4.0 -ARG BASE_IMAGE=alpine:3.20.3 +ARG BASE_IMAGE=alpine:3.21.0 FROM ${OPENAPI_BUILD_IMAGE} AS OPENAPI LABEL org.opencontainers.image.title=armada-full-bundle diff --git a/build/bundles/lookout/Dockerfile b/build/bundles/lookout/Dockerfile index cde952af59f..c01574341f8 100644 --- a/build/bundles/lookout/Dockerfile +++ b/build/bundles/lookout/Dockerfile @@ -1,4 +1,4 @@ -ARG BASE_IMAGE=alpine:3.20.3 +ARG BASE_IMAGE=alpine:3.21.0 FROM ${BASE_IMAGE} LABEL org.opencontainers.image.title=armada-lookout-bundle diff --git a/build/eventingester/Dockerfile b/build/eventingester/Dockerfile index 395792b0013..2b1a8bb8d0b 100644 --- a/build/eventingester/Dockerfile +++ b/build/eventingester/Dockerfile @@ -1,4 +1,4 @@ -ARG BASE_IMAGE=alpine:3.20.3 +ARG BASE_IMAGE=alpine:3.21.0 FROM ${BASE_IMAGE} LABEL org.opencontainers.image.title=eventingester diff --git a/build/executor/Dockerfile b/build/executor/Dockerfile index 083a7141999..343896ad9d7 100644 --- a/build/executor/Dockerfile +++ b/build/executor/Dockerfile @@ -1,4 +1,4 @@ -ARG BASE_IMAGE=alpine:3.20.3 +ARG BASE_IMAGE=alpine:3.21.0 FROM ${BASE_IMAGE} LABEL org.opencontainers.image.title=executor diff --git a/build/fakeexecutor/Dockerfile b/build/fakeexecutor/Dockerfile index bae8be1cb75..efdb47f30aa 100644 --- a/build/fakeexecutor/Dockerfile +++ b/build/fakeexecutor/Dockerfile @@ -1,4 +1,4 @@ -ARG BASE_IMAGE=alpine:3.20.3 +ARG BASE_IMAGE=alpine:3.21.0 FROM ${BASE_IMAGE} LABEL org.opencontainers.image.title=fakeexecutor LABEL org.opencontainers.image.description="Fake Executor" diff --git a/build/loadtester/Dockerfile b/build/loadtester/Dockerfile index 543c14617a1..fa763ecfb4b 100644 --- a/build/loadtester/Dockerfile +++ b/build/loadtester/Dockerfile @@ -1,4 +1,4 @@ -ARG BASE_IMAGE=alpine:3.20.3 +ARG BASE_IMAGE=alpine:3.21.0 FROM ${BASE_IMAGE} LABEL org.opencontainers.image.title=loadtester diff --git a/build/lookoutingesterv2/Dockerfile b/build/lookoutingesterv2/Dockerfile index 5c7ff56235c..c2f953facd2 100644 --- a/build/lookoutingesterv2/Dockerfile +++ b/build/lookoutingesterv2/Dockerfile @@ -1,4 +1,4 @@ -ARG BASE_IMAGE=alpine:3.20.3 +ARG BASE_IMAGE=alpine:3.21.0 FROM ${BASE_IMAGE} LABEL org.opencontainers.image.title=lookoutingesterv2 diff --git a/build/lookoutv2/Dockerfile b/build/lookoutv2/Dockerfile index 4aaaf1d009c..79b259acc98 100644 --- a/build/lookoutv2/Dockerfile +++ b/build/lookoutv2/Dockerfile @@ -1,6 +1,6 @@ ARG NODE_BUILD_IMAGE=node:22.12-bullseye ARG OPENAPI_BUILD_IMAGE=openapitools/openapi-generator-cli:v5.4.0 -ARG BASE_IMAGE=alpine:3.20.3 +ARG BASE_IMAGE=alpine:3.21.0 FROM ${OPENAPI_BUILD_IMAGE} AS OPENAPI diff --git a/build/scheduler/Dockerfile b/build/scheduler/Dockerfile index 982bdc07af3..6aa5524bb7f 100644 --- a/build/scheduler/Dockerfile +++ b/build/scheduler/Dockerfile @@ -1,4 +1,4 @@ -ARG BASE_IMAGE=alpine:3.20.3 +ARG BASE_IMAGE=alpine:3.21.0 FROM ${BASE_IMAGE} LABEL org.opencontainers.image.title=scheduler diff --git a/build/scheduleringester/Dockerfile b/build/scheduleringester/Dockerfile index 9ce05f525ec..cf86787e726 100644 --- a/build/scheduleringester/Dockerfile +++ b/build/scheduleringester/Dockerfile @@ -1,4 +1,4 @@ -ARG BASE_IMAGE=alpine:3.20.3 +ARG BASE_IMAGE=alpine:3.21.0 FROM ${BASE_IMAGE} LABEL org.opencontainers.image.title=scheduleringester diff --git a/build/server/Dockerfile b/build/server/Dockerfile index c827f1b5d6e..4453057c381 100644 --- a/build/server/Dockerfile +++ b/build/server/Dockerfile @@ -1,4 +1,4 @@ -ARG BASE_IMAGE=alpine:3.20.3 +ARG BASE_IMAGE=alpine:3.21.0 FROM ${BASE_IMAGE} LABEL org.opencontainers.image.title=armada-server diff --git a/build/testsuite/Dockerfile b/build/testsuite/Dockerfile index bfd2ecce2cd..1eca00eaddf 100644 --- a/build/testsuite/Dockerfile +++ b/build/testsuite/Dockerfile @@ -1,4 +1,4 @@ -ARG BASE_IMAGE=alpine:3.20.3 +ARG BASE_IMAGE=alpine:3.21.0 FROM ${BASE_IMAGE} LABEL org.opencontainers.image.title=testsuite LABEL org.opencontainers.image.description="Test Suite" diff --git a/cmd/armada-load-tester/cmd/loadtest.go b/cmd/armada-load-tester/cmd/loadtest.go index a5805c73905..5a2e301490e 100644 --- a/cmd/armada-load-tester/cmd/loadtest.go +++ b/cmd/armada-load-tester/cmd/loadtest.go @@ -52,7 +52,7 @@ var loadtestCmd = &cobra.Command{ containers: - name: sleep imagePullPolicy: IfNotPresent - image: alpine:3.20.3 + image: alpine:3.21.0 command: - sh args: diff --git a/e2e/armadactl_test/armadactl_test.go b/e2e/armadactl_test/armadactl_test.go index a44a01f9a28..d1debbe9dc7 100644 --- a/e2e/armadactl_test/armadactl_test.go +++ b/e2e/armadactl_test/armadactl_test.go @@ -175,7 +175,7 @@ jobs: containers: - name: ls imagePullPolicy: IfNotPresent - image: alpine:3.20.3 + image: alpine:3.21.0 command: - sh - -c diff --git a/third_party/airflow/README.md b/third_party/airflow/README.md index 310695a6260..725460abe45 100644 --- a/third_party/airflow/README.md +++ b/third_party/airflow/README.md @@ -39,7 +39,7 @@ def create_dummy_job(): containers=[ core_v1.Container( name="sleep", - image="alpine:3.20.3", + image="alpine:3.21.0", args=["sh", "-c", "for i in $(seq 1 60); do echo $i; sleep 1; done"], securityContext=core_v1.SecurityContext(runAsUser=1000), resources=core_v1.ResourceRequirements( From e1e91a9cbe38c632257ad68ef3ad29dd4cec705b Mon Sep 17 00:00:00 2001 From: Rob Smith <34475852+robertdavidsmith@users.noreply.github.com> Date: Fri, 3 Jan 2025 11:40:18 +0000 Subject: [PATCH 04/17] Scheduler: refactor: move scheduling tests to internaltypes (#4113) Signed-off-by: Robert Smith --- .../internaltypes/resource_list_map_util.go | 1 + .../resource_list_map_util_test.go | 3 +- internal/scheduler/nodedb/nodedb_test.go | 6 +- .../scheduler/nodedb/nodeiteration_test.go | 169 ++++++-------- .../scheduling/fairness/fairness_test.go | 218 +++++------------- .../scheduling/gang_scheduler_test.go | 153 ++++++------ ..._driven_preempting_queue_scheduler_test.go | 31 ++- .../preempting_queue_scheduler_test.go | 202 ++++++++-------- .../scheduling/queue_scheduler_test.go | 118 +++++----- .../scheduling/scheduling_algo_test.go | 14 +- .../scheduler/testfixtures/testfixtures.go | 60 ++--- 11 files changed, 388 insertions(+), 587 deletions(-) diff --git a/internal/scheduler/internaltypes/resource_list_map_util.go b/internal/scheduler/internaltypes/resource_list_map_util.go index b851e82c6cc..a26a1a80f6d 100644 --- a/internal/scheduler/internaltypes/resource_list_map_util.go +++ b/internal/scheduler/internaltypes/resource_list_map_util.go @@ -63,6 +63,7 @@ func NewAllocatableByPriorityAndResourceType(priorities []int32, rl ResourceList for _, priority := range priorities { result[priority] = rl } + result[EvictedPriority] = rl return result } diff --git a/internal/scheduler/internaltypes/resource_list_map_util_test.go b/internal/scheduler/internaltypes/resource_list_map_util_test.go index 85457c42a1a..5e2b61f729f 100644 --- a/internal/scheduler/internaltypes/resource_list_map_util_test.go +++ b/internal/scheduler/internaltypes/resource_list_map_util_test.go @@ -83,9 +83,10 @@ func TestNewAllocatableByPriorityAndResourceType(t *testing.T) { rl := testResourceList(factory, "2", "2Ki") result := NewAllocatableByPriorityAndResourceType([]int32{1, 2}, rl) - assert.Equal(t, 2, len(result)) + assert.Equal(t, 3, len(result)) assert.Equal(t, int64(2000), result[1].GetByNameZeroIfMissing("cpu")) assert.Equal(t, int64(2000), result[2].GetByNameZeroIfMissing("cpu")) + assert.Equal(t, int64(2000), result[EvictedPriority].GetByNameZeroIfMissing("cpu")) } func TestMarkAllocated(t *testing.T) { diff --git a/internal/scheduler/nodedb/nodedb_test.go b/internal/scheduler/nodedb/nodedb_test.go index 5e87f425876..84eb691f41b 100644 --- a/internal/scheduler/nodedb/nodedb_test.go +++ b/internal/scheduler/nodedb/nodedb_test.go @@ -833,7 +833,7 @@ func BenchmarkScheduleMany100CpuNodes1CpuUnused(b *testing.B) { b, testfixtures.ItWithUsedResourcesNodes( 0, - cpu("31"), + testfixtures.Cpu("31"), testfixtures.ItN32CpuNodes(100, testfixtures.TestPriorities), ), testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 100), @@ -845,7 +845,7 @@ func BenchmarkScheduleMany1000CpuNodes1CpuUnused(b *testing.B) { b, testfixtures.ItWithUsedResourcesNodes( 0, - cpu("31"), + testfixtures.Cpu("31"), testfixtures.ItN32CpuNodes(1000, testfixtures.TestPriorities), ), testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1000), @@ -857,7 +857,7 @@ func BenchmarkScheduleMany10000CpuNodes1CpuUnused(b *testing.B) { b, testfixtures.ItWithUsedResourcesNodes( 0, - cpu("31"), + testfixtures.Cpu("31"), testfixtures.ItN32CpuNodes(10000, testfixtures.TestPriorities), ), testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 10000), diff --git a/internal/scheduler/nodedb/nodeiteration_test.go b/internal/scheduler/nodedb/nodeiteration_test.go index aa699293139..27f9d901329 100644 --- a/internal/scheduler/nodedb/nodeiteration_test.go +++ b/internal/scheduler/nodedb/nodeiteration_test.go @@ -112,24 +112,24 @@ func TestNodeTypeIterator(t *testing.T) { armadaslices.Concatenate( testfixtures.ItWithUsedResourcesNodes( 0, - cpu("15"), + testfixtures.Cpu("15"), testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), ), testfixtures.ItWithUsedResourcesNodes( 0, - cpu("16"), + testfixtures.Cpu("16"), testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), ), testfixtures.ItWithUsedResourcesNodes( 0, - cpu("17"), + testfixtures.Cpu("17"), testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), ), ), ), nodeTypeId: nodeTypeA.GetId(), priority: 0, - resourceRequests: cpu("16"), + resourceRequests: testfixtures.Cpu("16"), expected: []int{1, 0}, }, "filter nodes with insufficient resources at priority and return in increasing order": { @@ -138,54 +138,54 @@ func TestNodeTypeIterator(t *testing.T) { armadaslices.Concatenate( testfixtures.ItWithUsedResourcesNodes( 0, - cpu("15"), + testfixtures.Cpu("15"), testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), ), testfixtures.ItWithUsedResourcesNodes( 0, - cpu("16"), + testfixtures.Cpu("16"), testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), ), testfixtures.ItWithUsedResourcesNodes( 0, - cpu("17"), + testfixtures.Cpu("17"), testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), ), testfixtures.ItWithUsedResourcesNodes( 1, - cpu("15"), + testfixtures.Cpu("15"), testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), ), testfixtures.ItWithUsedResourcesNodes( 1, - cpu("16"), + testfixtures.Cpu("16"), testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), ), testfixtures.ItWithUsedResourcesNodes( 1, - cpu("17"), + testfixtures.Cpu("17"), testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), ), testfixtures.ItWithUsedResourcesNodes( 2, - cpu("15"), + testfixtures.Cpu("15"), testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), ), testfixtures.ItWithUsedResourcesNodes( 2, - cpu("16"), + testfixtures.Cpu("16"), testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), ), testfixtures.ItWithUsedResourcesNodes( 2, - cpu("17"), + testfixtures.Cpu("17"), testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), ), ), ), nodeTypeId: nodeTypeA.GetId(), priority: 1, - resourceRequests: cpu("16"), + resourceRequests: testfixtures.Cpu("16"), expected: []int{4, 7, 3, 6, 0, 1, 2}, }, "nested ordering": { @@ -194,54 +194,54 @@ func TestNodeTypeIterator(t *testing.T) { armadaslices.Concatenate( testfixtures.ItWithUsedResourcesNodes( 0, - cpuMem("15", "1Gi"), + testfixtures.CpuMem("15", "1Gi"), testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), ), testfixtures.ItWithUsedResourcesNodes( 0, - cpuMem("15", "2Gi"), + testfixtures.CpuMem("15", "2Gi"), testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), ), testfixtures.ItWithUsedResourcesNodes( 0, - cpuMem("15", "129Gi"), + testfixtures.CpuMem("15", "129Gi"), testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), ), testfixtures.ItWithUsedResourcesNodes( 0, - cpuMem("15", "130Gi"), + testfixtures.CpuMem("15", "130Gi"), testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), ), testfixtures.ItWithUsedResourcesNodes( 0, - cpuMem("15", "131Gi"), + testfixtures.CpuMem("15", "131Gi"), testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), ), testfixtures.ItWithUsedResourcesNodes( 0, - cpuMem("16", "130Gi"), + testfixtures.CpuMem("16", "130Gi"), testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), ), testfixtures.ItWithUsedResourcesNodes( 0, - cpuMem("16", "128Gi"), + testfixtures.CpuMem("16", "128Gi"), testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), ), testfixtures.ItWithUsedResourcesNodes( 0, - cpuMem("16", "129Gi"), + testfixtures.CpuMem("16", "129Gi"), testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), ), testfixtures.ItWithUsedResourcesNodes( 0, - cpu("17"), + testfixtures.Cpu("17"), testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), ), ), ), nodeTypeId: nodeTypeA.GetId(), priority: 0, - resourceRequests: cpuMem("16", "128Gi"), + resourceRequests: testfixtures.CpuMem("16", "128Gi"), expected: []int{6, 1, 0}, }, "double-nested ordering": { @@ -250,59 +250,59 @@ func TestNodeTypeIterator(t *testing.T) { armadaslices.Concatenate( testfixtures.ItWithUsedResourcesNodes( 0, - cpuMem("31", "1Gi"), + testfixtures.CpuMem("31", "1Gi"), testfixtures.ItN8GpuNodes(1, testfixtures.TestPriorities), ), testfixtures.ItWithUsedResourcesNodes( 0, - cpuMemGpu("31", "1Gi", "1"), + testfixtures.CpuMemGpu("31", "1Gi", "1"), testfixtures.ItN8GpuNodes(1, testfixtures.TestPriorities), ), testfixtures.ItWithUsedResourcesNodes( 0, - cpuMemGpu("31", "1Gi", "2"), + testfixtures.CpuMemGpu("31", "1Gi", "2"), testfixtures.ItN8GpuNodes(1, testfixtures.TestPriorities), ), testfixtures.ItWithUsedResourcesNodes( 0, - cpuMemGpu("31", "1Gi", "5"), + testfixtures.CpuMemGpu("31", "1Gi", "5"), testfixtures.ItN8GpuNodes(1, testfixtures.TestPriorities), ), testfixtures.ItWithUsedResourcesNodes( 0, - cpuMem("31", "2Gi"), + testfixtures.CpuMem("31", "2Gi"), testfixtures.ItN8GpuNodes(1, testfixtures.TestPriorities), ), testfixtures.ItWithUsedResourcesNodes( 0, - cpuMemGpu("31", "2Gi", "1"), + testfixtures.CpuMemGpu("31", "2Gi", "1"), testfixtures.ItN8GpuNodes(1, testfixtures.TestPriorities), ), testfixtures.ItWithUsedResourcesNodes( 0, - cpuMem("32", "514Gi"), + testfixtures.CpuMem("32", "514Gi"), testfixtures.ItN8GpuNodes(1, testfixtures.TestPriorities), ), testfixtures.ItWithUsedResourcesNodes( 0, - cpuMem("32", "512Gi"), + testfixtures.CpuMem("32", "512Gi"), testfixtures.ItN8GpuNodes(1, testfixtures.TestPriorities), ), testfixtures.ItWithUsedResourcesNodes( 0, - cpuMem("32", "513Gi"), + testfixtures.CpuMem("32", "513Gi"), testfixtures.ItN8GpuNodes(1, testfixtures.TestPriorities), ), testfixtures.ItWithUsedResourcesNodes( 0, - cpu("33"), + testfixtures.Cpu("33"), testfixtures.ItN8GpuNodes(1, testfixtures.TestPriorities), ), ), ), nodeTypeId: nodeTypeA.GetId(), priority: 0, - resourceRequests: cpuMemGpu("32", "512Gi", "4"), + resourceRequests: testfixtures.CpuMemGpu("32", "512Gi", "4"), expected: []int{7, 5, 4, 2, 1, 0}, }, } @@ -414,7 +414,7 @@ func TestNodeTypesIterator(t *testing.T) { testfixtures.ItWithNodeTypeNodes( nodeTypeA, testfixtures.ItWithUsedResourcesNodes(0, - cpu("15"), + testfixtures.Cpu("15"), testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), ), ), @@ -422,7 +422,7 @@ func TestNodeTypesIterator(t *testing.T) { nodeTypeB, testfixtures.ItWithUsedResourcesNodes( 0, - cpu("16"), + testfixtures.Cpu("16"), testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), ), ), @@ -430,7 +430,7 @@ func TestNodeTypesIterator(t *testing.T) { nodeTypeC, testfixtures.ItWithUsedResourcesNodes( 0, - cpu("17"), + testfixtures.Cpu("17"), testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), ), ), @@ -438,14 +438,14 @@ func TestNodeTypesIterator(t *testing.T) { nodeTypeD, testfixtures.ItWithUsedResourcesNodes( 0, - cpu("14"), + testfixtures.Cpu("14"), testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), ), ), ), nodeTypeIds: []uint64{nodeTypeA.GetId(), nodeTypeB.GetId(), nodeTypeC.GetId()}, priority: 0, - resourceRequests: cpu("16"), + resourceRequests: testfixtures.Cpu("16"), expected: []int{1, 0}, }, "filter nodes with insufficient resources at priority and return in increasing order": { @@ -454,54 +454,54 @@ func TestNodeTypesIterator(t *testing.T) { armadaslices.Concatenate( testfixtures.ItWithUsedResourcesNodes( 0, - cpu("15"), + testfixtures.Cpu("15"), testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), ), testfixtures.ItWithUsedResourcesNodes( 0, - cpu("16"), + testfixtures.Cpu("16"), testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), ), testfixtures.ItWithUsedResourcesNodes( 0, - cpu("17"), + testfixtures.Cpu("17"), testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), ), testfixtures.ItWithUsedResourcesNodes( 1, - cpu("15"), + testfixtures.Cpu("15"), testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), ), testfixtures.ItWithUsedResourcesNodes( 1, - cpu("16"), + testfixtures.Cpu("16"), testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), ), testfixtures.ItWithUsedResourcesNodes( 1, - cpu("17"), + testfixtures.Cpu("17"), testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), ), testfixtures.ItWithUsedResourcesNodes( 2, - cpu("15"), + testfixtures.Cpu("15"), testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), ), testfixtures.ItWithUsedResourcesNodes( 2, - cpu("16"), + testfixtures.Cpu("16"), testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), ), testfixtures.ItWithUsedResourcesNodes( 2, - cpu("17"), + testfixtures.Cpu("17"), testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), ), ), ), nodeTypeIds: []uint64{nodeTypeA.GetId()}, priority: 1, - resourceRequests: cpu("16"), + resourceRequests: testfixtures.Cpu("16"), expected: []int{4, 7, 3, 6, 0, 1, 2}, }, "nested ordering": { @@ -510,54 +510,54 @@ func TestNodeTypesIterator(t *testing.T) { armadaslices.Concatenate( testfixtures.ItWithUsedResourcesNodes( 0, - cpuMem("15", "1Gi"), + testfixtures.CpuMem("15", "1Gi"), testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), ), testfixtures.ItWithUsedResourcesNodes( 0, - cpuMem("15", "2Gi"), + testfixtures.CpuMem("15", "2Gi"), testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), ), testfixtures.ItWithUsedResourcesNodes( 0, - cpuMem("15", "129Gi"), + testfixtures.CpuMem("15", "129Gi"), testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), ), testfixtures.ItWithUsedResourcesNodes( 0, - cpuMem("15", "130Gi"), + testfixtures.CpuMem("15", "130Gi"), testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), ), testfixtures.ItWithUsedResourcesNodes( 0, - cpuMem("15", "131Gi"), + testfixtures.CpuMem("15", "131Gi"), testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), ), testfixtures.ItWithUsedResourcesNodes( 0, - cpuMem("16", "130Gi"), + testfixtures.CpuMem("16", "130Gi"), testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), ), testfixtures.ItWithUsedResourcesNodes( 0, - cpuMem("16", "128Gi"), + testfixtures.CpuMem("16", "128Gi"), testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), ), testfixtures.ItWithUsedResourcesNodes( 0, - cpuMem("16", "129Gi"), + testfixtures.CpuMem("16", "129Gi"), testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), ), testfixtures.ItWithUsedResourcesNodes( 0, - cpu("17"), + testfixtures.Cpu("17"), testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), ), ), ), nodeTypeIds: []uint64{nodeTypeA.GetId()}, priority: 0, - resourceRequests: cpuMem("16", "128Gi"), + resourceRequests: testfixtures.CpuMem("16", "128Gi"), expected: []int{6, 1, 0}, }, "double-nested ordering": { @@ -567,22 +567,22 @@ func TestNodeTypesIterator(t *testing.T) { armadaslices.Concatenate( testfixtures.ItWithUsedResourcesNodes( 0, - cpuMem("31", "1Gi"), + testfixtures.CpuMem("31", "1Gi"), testfixtures.ItN8GpuNodes(1, testfixtures.TestPriorities), ), testfixtures.ItWithUsedResourcesNodes( 0, - cpuMemGpu("31", "1Gi", "1"), + testfixtures.CpuMemGpu("31", "1Gi", "1"), testfixtures.ItN8GpuNodes(1, testfixtures.TestPriorities), ), testfixtures.ItWithUsedResourcesNodes( 0, - cpuMemGpu("31", "1Gi", "2"), + testfixtures.CpuMemGpu("31", "1Gi", "2"), testfixtures.ItN8GpuNodes(1, testfixtures.TestPriorities), ), testfixtures.ItWithUsedResourcesNodes( 0, - cpuMemGpu("31", "1Gi", "5"), + testfixtures.CpuMemGpu("31", "1Gi", "5"), testfixtures.ItN8GpuNodes(1, testfixtures.TestPriorities), ), ), @@ -592,22 +592,22 @@ func TestNodeTypesIterator(t *testing.T) { armadaslices.Concatenate( testfixtures.ItWithUsedResourcesNodes( 0, - cpuMem("31", "2Gi"), + testfixtures.CpuMem("31", "2Gi"), testfixtures.ItN8GpuNodes(1, testfixtures.TestPriorities), ), testfixtures.ItWithUsedResourcesNodes( 0, - cpuMemGpu("31", "2Gi", "1"), + testfixtures.CpuMemGpu("31", "2Gi", "1"), testfixtures.ItN8GpuNodes(1, testfixtures.TestPriorities), ), testfixtures.ItWithUsedResourcesNodes( 0, - cpuMem("32", "514Gi"), + testfixtures.CpuMem("32", "514Gi"), testfixtures.ItN8GpuNodes(1, testfixtures.TestPriorities), ), testfixtures.ItWithUsedResourcesNodes( 0, - cpuMem("32", "512Gi"), + testfixtures.CpuMem("32", "512Gi"), testfixtures.ItN8GpuNodes(1, testfixtures.TestPriorities), ), ), @@ -617,12 +617,12 @@ func TestNodeTypesIterator(t *testing.T) { armadaslices.Concatenate( testfixtures.ItWithUsedResourcesNodes( 0, - cpuMem("32", "513Gi"), + testfixtures.CpuMem("32", "513Gi"), testfixtures.ItN8GpuNodes(1, testfixtures.TestPriorities), ), testfixtures.ItWithUsedResourcesNodes( 0, - cpu("33"), + testfixtures.Cpu("33"), testfixtures.ItN8GpuNodes(1, testfixtures.TestPriorities), ), ), @@ -630,7 +630,7 @@ func TestNodeTypesIterator(t *testing.T) { ), nodeTypeIds: []uint64{nodeTypeA.GetId(), nodeTypeB.GetId(), nodeTypeC.GetId()}, priority: 0, - resourceRequests: cpuMemGpu("32", "512Gi", "4"), + resourceRequests: testfixtures.CpuMemGpu("32", "512Gi", "4"), expected: []int{7, 5, 4, 2, 1, 0}, }, } @@ -759,30 +759,3 @@ func labelsToNodeType(labels map[string]string) *internaltypes.NodeType { ) return nodeType } - -func cpu(cpu string) internaltypes.ResourceList { - return testfixtures.TestResourceListFactory.FromNodeProto( - map[string]resource.Quantity{ - "cpu": resource.MustParse(cpu), - }, - ) -} - -func cpuMem(cpu string, memory string) internaltypes.ResourceList { - return testfixtures.TestResourceListFactory.FromNodeProto( - map[string]resource.Quantity{ - "cpu": resource.MustParse(cpu), - "memory": resource.MustParse(memory), - }, - ) -} - -func cpuMemGpu(cpu string, memory string, gpu string) internaltypes.ResourceList { - return testfixtures.TestResourceListFactory.FromNodeProto( - map[string]resource.Quantity{ - "cpu": resource.MustParse(cpu), - "memory": resource.MustParse(memory), - "nvidia.com/gpu": resource.MustParse(gpu), - }, - ) -} diff --git a/internal/scheduler/scheduling/fairness/fairness_test.go b/internal/scheduler/scheduling/fairness/fairness_test.go index 16cc4b4554e..52a9a86584a 100644 --- a/internal/scheduler/scheduling/fairness/fairness_test.go +++ b/internal/scheduler/scheduling/fairness/fairness_test.go @@ -9,7 +9,6 @@ import ( "github.com/armadaproject/armada/internal/scheduler/configuration" "github.com/armadaproject/armada/internal/scheduler/internaltypes" - "github.com/armadaproject/armada/internal/scheduler/schedulerobjects" ) type MinimalQueue struct { @@ -28,201 +27,100 @@ func (q MinimalQueue) GetWeight() float64 { func TestNewDominantResourceFairness(t *testing.T) { rlFactory := makeTestResourceListFactory() _, err := NewDominantResourceFairness( - rlFactory.FromNodeProto(map[string]resource.Quantity{ - "foo": resource.MustParse("1"), - }, - ), + fooBarBaz(rlFactory, "1", "0", "0"), configuration.SchedulingConfig{DominantResourceFairnessResourcesToConsider: []string{}}, ) require.Error(t, err) } func TestDominantResourceFairness(t *testing.T) { + rlFactory := makeTestResourceListFactory() + tests := map[string]struct { - totalResources schedulerobjects.ResourceList + totalResources internaltypes.ResourceList config configuration.SchedulingConfig - allocation schedulerobjects.ResourceList + allocation internaltypes.ResourceList weight float64 expectedCost float64 }{ "single resource 1": { - totalResources: schedulerobjects.ResourceList{ - Resources: map[string]resource.Quantity{ - "foo": resource.MustParse("1"), - "bar": resource.MustParse("2"), - "baz": resource.MustParse("3"), - }, - }, - config: configuration.SchedulingConfig{DominantResourceFairnessResourcesToConsider: []string{"foo", "bar"}}, - allocation: schedulerobjects.ResourceList{ - Resources: map[string]resource.Quantity{ - "foo": resource.MustParse("0.5"), - }, - }, - weight: 1.0, - expectedCost: 0.5, + totalResources: fooBarBaz(rlFactory, "1", "2", "3"), + config: configuration.SchedulingConfig{DominantResourceFairnessResourcesToConsider: []string{"foo", "bar"}}, + allocation: fooBarBaz(rlFactory, "0.5", "0", "0"), + weight: 1.0, + expectedCost: 0.5, }, "single resource 2": { - totalResources: schedulerobjects.ResourceList{ - Resources: map[string]resource.Quantity{ - "foo": resource.MustParse("1"), - "bar": resource.MustParse("2"), - "baz": resource.MustParse("3"), - }, - }, - config: configuration.SchedulingConfig{DominantResourceFairnessResourcesToConsider: []string{"foo", "bar"}}, - allocation: schedulerobjects.ResourceList{ - Resources: map[string]resource.Quantity{ - "bar": resource.MustParse("0.5"), - }, - }, - weight: 1.0, - expectedCost: 0.25, + totalResources: fooBarBaz(rlFactory, "1", "2", "3"), + config: configuration.SchedulingConfig{DominantResourceFairnessResourcesToConsider: []string{"foo", "bar"}}, + allocation: fooBarBaz(rlFactory, "0", "0.5", "0"), + weight: 1.0, + expectedCost: 0.25, }, "multiple resources": { - totalResources: schedulerobjects.ResourceList{ - Resources: map[string]resource.Quantity{ - "foo": resource.MustParse("1"), - "bar": resource.MustParse("2"), - "baz": resource.MustParse("3"), - }, - }, - config: configuration.SchedulingConfig{DominantResourceFairnessResourcesToConsider: []string{"foo", "bar"}}, - allocation: schedulerobjects.ResourceList{ - Resources: map[string]resource.Quantity{ - "foo": resource.MustParse("0.5"), - "bar": resource.MustParse("1.1"), - }, - }, - weight: 1.0, - expectedCost: 1.1 / 2, + totalResources: fooBarBaz(rlFactory, "1", "2", "3"), + config: configuration.SchedulingConfig{DominantResourceFairnessResourcesToConsider: []string{"foo", "bar"}}, + allocation: fooBarBaz(rlFactory, "0.5", "1.1", "0"), + weight: 1.0, + expectedCost: 1.1 / 2, }, "considered resources": { - totalResources: schedulerobjects.ResourceList{ - Resources: map[string]resource.Quantity{ - "foo": resource.MustParse("1"), - "bar": resource.MustParse("2"), - "baz": resource.MustParse("3"), - }, - }, - config: configuration.SchedulingConfig{DominantResourceFairnessResourcesToConsider: []string{"foo", "bar"}}, - allocation: schedulerobjects.ResourceList{ - Resources: map[string]resource.Quantity{ - "foo": resource.MustParse("0.5"), - "baz": resource.MustParse("3"), - }, - }, - weight: 1.0, - expectedCost: 0.5, + totalResources: fooBarBaz(rlFactory, "1", "2", "3"), + config: configuration.SchedulingConfig{DominantResourceFairnessResourcesToConsider: []string{"foo", "bar"}}, + allocation: fooBarBaz(rlFactory, "0.5", "0", "3"), + weight: 1.0, + expectedCost: 0.5, }, "zero available resource": { - totalResources: schedulerobjects.ResourceList{ - Resources: map[string]resource.Quantity{ - "foo": resource.MustParse("1"), - "bar": resource.MustParse("0"), - "baz": resource.MustParse("3"), - }, - }, - config: configuration.SchedulingConfig{DominantResourceFairnessResourcesToConsider: []string{"foo", "bar"}}, - allocation: schedulerobjects.ResourceList{ - Resources: map[string]resource.Quantity{ - "foo": resource.MustParse("0.5"), - "bar": resource.MustParse("2.0"), - }, - }, - weight: 1.0, - expectedCost: 0.5, + totalResources: fooBarBaz(rlFactory, "1", "0", "3"), + config: configuration.SchedulingConfig{DominantResourceFairnessResourcesToConsider: []string{"foo", "bar"}}, + allocation: fooBarBaz(rlFactory, "0.5", "2.0", "0"), + weight: 1.0, + expectedCost: 0.5, }, "weight": { - totalResources: schedulerobjects.ResourceList{ - Resources: map[string]resource.Quantity{ - "foo": resource.MustParse("1"), - "bar": resource.MustParse("2"), - "baz": resource.MustParse("3"), - }, - }, - config: configuration.SchedulingConfig{DominantResourceFairnessResourcesToConsider: []string{"foo", "bar"}}, - allocation: schedulerobjects.ResourceList{ - Resources: map[string]resource.Quantity{ - "foo": resource.MustParse("0.5"), - }, - }, - weight: 2.0, - expectedCost: 0.25, + totalResources: fooBarBaz(rlFactory, "1", "2", "3"), + config: configuration.SchedulingConfig{DominantResourceFairnessResourcesToConsider: []string{"foo", "bar"}}, + allocation: fooBarBaz(rlFactory, "0.5", "0", "0"), + weight: 2.0, + expectedCost: 0.25, }, "experimental config": { - totalResources: schedulerobjects.ResourceList{ - Resources: map[string]resource.Quantity{ - "foo": resource.MustParse("1"), - "bar": resource.MustParse("2"), - "baz": resource.MustParse("3"), - }, - }, - config: configuration.SchedulingConfig{ExperimentalDominantResourceFairnessResourcesToConsider: []configuration.DominantResourceFairnessResource{{"foo", 1}, {"bar", 1}}}, - allocation: schedulerobjects.ResourceList{ - Resources: map[string]resource.Quantity{ - "foo": resource.MustParse("0.5"), - "bar": resource.MustParse("1.1"), - }, - }, - weight: 1.0, - expectedCost: 1.1 / 2, + totalResources: fooBarBaz(rlFactory, "1", "2", "3"), + config: configuration.SchedulingConfig{ExperimentalDominantResourceFairnessResourcesToConsider: []configuration.DominantResourceFairnessResource{{"foo", 1}, {"bar", 1}}}, + allocation: fooBarBaz(rlFactory, "0.5", "1.1", "0"), + weight: 1.0, + expectedCost: 1.1 / 2, }, "experimental config defaults multipliers to one": { - totalResources: schedulerobjects.ResourceList{ - Resources: map[string]resource.Quantity{ - "foo": resource.MustParse("1"), - "bar": resource.MustParse("2"), - "baz": resource.MustParse("3"), - }, - }, - config: configuration.SchedulingConfig{ExperimentalDominantResourceFairnessResourcesToConsider: []configuration.DominantResourceFairnessResource{{"foo", 0}, {"bar", 0}}}, - allocation: schedulerobjects.ResourceList{ - Resources: map[string]resource.Quantity{ - "foo": resource.MustParse("0.5"), - "bar": resource.MustParse("1.1"), - }, - }, - weight: 1.0, - expectedCost: 1.1 / 2, + totalResources: fooBarBaz(rlFactory, "1", "2", "3"), + config: configuration.SchedulingConfig{ExperimentalDominantResourceFairnessResourcesToConsider: []configuration.DominantResourceFairnessResource{{"foo", 0}, {"bar", 0}}}, + allocation: fooBarBaz(rlFactory, "0.5", "1.1", "0"), + weight: 1.0, + expectedCost: 1.1 / 2, }, "experimental config non-unit multiplier": { - totalResources: schedulerobjects.ResourceList{ - Resources: map[string]resource.Quantity{ - "foo": resource.MustParse("1"), - "bar": resource.MustParse("2"), - "baz": resource.MustParse("3"), - }, - }, - config: configuration.SchedulingConfig{ExperimentalDominantResourceFairnessResourcesToConsider: []configuration.DominantResourceFairnessResource{{"foo", 4}, {"bar", 1}}}, - allocation: schedulerobjects.ResourceList{ - Resources: map[string]resource.Quantity{ - "foo": resource.MustParse("0.5"), - "bar": resource.MustParse("1.1"), - }, - }, - weight: 1.0, - expectedCost: 2, + totalResources: fooBarBaz(rlFactory, "1", "2", "3"), + config: configuration.SchedulingConfig{ExperimentalDominantResourceFairnessResourcesToConsider: []configuration.DominantResourceFairnessResource{{"foo", 4}, {"bar", 1}}}, + allocation: fooBarBaz(rlFactory, "0.5", "1.1", "0"), + weight: 1.0, + expectedCost: 2, }, } - rlFactory := makeTestResourceListFactory() - for name, tc := range tests { t.Run(name, func(t *testing.T) { - totalResources := rlFactory.FromNodeProto(tc.totalResources.Resources) - allocation := rlFactory.FromJobResourceListIgnoreUnknown(tc.allocation.Resources) - f, err := NewDominantResourceFairness(totalResources, tc.config) + f, err := NewDominantResourceFairness(tc.totalResources, tc.config) require.NoError(t, err) assert.Equal( t, tc.expectedCost, - f.WeightedCostFromAllocation(allocation, tc.weight), + f.WeightedCostFromAllocation(tc.allocation, tc.weight), ) assert.Equal( t, - f.WeightedCostFromAllocation(allocation, tc.weight), - f.WeightedCostFromQueue(MinimalQueue{allocation: allocation, weight: tc.weight}), + f.WeightedCostFromAllocation(tc.allocation, tc.weight), + f.WeightedCostFromQueue(MinimalQueue{allocation: tc.allocation, weight: tc.weight}), ) }) } @@ -243,3 +141,11 @@ func makeTestResourceListFactory() *internaltypes.ResourceListFactory { } return rlFactory } + +func fooBarBaz(factory *internaltypes.ResourceListFactory, foo, bar, baz string) internaltypes.ResourceList { + return factory.FromNodeProto(map[string]resource.Quantity{ + "foo": resource.MustParse(foo), + "bar": resource.MustParse(bar), + "baz": resource.MustParse(baz), + }) +} diff --git a/internal/scheduler/scheduling/gang_scheduler_test.go b/internal/scheduler/scheduling/gang_scheduler_test.go index d1626839282..f21338d20ca 100644 --- a/internal/scheduler/scheduling/gang_scheduler_test.go +++ b/internal/scheduler/scheduling/gang_scheduler_test.go @@ -33,10 +33,10 @@ func TestGangScheduler(t *testing.T) { tests := map[string]struct { SchedulingConfig configuration.SchedulingConfig // Nodes to be considered by the scheduler. - Nodes []*schedulerobjects.Node + Nodes []*internaltypes.Node // Total resources across all clusters. // Set to the total resources across all nodes if not provided. - TotalResources schedulerobjects.ResourceList + TotalResources internaltypes.ResourceList // Gangs to try scheduling. Gangs [][]*jobdb.Job // Indices of gangs expected to be scheduled. @@ -54,7 +54,7 @@ func TestGangScheduler(t *testing.T) { }{ "simple success": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), Gangs: [][]*jobdb.Job{ testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 32)), }, @@ -65,7 +65,7 @@ func TestGangScheduler(t *testing.T) { }, "simple failure": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), Gangs: [][]*jobdb.Job{ testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 33)), }, @@ -76,7 +76,7 @@ func TestGangScheduler(t *testing.T) { }, "one success and one failure": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), Gangs: [][]*jobdb.Job{ testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 32)), testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1)), @@ -88,7 +88,7 @@ func TestGangScheduler(t *testing.T) { }, "multiple nodes": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.N32CpuNodes(2, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(2, testfixtures.TestPriorities), Gangs: [][]*jobdb.Job{ testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 64)), }, @@ -103,7 +103,7 @@ func TestGangScheduler(t *testing.T) { cfg.ExperimentalFloatingResources = testfixtures.TestFloatingResourceConfig return cfg }(), - Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), Gangs: [][]*jobdb.Job{ // we have 10 of test-floating-resource so only the first of these two jobs should fit addFloatingResourceRequest("6", testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1))), @@ -119,7 +119,7 @@ func TestGangScheduler(t *testing.T) { map[string]float64{"cpu": 0.5}, testfixtures.TestSchedulingConfig(), ), - Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), Gangs: [][]*jobdb.Job{ testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 8)), testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 16)), @@ -138,7 +138,7 @@ func TestGangScheduler(t *testing.T) { testfixtures.TestSchedulingConfig(), ), ), - Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), Gangs: [][]*jobdb.Job{ testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1)), testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1)), @@ -159,7 +159,7 @@ func TestGangScheduler(t *testing.T) { testfixtures.TestSchedulingConfig(), ), ), - Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), Gangs: [][]*jobdb.Job{ testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1)), testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1)), @@ -182,7 +182,7 @@ func TestGangScheduler(t *testing.T) { }, testfixtures.TestSchedulingConfig(), ), - Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), Gangs: [][]*jobdb.Job{ testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1)), testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 2)), @@ -206,7 +206,7 @@ func TestGangScheduler(t *testing.T) { }, testfixtures.TestSchedulingConfig(), ), - Nodes: testfixtures.N32CpuNodes(3, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(3, testfixtures.TestPriorities), Gangs: [][]*jobdb.Job{ testfixtures.WithGangAnnotationsJobs(testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 1)), testfixtures.WithGangAnnotationsJobs(testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 1)), @@ -228,7 +228,7 @@ func TestGangScheduler(t *testing.T) { }, testfixtures.TestSchedulingConfig(), ), - Nodes: testfixtures.N32CpuNodes(3, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(3, testfixtures.TestPriorities), Gangs: [][]*jobdb.Job{ testfixtures.WithGangAnnotationsJobs(testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 1)), testfixtures.WithGangAnnotationsJobs(testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 1)), @@ -250,25 +250,15 @@ func TestGangScheduler(t *testing.T) { testfixtures.TestSchedulingConfig(), ), Nodes: armadaslices.Concatenate( - testfixtures.WithUsedResourcesNodes( + testfixtures.ItWithUsedResourcesNodes( 0, - schedulerobjects.ResourceList{ - Resources: map[string]resource.Quantity{ - "cpu": resource.MustParse("31.5"), - "memory": resource.MustParse("512Gi"), - "nvidia.com/gpu": resource.MustParse("8"), - }, - }, - testfixtures.N8GpuNodes(1, testfixtures.TestPriorities), + testfixtures.CpuMemGpu("31.5", "512Gi", "8"), + testfixtures.ItN8GpuNodes(1, testfixtures.TestPriorities), ), - testfixtures.WithUsedResourcesNodes( + testfixtures.ItWithUsedResourcesNodes( 0, - schedulerobjects.ResourceList{ - Resources: map[string]resource.Quantity{ - "cpu": resource.MustParse("32"), - }, - }, - testfixtures.N8GpuNodes(1, testfixtures.TestPriorities), + testfixtures.Cpu("32"), + testfixtures.ItN8GpuNodes(1, testfixtures.TestPriorities), ), ), Gangs: [][]*jobdb.Job{ @@ -281,9 +271,9 @@ func TestGangScheduler(t *testing.T) { }, "NodeUniformityLabel set but not indexed": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.WithLabelsNodes( + Nodes: testfixtures.TestNodeFactory.AddLabels( + testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), map[string]string{"foo": "foov"}, - testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), ), Gangs: [][]*jobdb.Job{ testfixtures.WithGangAnnotationsJobs( @@ -302,7 +292,7 @@ func TestGangScheduler(t *testing.T) { []string{"foo", "bar"}, testfixtures.TestSchedulingConfig(), ), - Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), Gangs: [][]*jobdb.Job{ testfixtures.WithGangAnnotationsJobs( testfixtures.WithNodeUniformityLabelAnnotationJobs( @@ -321,13 +311,13 @@ func TestGangScheduler(t *testing.T) { testfixtures.TestSchedulingConfig(), ), Nodes: armadaslices.Concatenate( - testfixtures.WithLabelsNodes( + testfixtures.TestNodeFactory.AddLabels( + testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), map[string]string{"foo": "foov1"}, - testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), ), - testfixtures.WithLabelsNodes( + testfixtures.TestNodeFactory.AddLabels( + testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), map[string]string{"foo": "foov2"}, - testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), ), ), Gangs: [][]*jobdb.Job{ @@ -346,25 +336,25 @@ func TestGangScheduler(t *testing.T) { testfixtures.TestSchedulingConfig(), ), Nodes: armadaslices.Concatenate( - testfixtures.WithLabelsNodes( - map[string]string{"foo": "foov1"}, - testfixtures.WithUsedResourcesNodes( + testfixtures.TestNodeFactory.AddLabels( + testfixtures.ItWithUsedResourcesNodes( 0, - schedulerobjects.ResourceList{Resources: map[string]resource.Quantity{"cpu": resource.MustParse("1")}}, - testfixtures.N32CpuNodes(2, testfixtures.TestPriorities), + testfixtures.Cpu("1"), + testfixtures.ItN32CpuNodes(2, testfixtures.TestPriorities), ), + map[string]string{"foo": "foov1"}, ), - testfixtures.WithLabelsNodes( + testfixtures.TestNodeFactory.AddLabels( + testfixtures.ItN32CpuNodes(2, testfixtures.TestPriorities), map[string]string{"foo": "foov2"}, - testfixtures.N32CpuNodes(2, testfixtures.TestPriorities), ), - testfixtures.WithLabelsNodes( - map[string]string{"foo": "foov3"}, - testfixtures.WithUsedResourcesNodes( + testfixtures.TestNodeFactory.AddLabels( + testfixtures.ItWithUsedResourcesNodes( 0, - schedulerobjects.ResourceList{Resources: map[string]resource.Quantity{"cpu": resource.MustParse("1")}}, - testfixtures.N32CpuNodes(2, testfixtures.TestPriorities), + testfixtures.Cpu("1"), + testfixtures.ItN32CpuNodes(2, testfixtures.TestPriorities), ), + map[string]string{"foo": "foov3"}, ), ), Gangs: [][]*jobdb.Job{ @@ -385,13 +375,13 @@ func TestGangScheduler(t *testing.T) { testfixtures.TestSchedulingConfig(), ), Nodes: append( - testfixtures.WithLabelsNodes( + testfixtures.TestNodeFactory.AddLabels( + testfixtures.ItN32CpuNodes(2, testfixtures.TestPriorities), map[string]string{"my-cool-node-uniformity": "a"}, - testfixtures.N32CpuNodes(2, testfixtures.TestPriorities), ), - testfixtures.WithLabelsNodes( + testfixtures.TestNodeFactory.AddLabels( + testfixtures.ItN32CpuNodes(3, testfixtures.TestPriorities), map[string]string{"my-cool-node-uniformity": "b"}, - testfixtures.N32CpuNodes(3, testfixtures.TestPriorities), )..., ), Gangs: [][]*jobdb.Job{ @@ -454,16 +444,12 @@ func TestGangScheduler(t *testing.T) { } return config }(), - Nodes: func() []*schedulerobjects.Node { - nodes := testfixtures.N8GpuNodes(1, []int32{29000, 30000}) - for _, node := range nodes { - node.Taints = []v1.Taint{ - {Key: "taint-a", Value: "true", Effect: v1.TaintEffectNoSchedule}, - {Key: "taint-b", Value: "true", Effect: v1.TaintEffectNoSchedule}, - } - } - return nodes - }(), + Nodes: testfixtures.TestNodeFactory.AddTaints( + testfixtures.ItN8GpuNodes(1, []int32{29000, 30000}), + []v1.Taint{ + {Key: "taint-a", Value: "true", Effect: v1.TaintEffectNoSchedule}, + {Key: "taint-b", Value: "true", Effect: v1.TaintEffectNoSchedule}, + }), Gangs: func() (gangs [][]*jobdb.Job) { var jobId ulid.ULID jobId = util.ULID() @@ -507,15 +493,12 @@ func TestGangScheduler(t *testing.T) { } return config }(), - Nodes: func() []*schedulerobjects.Node { - nodes := testfixtures.N32CpuNodes(1, []int32{29000, 30000}) - for _, node := range nodes { - node.Taints = []v1.Taint{ - {Key: "taint-a", Value: "true", Effect: v1.TaintEffectNoSchedule}, - } - } - return nodes - }(), + Nodes: testfixtures.TestNodeFactory.AddTaints( + testfixtures.ItN32CpuNodes(1, []int32{29000, 30000}), + []v1.Taint{ + {Key: "taint-a", Value: "true", Effect: v1.TaintEffectNoSchedule}, + }, + ), Gangs: func() (gangs [][]*jobdb.Job) { jobId := util.ULID() gangs = append(gangs, []*jobdb.Job{testfixtures.TestJob("A", jobId, "armada-preemptible-away", testfixtures.Test32Cpu256GiPodReqs("A", jobId, 30000))}) @@ -535,7 +518,7 @@ func TestGangScheduler(t *testing.T) { cfg.MaximumPerQueueSchedulingRate = 2 return cfg }(), - Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), Gangs: [][]*jobdb.Job{ testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 2)), testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1)), @@ -548,7 +531,7 @@ func TestGangScheduler(t *testing.T) { }, "Hitting globally applicable constraint makes job scheduling key unfeasible": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), Gangs: [][]*jobdb.Job{ testfixtures.WithGangAnnotationsJobs(testfixtures.N1GpuJobs("A", testfixtures.PriorityClass0, 1)), testfixtures.WithGangAnnotationsJobs(testfixtures.N1GpuJobs("B", testfixtures.PriorityClass0, 1)), @@ -584,9 +567,9 @@ func TestGangScheduler(t *testing.T) { expectedUnfeasibleJobSchedulingKeys = append(expectedUnfeasibleJobSchedulingKeys, key) } - nodesById := make(map[string]*schedulerobjects.Node, len(tc.Nodes)) + nodesById := make(map[string]*internaltypes.Node, len(tc.Nodes)) for _, node := range tc.Nodes { - nodesById[node.Id] = node + nodesById[node.GetId()] = node } nodeDb, err := nodedb.NewNodeDb( tc.SchedulingConfig.PriorityClasses, @@ -599,17 +582,13 @@ func TestGangScheduler(t *testing.T) { require.NoError(t, err) txn := nodeDb.Txn(true) for _, node := range tc.Nodes { - dbNode, err := testfixtures.TestNodeFactory.FromSchedulerObjectsNode(node) - require.NoError(t, err) - err = nodeDb.CreateAndInsertWithJobDbJobsWithTxn(txn, nil, dbNode) + err = nodeDb.CreateAndInsertWithJobDbJobsWithTxn(txn, nil, node.DeepCopyNilKeys()) require.NoError(t, err) } txn.Commit() - if tc.TotalResources.Resources == nil { + if tc.TotalResources.AllZero() { // Default to NodeDb total. - tc.TotalResources = schedulerobjects.ResourceList{ - Resources: nodeDb.TotalKubernetesResources().ToMap(), - } + tc.TotalResources = nodeDb.TotalKubernetesResources() } priorityFactorByQueue := make(map[string]float64) for _, jobs := range tc.Gangs { @@ -618,10 +597,8 @@ func TestGangScheduler(t *testing.T) { } } - totalResources := testfixtures.TestResourceListFactory.FromNodeProto(tc.TotalResources.Resources) - fairnessCostProvider, err := fairness.NewDominantResourceFairness( - totalResources, + tc.TotalResources, tc.SchedulingConfig, ) require.NoError(t, err) @@ -632,7 +609,7 @@ func TestGangScheduler(t *testing.T) { rate.Limit(tc.SchedulingConfig.MaximumSchedulingRate), tc.SchedulingConfig.MaximumSchedulingBurst, ), - totalResources, + tc.TotalResources, ) for queue, priorityFactor := range priorityFactorByQueue { err := sctx.AddQueueSchedulingContext( @@ -650,7 +627,7 @@ func TestGangScheduler(t *testing.T) { } constraints := schedulerconstraints.NewSchedulingConstraints( "pool", - totalResources, + tc.TotalResources, tc.SchedulingConfig, armadaslices.Map( maps.Keys(priorityFactorByQueue), @@ -687,7 +664,7 @@ func TestGangScheduler(t *testing.T) { } node := nodesById[pctx.NodeId] require.NotNil(t, node) - value, ok := node.Labels[nodeUniformity] + value, ok := node.GetLabelValue(nodeUniformity) require.True(t, ok, "gang job scheduled onto node with missing nodeUniformityLabel") nodeUniformityLabelValues[value] = true } diff --git a/internal/scheduler/scheduling/market_driven_preempting_queue_scheduler_test.go b/internal/scheduler/scheduling/market_driven_preempting_queue_scheduler_test.go index 2a63f5ec6ea..4c5e8e1257a 100644 --- a/internal/scheduler/scheduling/market_driven_preempting_queue_scheduler_test.go +++ b/internal/scheduler/scheduling/market_driven_preempting_queue_scheduler_test.go @@ -19,7 +19,6 @@ import ( "github.com/armadaproject/armada/internal/scheduler/internaltypes" "github.com/armadaproject/armada/internal/scheduler/jobdb" "github.com/armadaproject/armada/internal/scheduler/nodedb" - "github.com/armadaproject/armada/internal/scheduler/schedulerobjects" schedulerconstraints "github.com/armadaproject/armada/internal/scheduler/scheduling/constraints" "github.com/armadaproject/armada/internal/scheduler/scheduling/context" "github.com/armadaproject/armada/internal/scheduler/scheduling/fairness" @@ -45,7 +44,7 @@ func TestMarketDrivenPreemptingQueueScheduler(t *testing.T) { tests := map[string]struct { SchedulingConfig configuration.SchedulingConfig // Nodes to be considered by the scheduler. - Nodes []*schedulerobjects.Node + Nodes []*internaltypes.Node // Each item corresponds to a call to Reschedule(). Rounds []SchedulingRound // Map from queue to the priority factor associated with that queue. @@ -55,7 +54,7 @@ func TestMarketDrivenPreemptingQueueScheduler(t *testing.T) { }{ "three users, highest price jobs from single queue get on": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -79,7 +78,7 @@ func TestMarketDrivenPreemptingQueueScheduler(t *testing.T) { }, "three users, highest price jobs between queues get on": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -115,7 +114,7 @@ func TestMarketDrivenPreemptingQueueScheduler(t *testing.T) { }, "Two users, no preemption if price lower": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -135,7 +134,7 @@ func TestMarketDrivenPreemptingQueueScheduler(t *testing.T) { }, "Two users, preemption if price higher": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -163,7 +162,7 @@ func TestMarketDrivenPreemptingQueueScheduler(t *testing.T) { }, "Two users, partial preemption if price higher": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -194,7 +193,7 @@ func TestMarketDrivenPreemptingQueueScheduler(t *testing.T) { }, "Self Preemption If Price Is Higher": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -225,7 +224,7 @@ func TestMarketDrivenPreemptingQueueScheduler(t *testing.T) { }, "Two Users. Self preemption plus cross user preemption": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -255,7 +254,7 @@ func TestMarketDrivenPreemptingQueueScheduler(t *testing.T) { }, "gang preemption": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -355,11 +354,11 @@ func TestMarketDrivenPreemptingQueueScheduler(t *testing.T) { ctx.FieldLogger = ctx.WithField("round", i) ctx.Infof("starting scheduling round %d", i) - jobsByNode := map[string][]*jobdb.Job{} + jobsByNodeId := map[string][]*jobdb.Job{} for _, job := range jobDbTxn.GetAll() { if job.LatestRun() != nil && !job.LatestRun().InTerminalState() { - node := job.LatestRun().NodeId() - jobsByNode[node] = append(jobsByNode[node], job) + nodeId := job.LatestRun().NodeId() + jobsByNodeId[nodeId] = append(jobsByNodeId[nodeId], job) } } @@ -367,9 +366,7 @@ func TestMarketDrivenPreemptingQueueScheduler(t *testing.T) { require.NoError(t, err) nodeDbTxn := nodeDb.Txn(true) for _, node := range tc.Nodes { - dbNode, err := testfixtures.TestNodeFactory.FromSchedulerObjectsNode(node) - require.NoError(t, err) - err = nodeDb.CreateAndInsertWithJobDbJobsWithTxn(nodeDbTxn, jobsByNode[node.Name], dbNode) + err = nodeDb.CreateAndInsertWithJobDbJobsWithTxn(nodeDbTxn, jobsByNodeId[node.GetId()], node.DeepCopyNilKeys()) require.NoError(t, err) } nodeDbTxn.Commit() @@ -416,7 +413,7 @@ func TestMarketDrivenPreemptingQueueScheduler(t *testing.T) { } for idx, isCordoned := range cordonedNodes { if isCordoned { - node, err := nodeDb.GetNode(tc.Nodes[idx].Id) + node, err := nodeDb.GetNode(tc.Nodes[idx].GetId()) require.NoError(t, err) ctx.Infof("Cordoned node %s", node.GetId()) taints := append(slices.Clone(node.GetTaints()), internaltypes.UnschedulableTaint()) diff --git a/internal/scheduler/scheduling/preempting_queue_scheduler_test.go b/internal/scheduler/scheduling/preempting_queue_scheduler_test.go index 76e6dc492fb..775a5b66ebf 100644 --- a/internal/scheduler/scheduling/preempting_queue_scheduler_test.go +++ b/internal/scheduler/scheduling/preempting_queue_scheduler_test.go @@ -24,7 +24,6 @@ import ( "github.com/armadaproject/armada/internal/scheduler/internaltypes" "github.com/armadaproject/armada/internal/scheduler/jobdb" "github.com/armadaproject/armada/internal/scheduler/nodedb" - "github.com/armadaproject/armada/internal/scheduler/schedulerobjects" schedulerconstraints "github.com/armadaproject/armada/internal/scheduler/scheduling/constraints" "github.com/armadaproject/armada/internal/scheduler/scheduling/context" "github.com/armadaproject/armada/internal/scheduler/scheduling/fairness" @@ -52,13 +51,14 @@ func TestEvictOversubscribed(t *testing.T) { stringInterner := stringinterner.New(1024) - node := testfixtures.Test32CpuNode(priorities) nodeDb, err := NewNodeDb(config, stringInterner) require.NoError(t, err) - dbNode, err := testfixtures.TestNodeFactory.FromSchedulerObjectsNode(node) - require.NoError(t, err) nodeDbTxn := nodeDb.Txn(true) - err = nodeDb.CreateAndInsertWithJobDbJobsWithTxn(nodeDbTxn, jobs, dbNode) + err = nodeDb.CreateAndInsertWithJobDbJobsWithTxn( + nodeDbTxn, + jobs, + testfixtures.ItTest32CpuNode(priorities), + ) require.NoError(t, err) jobDb := jobdb.NewJobDb(config.PriorityClasses, config.DefaultPriorityClassName, stringInterner, testfixtures.TestResourceListFactory) @@ -100,7 +100,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { tests := map[string]struct { SchedulingConfig configuration.SchedulingConfig // Nodes to be considered by the scheduler. - Nodes []*schedulerobjects.Node + Nodes []*internaltypes.Node // Each item corresponds to a call to Reschedule(). Rounds []SchedulingRound // Map from queue to the priority factor associated with that queue. @@ -110,7 +110,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { }{ "balancing three queues": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -166,7 +166,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { }, "balancing two queues weighted": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -204,7 +204,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { }, "balancing two queues weighted with inactive queues": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -244,7 +244,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { }, "don't prempt jobs where we don't know the queue": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -264,7 +264,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { }, "avoid preemption when not improving fairness": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -287,7 +287,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { }, "avoid preemption when not improving fairness reverse queue naming": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -310,7 +310,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { }, "preemption when improving fairness": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.N32CpuNodes(2, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(2, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -341,7 +341,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { }, "reschedule onto same node": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.N32CpuNodes(2, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(2, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -368,7 +368,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { }, "reschedule onto same node reverse order": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.N32CpuNodes(2, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(2, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -395,7 +395,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { }, "urgency-based preemption stability": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -432,7 +432,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { }, "avoid urgency-based preemption when possible": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.N32CpuNodes(2, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(2, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -458,7 +458,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { }, "preempt in order of priority": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.N32CpuNodes(2, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(2, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -497,7 +497,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { }, "avoid urgency-based preemption when possible cross-queue": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.N32CpuNodes(3, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(3, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -549,7 +549,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { }, "gang preemption": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.N32CpuNodes(2, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(2, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { // Fill half of node 1 and half of node 2. @@ -596,7 +596,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { "gang preemption avoid cascading preemption": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.N32CpuNodes(3, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(3, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { // Schedule a gang spanning nodes 1 and 2. @@ -642,7 +642,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { }, "rescheduled jobs don't count towards global scheduling rate limit": { SchedulingConfig: testfixtures.WithGlobalSchedulingRateLimiterConfig(2, 5, testfixtures.TestSchedulingConfig()), - Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -667,7 +667,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { }, "MaximumSchedulingRate": { SchedulingConfig: testfixtures.WithGlobalSchedulingRateLimiterConfig(2, 4, testfixtures.TestSchedulingConfig()), - Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -703,7 +703,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { }, "rescheduled jobs don't count towards maxQueueLookback": { SchedulingConfig: testfixtures.WithMaxLookbackPerQueueConfig(5, testfixtures.TestSchedulingConfig()), - Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -733,7 +733,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { }, testfixtures.TestSchedulingConfig(), ), - Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -758,7 +758,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { }, "priority class preemption two classes": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -788,7 +788,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { }, "priority class preemption cross-queue": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -819,7 +819,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { }, "priority class preemption not scheduled": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -840,7 +840,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { // - 17 jobs should be preempted to make space; all the priority zero jobs and a single priority 1 job "priority class preemption through multiple levels": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -884,7 +884,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { // time="2024-09-27T09:09:05+01:00" level=info msg="Scheduling new jobs; affected queues [B]; resources map[B:{cpu: 16, memory: 64Gi}]; jobs per queue map[B:16]" round=1 //"broken priority class preemption with non-preemptible priority classes": { // SchedulingConfig: testfixtures.TestSchedulingConfig(), - // Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + // Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), // Rounds: []SchedulingRound{ // { // JobsByQueue: map[string][]*jobdb.Job{ @@ -906,7 +906,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { //}, "priority class preemption four classes": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -948,7 +948,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { }, testfixtures.TestSchedulingConfig(), ), - Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -984,7 +984,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { }, testfixtures.TestSchedulingConfig(), ), - Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -1052,7 +1052,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { map[string]float64{"cpu": 15.0 / 64.0}, testfixtures.TestSchedulingConfig(), ), - Nodes: testfixtures.N32CpuNodes(2, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(2, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -1099,7 +1099,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { }, "Queued jobs are not preempted cross queue": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -1119,7 +1119,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { }, "Queued jobs are not preempted cross queue with some scheduled": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -1140,7 +1140,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { }, "Queued jobs are not preempted cross queue with non-preemptible jobs": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -1160,7 +1160,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { }, "Queued jobs are not preempted cross queue multiple rounds": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -1188,7 +1188,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { }, "Oversubscribed eviction does not evict non-preemptible": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.N32CpuNodes(2, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(2, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -1226,7 +1226,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { }, "Cordoning prevents scheduling new jobs but not re-scheduling running jobs": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -1259,7 +1259,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { 1.0, testfixtures.TestSchedulingConfig(), ), - Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -1295,7 +1295,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { 0.5, testfixtures.TestSchedulingConfig(), ), - Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -1331,7 +1331,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { 0.5, testfixtures.TestSchedulingConfig(), ), - Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -1375,7 +1375,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { 1.0, testfixtures.TestSchedulingConfig(), ), - Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -1411,7 +1411,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { 1.0, testfixtures.TestSchedulingConfig(), ), - Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -1439,7 +1439,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { }, "DominantResourceFairness": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -1461,19 +1461,19 @@ func TestPreemptingQueueScheduler(t *testing.T) { "nodeAffinity node notIn": { SchedulingConfig: testfixtures.TestSchedulingConfig(), Nodes: armadaslices.Concatenate( - testfixtures.WithLabelsNodes( + testfixtures.TestNodeFactory.AddLabels( + testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), map[string]string{"key": "val1"}, - testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), ), - testfixtures.WithLabelsNodes( + testfixtures.TestNodeFactory.AddLabels( + testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), map[string]string{"key": "val2"}, - testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), ), - testfixtures.WithLabelsNodes( + testfixtures.TestNodeFactory.AddLabels( + testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), map[string]string{"key": "val3"}, - testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), ), - testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), ), Rounds: []SchedulingRound{ { @@ -1507,7 +1507,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { // Schedule job from queue C. // This does not prevent re-scheduling. SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.N32CpuNodes(2, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(2, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -1544,7 +1544,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { // Try to schedule a job from queue B. This fails as the node is cordoned. // This should not prevent re-scheduling jobs in queue A. SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -1575,13 +1575,10 @@ func TestPreemptingQueueScheduler(t *testing.T) { } return config }(), - Nodes: func() []*schedulerobjects.Node { - nodes := testfixtures.N8GpuNodes(2, []int32{29000, 30000}) - for _, node := range nodes { - node.Taints = []v1.Taint{{Key: "gpu", Value: "true", Effect: v1.TaintEffectNoSchedule}} - } - return nodes - }(), + Nodes: testfixtures.TestNodeFactory.AddTaints( + testfixtures.ItN8GpuNodes(2, []int32{29000, 30000}), + []v1.Taint{{Key: "gpu", Value: "true", Effect: v1.TaintEffectNoSchedule}}, + ), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -1635,13 +1632,10 @@ func TestPreemptingQueueScheduler(t *testing.T) { config.DefaultPriorityClassName = "armada-preemptible" return config }(), - Nodes: func() []*schedulerobjects.Node { - nodes := testfixtures.N8GpuNodes(2, []int32{29000, 30000}) - for _, node := range nodes { - node.Taints = []v1.Taint{{Key: "gpu", Value: "true", Effect: v1.TaintEffectNoSchedule}} - } - return nodes - }(), + Nodes: testfixtures.TestNodeFactory.AddTaints( + testfixtures.ItN8GpuNodes(2, []int32{29000, 30000}), + []v1.Taint{{Key: "gpu", Value: "true", Effect: v1.TaintEffectNoSchedule}}, + ), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -1701,13 +1695,13 @@ func TestPreemptingQueueScheduler(t *testing.T) { } return config }(), - Nodes: func() []*schedulerobjects.Node { + Nodes: func() []*internaltypes.Node { priorities := []int32{29000, 30000} - gpuNodes := testfixtures.N8GpuNodes(1, priorities) - for _, node := range gpuNodes { - node.Taints = []v1.Taint{{Key: "gpu", Value: "true", Effect: v1.TaintEffectNoSchedule}} - } - return append(testfixtures.N32CpuNodes(1, priorities), gpuNodes...) + gpuNodes := testfixtures.TestNodeFactory.AddTaints( + testfixtures.ItN8GpuNodes(1, priorities), + []v1.Taint{{Key: "gpu", Value: "true", Effect: v1.TaintEffectNoSchedule}}, + ) + return append(testfixtures.ItN32CpuNodes(1, priorities), gpuNodes...) }(), Rounds: []SchedulingRound{ { @@ -1762,13 +1756,13 @@ func TestPreemptingQueueScheduler(t *testing.T) { config.DefaultPriorityClassName = "armada-preemptible" return config }(), - Nodes: func() []*schedulerobjects.Node { + Nodes: func() []*internaltypes.Node { priorities := []int32{29000, 30000} - gpuNodes := testfixtures.N8GpuNodes(1, priorities) - for _, node := range gpuNodes { - node.Taints = []v1.Taint{{Key: "gpu", Value: "true", Effect: v1.TaintEffectNoSchedule}} - } - return append(testfixtures.N32CpuNodes(1, priorities), gpuNodes...) + gpuNodes := testfixtures.TestNodeFactory.AddTaints( + testfixtures.ItN8GpuNodes(1, priorities), + []v1.Taint{{Key: "gpu", Value: "true", Effect: v1.TaintEffectNoSchedule}}, + ) + return append(testfixtures.ItN32CpuNodes(1, priorities), gpuNodes...) }(), Rounds: []SchedulingRound{ { @@ -1836,12 +1830,10 @@ func TestPreemptingQueueScheduler(t *testing.T) { } return config }(), - Nodes: func() []*schedulerobjects.Node { - priorities := []int32{29000, 28000, 30000} - nodes := testfixtures.N32CpuNodes(1, priorities) - nodes[0].Taints = []v1.Taint{{Key: "gpu", Value: "true", Effect: v1.TaintEffectNoSchedule}} - return nodes - }(), + Nodes: testfixtures.TestNodeFactory.AddTaints( + testfixtures.ItN32CpuNodes(1, []int32{29000, 28000, 30000}), + []v1.Taint{{Key: "gpu", Value: "true", Effect: v1.TaintEffectNoSchedule}}, + ), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -1950,11 +1942,11 @@ func TestPreemptingQueueScheduler(t *testing.T) { ctx.FieldLogger = ctx.WithField("round", i) ctx.Infof("starting scheduling round %d", i) - jobsByNode := map[string][]*jobdb.Job{} + jobsByNodeId := map[string][]*jobdb.Job{} for _, job := range jobDbTxn.GetAll() { if job.LatestRun() != nil && !job.LatestRun().InTerminalState() { node := job.LatestRun().NodeId() - jobsByNode[node] = append(jobsByNode[node], job) + jobsByNodeId[node] = append(jobsByNodeId[node], job) } } @@ -1962,9 +1954,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { require.NoError(t, err) nodeDbTxn := nodeDb.Txn(true) for _, node := range tc.Nodes { - dbNode, err := testfixtures.TestNodeFactory.FromSchedulerObjectsNode(node) - require.NoError(t, err) - err = nodeDb.CreateAndInsertWithJobDbJobsWithTxn(nodeDbTxn, jobsByNode[node.Name], dbNode) + err = nodeDb.CreateAndInsertWithJobDbJobsWithTxn(nodeDbTxn, jobsByNodeId[node.GetId()], node.DeepCopyNilKeys()) require.NoError(t, err) } nodeDbTxn.Commit() @@ -2011,7 +2001,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { } for idx, isCordoned := range cordonedNodes { if isCordoned { - node, err := nodeDb.GetNode(tc.Nodes[idx].Id) + node, err := nodeDb.GetNode(tc.Nodes[idx].GetId()) require.NoError(t, err) ctx.Infof("Cordoned node %s", node.GetId()) taints := append(slices.Clone(node.GetTaints()), internaltypes.UnschedulableTaint()) @@ -2260,7 +2250,7 @@ func jobIdsByQueueFromJobContexts(jctxs []*context.JobSchedulingContext) map[str func BenchmarkPreemptingQueueScheduler(b *testing.B) { tests := map[string]struct { SchedulingConfig configuration.SchedulingConfig - Nodes []*schedulerobjects.Node + Nodes []*internaltypes.Node JobFunc func(queue string, priorityClassName string, n int) []*jobdb.Job NumQueues int NumJobsPerQueue int @@ -2269,7 +2259,7 @@ func BenchmarkPreemptingQueueScheduler(b *testing.B) { }{ "1 node 1 queue 320 jobs": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), JobFunc: testfixtures.N1Cpu4GiJobs, NumQueues: 1, NumJobsPerQueue: 320, @@ -2278,7 +2268,7 @@ func BenchmarkPreemptingQueueScheduler(b *testing.B) { }, "1 node 10 queues 320 jobs": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), JobFunc: testfixtures.N1Cpu4GiJobs, NumQueues: 10, NumJobsPerQueue: 320, @@ -2287,7 +2277,7 @@ func BenchmarkPreemptingQueueScheduler(b *testing.B) { }, "10 nodes 1 queue 3200 jobs": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.N32CpuNodes(10, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(10, testfixtures.TestPriorities), JobFunc: testfixtures.N1Cpu4GiJobs, NumQueues: 1, NumJobsPerQueue: 3200, @@ -2296,7 +2286,7 @@ func BenchmarkPreemptingQueueScheduler(b *testing.B) { }, "10 nodes 10 queues 3200 jobs": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.N32CpuNodes(10, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(10, testfixtures.TestPriorities), JobFunc: testfixtures.N1Cpu4GiJobs, NumQueues: 10, NumJobsPerQueue: 3200, @@ -2305,7 +2295,7 @@ func BenchmarkPreemptingQueueScheduler(b *testing.B) { }, "100 nodes 1 queue 32000 jobs": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.N32CpuNodes(100, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(100, testfixtures.TestPriorities), JobFunc: testfixtures.N1Cpu4GiJobs, NumQueues: 1, NumJobsPerQueue: 32000, @@ -2314,7 +2304,7 @@ func BenchmarkPreemptingQueueScheduler(b *testing.B) { }, "100 nodes 10 queues 32000 jobs": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.N32CpuNodes(100, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(100, testfixtures.TestPriorities), JobFunc: testfixtures.N1Cpu4GiJobs, NumQueues: 10, NumJobsPerQueue: 32000, @@ -2323,7 +2313,7 @@ func BenchmarkPreemptingQueueScheduler(b *testing.B) { }, "1000 nodes 1 queue 320000 jobs": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.N32CpuNodes(1000, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1000, testfixtures.TestPriorities), JobFunc: testfixtures.N1Cpu4GiJobs, NumQueues: 1, NumJobsPerQueue: 320000, @@ -2332,7 +2322,7 @@ func BenchmarkPreemptingQueueScheduler(b *testing.B) { }, "1000 nodes 10 queues 320000 jobs": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.N32CpuNodes(1000, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1000, testfixtures.TestPriorities), JobFunc: testfixtures.N1Cpu4GiJobs, NumQueues: 1, NumJobsPerQueue: 32000, @@ -2357,9 +2347,7 @@ func BenchmarkPreemptingQueueScheduler(b *testing.B) { require.NoError(b, err) txn := nodeDb.Txn(true) for _, node := range tc.Nodes { - dbNode, err := testfixtures.TestNodeFactory.FromSchedulerObjectsNode(node) - require.NoError(b, err) - err = nodeDb.CreateAndInsertWithJobDbJobsWithTxn(txn, nil, dbNode) + err = nodeDb.CreateAndInsertWithJobDbJobsWithTxn(txn, nil, node.DeepCopyNilKeys()) require.NoError(b, err) } txn.Commit() @@ -2454,9 +2442,7 @@ func BenchmarkPreemptingQueueScheduler(b *testing.B) { require.NoError(b, err) txn = nodeDb.Txn(true) for _, node := range tc.Nodes { - dbNode, err := testfixtures.TestNodeFactory.FromSchedulerObjectsNode(node) - require.NoError(b, err) - err = nodeDb.CreateAndInsertWithJobDbJobsWithTxn(txn, jobsByNodeId[node.Id], dbNode) + err = nodeDb.CreateAndInsertWithJobDbJobsWithTxn(txn, jobsByNodeId[node.GetId()], node.DeepCopyNilKeys()) require.NoError(b, err) } txn.Commit() diff --git a/internal/scheduler/scheduling/queue_scheduler_test.go b/internal/scheduler/scheduling/queue_scheduler_test.go index fdb6f47cc2c..36b23d6a51b 100644 --- a/internal/scheduler/scheduling/queue_scheduler_test.go +++ b/internal/scheduler/scheduling/queue_scheduler_test.go @@ -40,7 +40,7 @@ func TestQueueScheduler(t *testing.T) { // Initial resource usage for all queues. InitialAllocatedByQueueAndPriorityClass map[string]schedulerobjects.QuantityByTAndResourceType[string] // Nodes to be considered by the scheduler. - Nodes []*schedulerobjects.Node + Nodes []*internaltypes.Node // Jobs to try scheduling. Jobs []*jobdb.Job // Indices of jobs expected to be scheduled. @@ -51,48 +51,48 @@ func TestQueueScheduler(t *testing.T) { "simple success": { SchedulingConfig: testfixtures.TestSchedulingConfig(), Queues: testfixtures.SingleQueuePriorityOne("A"), - Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), Jobs: testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 32), ExpectedScheduledIndices: testfixtures.IntRange(0, 31), }, "simple failure": { SchedulingConfig: testfixtures.TestSchedulingConfig(), Queues: testfixtures.SingleQueuePriorityOne("A"), - Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), Jobs: testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 33), ExpectedScheduledIndices: testfixtures.IntRange(0, 31), }, "multiple nodes": { SchedulingConfig: testfixtures.TestSchedulingConfig(), Queues: testfixtures.SingleQueuePriorityOne("A"), - Nodes: testfixtures.N32CpuNodes(2, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(2, testfixtures.TestPriorities), Jobs: testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 64), ExpectedScheduledIndices: testfixtures.IntRange(0, 63), }, "preempt lower-priority jobs": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), Jobs: armadaslices.Concatenate(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1), testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass1, 1)), Queues: testfixtures.SingleQueuePriorityOne("A"), ExpectedScheduledIndices: testfixtures.IntRange(0, 1), }, "no preemption of higher-priority jobs": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), Jobs: armadaslices.Concatenate(testfixtures.N32Cpu256GiJobs("A", testfixtures.PriorityClass1, 1), testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1)), Queues: testfixtures.SingleQueuePriorityOne("A"), ExpectedScheduledIndices: testfixtures.IntRange(0, 0), }, "unschedulable jobs do not block schedulable jobs": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), Jobs: armadaslices.Concatenate(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1), testfixtures.N32Cpu256GiJobs("A", testfixtures.PriorityClass0, 10), testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1)), Queues: testfixtures.SingleQueuePriorityOne("A"), ExpectedScheduledIndices: []int{0, 11}, }, "MaximumSchedulingBurst": { SchedulingConfig: testfixtures.WithGlobalSchedulingRateLimiterConfig(10, 2, testfixtures.TestSchedulingConfig()), - Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), Jobs: armadaslices.Concatenate( testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1), testfixtures.N32Cpu256GiJobs("A", testfixtures.PriorityClass0, 10), @@ -104,7 +104,7 @@ func TestQueueScheduler(t *testing.T) { }, "MaximumPerQueueSchedulingBurst": { SchedulingConfig: testfixtures.WithPerQueueSchedulingLimiterConfig(10, 2, testfixtures.TestSchedulingConfig()), - Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), Jobs: armadaslices.Concatenate( testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1), testfixtures.N32Cpu256GiJobs("A", testfixtures.PriorityClass0, 10), @@ -117,7 +117,7 @@ func TestQueueScheduler(t *testing.T) { }, "MaximumSchedulingBurst is not exceeded by gangs": { SchedulingConfig: testfixtures.WithGlobalSchedulingRateLimiterConfig(10, 2, testfixtures.TestSchedulingConfig()), - Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), Jobs: armadaslices.Concatenate( testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1), testfixtures.N32Cpu256GiJobs("A", testfixtures.PriorityClass0, 1), @@ -131,7 +131,7 @@ func TestQueueScheduler(t *testing.T) { }, "MaximumPerQueueSchedulingBurst is not exceeded by gangs": { SchedulingConfig: testfixtures.WithPerQueueSchedulingLimiterConfig(10, 2, testfixtures.TestSchedulingConfig()), - Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), Jobs: armadaslices.Concatenate( testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1), testfixtures.N32Cpu256GiJobs("A", testfixtures.PriorityClass0, 1), @@ -149,7 +149,7 @@ func TestQueueScheduler(t *testing.T) { testfixtures.TestSchedulingConfig(), ), Queues: testfixtures.SingleQueuePriorityOne("A"), - Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), Jobs: testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 32), ExpectedScheduledIndices: testfixtures.IntRange(0, 16), ExpectedNeverAttemptedIndices: testfixtures.IntRange(17, 31), @@ -165,7 +165,7 @@ func TestQueueScheduler(t *testing.T) { testfixtures.TestSchedulingConfig(), ), Queues: testfixtures.SingleQueuePriorityOne("A"), - Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), Jobs: armadaslices.Concatenate( testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1), testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 2), @@ -201,20 +201,20 @@ func TestQueueScheduler(t *testing.T) { }, }, }, - Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), Jobs: testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 32), ExpectedScheduledIndices: testfixtures.IntRange(0, 15), }, "fairness two queues": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), Jobs: armadaslices.Concatenate(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 32), testfixtures.N1Cpu4GiJobs("B", testfixtures.PriorityClass0, 32)), Queues: []*api.Queue{{Name: "A", PriorityFactor: 1.0}, {Name: "B", PriorityFactor: 1.0}}, ExpectedScheduledIndices: armadaslices.Concatenate(testfixtures.IntRange(0, 15), testfixtures.IntRange(32, 47)), }, "fairness three queues": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), Jobs: armadaslices.Concatenate( testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 32), testfixtures.N1Cpu4GiJobs("B", testfixtures.PriorityClass0, 32), @@ -229,7 +229,7 @@ func TestQueueScheduler(t *testing.T) { }, "weighted fairness two queues": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.N32CpuNodes(3, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(3, testfixtures.TestPriorities), Jobs: armadaslices.Concatenate( testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 96), testfixtures.N1Cpu4GiJobs("B", testfixtures.PriorityClass0, 96), @@ -242,7 +242,7 @@ func TestQueueScheduler(t *testing.T) { }, "weighted fairness three queues": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.N32CpuNodes(3, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(3, testfixtures.TestPriorities), Jobs: armadaslices.Concatenate( testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 96), testfixtures.N1Cpu4GiJobs("B", testfixtures.PriorityClass0, 96), @@ -257,7 +257,7 @@ func TestQueueScheduler(t *testing.T) { }, "fairness two queues with initial allocation": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), Jobs: armadaslices.Concatenate( testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 32), testfixtures.N1Cpu4GiJobs("B", testfixtures.PriorityClass0, 32), @@ -276,14 +276,10 @@ func TestQueueScheduler(t *testing.T) { }, "node with no available capacity": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.WithUsedResourcesNodes( + Nodes: testfixtures.ItWithUsedResourcesNodes( 0, - schedulerobjects.ResourceList{ - Resources: map[string]resource.Quantity{ - "cpu": resource.MustParse("32"), - }, - }, - testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.Cpu("32"), + testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), ), Jobs: testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1), Queues: testfixtures.SingleQueuePriorityOne("A"), @@ -291,14 +287,10 @@ func TestQueueScheduler(t *testing.T) { }, "node with some available capacity": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.WithUsedResourcesNodes( + Nodes: testfixtures.ItWithUsedResourcesNodes( 0, - schedulerobjects.ResourceList{ - Resources: map[string]resource.Quantity{ - "cpu": resource.MustParse("31"), - }, - }, - testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.Cpu("31"), + testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), ), Jobs: testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 2), Queues: testfixtures.SingleQueuePriorityOne("A"), @@ -306,14 +298,10 @@ func TestQueueScheduler(t *testing.T) { }, "preempt used resources of lower-priority jobs": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.WithUsedResourcesNodes( + Nodes: testfixtures.ItWithUsedResourcesNodes( 0, - schedulerobjects.ResourceList{ - Resources: map[string]resource.Quantity{ - "cpu": resource.MustParse("32"), - }, - }, - testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.Cpu("32"), + testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), ), Jobs: testfixtures.N32Cpu256GiJobs("A", testfixtures.PriorityClass1, 1), Queues: testfixtures.SingleQueuePriorityOne("A"), @@ -321,14 +309,14 @@ func TestQueueScheduler(t *testing.T) { }, "respect taints": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.NTainted32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItNTainted32CpuNodes(1, testfixtures.TestPriorities), Jobs: armadaslices.Concatenate(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1), testfixtures.N32Cpu256GiJobs("A", testfixtures.PriorityClass0, 1)), Queues: testfixtures.SingleQueuePriorityOne("A"), ExpectedScheduledIndices: []int{1}, }, "taints and tolerations": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.NTainted32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItNTainted32CpuNodes(1, testfixtures.TestPriorities), Jobs: armadaslices.Concatenate(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1), testfixtures.N32Cpu256GiJobs("A", testfixtures.PriorityClass0, 1)), Queues: testfixtures.SingleQueuePriorityOne("A"), ExpectedScheduledIndices: []int{1}, @@ -336,8 +324,11 @@ func TestQueueScheduler(t *testing.T) { "Node selector": { SchedulingConfig: testfixtures.TestSchedulingConfig(), Nodes: armadaslices.Concatenate( - testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), - testfixtures.WithLabelsNodes(map[string]string{"foo": "foo"}, testfixtures.N32CpuNodes(1, testfixtures.TestPriorities)), + testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.TestNodeFactory.AddLabels( + testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + map[string]string{"foo": "foo"}, + ), ), Jobs: testfixtures.WithNodeSelectorJobs(map[string]string{"foo": "foo"}, testfixtures.N32Cpu256GiJobs("A", testfixtures.PriorityClass0, 2)), Queues: testfixtures.SingleQueuePriorityOne("A"), @@ -345,7 +336,7 @@ func TestQueueScheduler(t *testing.T) { }, "taints and tolerations (indexed)": { SchedulingConfig: testfixtures.WithIndexedTaintsConfig([]string{"largeJobsOnly"}, testfixtures.TestSchedulingConfig()), - Nodes: testfixtures.NTainted32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItNTainted32CpuNodes(1, testfixtures.TestPriorities), Jobs: armadaslices.Concatenate(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1), testfixtures.N32Cpu256GiJobs("A", testfixtures.PriorityClass0, 1)), Queues: testfixtures.SingleQueuePriorityOne("A"), ExpectedScheduledIndices: []int{1}, @@ -353,8 +344,11 @@ func TestQueueScheduler(t *testing.T) { "Node selector (indexed)": { SchedulingConfig: testfixtures.WithIndexedNodeLabelsConfig([]string{"foo"}, testfixtures.TestSchedulingConfig()), Nodes: append( - testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), - testfixtures.WithLabelsNodes(map[string]string{"foo": "foo"}, testfixtures.N32CpuNodes(1, testfixtures.TestPriorities))..., + testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.TestNodeFactory.AddLabels( + testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + map[string]string{"foo": "foo"}, + )..., ), Jobs: testfixtures.WithNodeSelectorJobs(map[string]string{"foo": "foo"}, testfixtures.N32Cpu256GiJobs("A", testfixtures.PriorityClass0, 2)), Queues: testfixtures.SingleQueuePriorityOne("A"), @@ -362,7 +356,7 @@ func TestQueueScheduler(t *testing.T) { }, "MaxQueueLookback": { SchedulingConfig: testfixtures.WithMaxQueueLookbackConfig(3, testfixtures.TestSchedulingConfig()), - Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), Jobs: armadaslices.Concatenate( testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1), testfixtures.N32Cpu256GiJobs("A", testfixtures.PriorityClass0, 3), @@ -374,14 +368,14 @@ func TestQueueScheduler(t *testing.T) { }, "gang success": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.N32CpuNodes(2, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(2, testfixtures.TestPriorities), Jobs: testfixtures.WithGangAnnotationsJobs(testfixtures.N32Cpu256GiJobs("A", testfixtures.PriorityClass0, 2)), Queues: testfixtures.SingleQueuePriorityOne("A"), ExpectedScheduledIndices: []int{0, 1}, }, "non-consecutive gang success": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.N32CpuNodes(3, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(3, testfixtures.TestPriorities), Jobs: armadaslices.Concatenate( testfixtures.WithAnnotationsJobs(map[string]string{ armadaconfiguration.GangIdAnnotation: "my-gang", @@ -400,14 +394,14 @@ func TestQueueScheduler(t *testing.T) { }, "gang failure": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.N32CpuNodes(2, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(2, testfixtures.TestPriorities), Jobs: testfixtures.WithGangAnnotationsJobs(testfixtures.N32Cpu256GiJobs("A", testfixtures.PriorityClass0, 3)), Queues: testfixtures.SingleQueuePriorityOne("A"), ExpectedScheduledIndices: nil, }, "non-consecutive gang failure": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.N32CpuNodes(2, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(2, testfixtures.TestPriorities), Jobs: armadaslices.Concatenate( testfixtures.WithAnnotationsJobs(map[string]string{ armadaconfiguration.GangIdAnnotation: "my-gang", @@ -426,7 +420,7 @@ func TestQueueScheduler(t *testing.T) { }, "job priority": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), Jobs: armadaslices.Concatenate( testfixtures.WithPriorityJobs(10, testfixtures.N32Cpu256GiJobs("A", testfixtures.PriorityClass0, 1)), testfixtures.WithPriorityJobs(1, testfixtures.N32Cpu256GiJobs("A", testfixtures.PriorityClass0, 1)), @@ -438,19 +432,19 @@ func TestQueueScheduler(t *testing.T) { "nodeAffinity node notIn": { SchedulingConfig: testfixtures.TestSchedulingConfig(), Nodes: armadaslices.Concatenate( - testfixtures.WithLabelsNodes( + testfixtures.TestNodeFactory.AddLabels( + testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), map[string]string{"key": "val1"}, - testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), ), - testfixtures.WithLabelsNodes( + testfixtures.TestNodeFactory.AddLabels( + testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), map[string]string{"key": "val2"}, - testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), ), - testfixtures.WithLabelsNodes( + testfixtures.TestNodeFactory.AddLabels( + testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), map[string]string{"key": "val3"}, - testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), ), - testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), ), Jobs: armadaslices.Concatenate( testfixtures.WithNodeAffinityJobs( @@ -478,9 +472,7 @@ func TestQueueScheduler(t *testing.T) { require.NoError(t, err) txn := nodeDb.Txn(true) for _, node := range tc.Nodes { - dbNode, err := testfixtures.TestNodeFactory.FromSchedulerObjectsNode(node) - require.NoError(t, err) - err = nodeDb.CreateAndInsertWithJobDbJobsWithTxn(txn, nil, dbNode) + err = nodeDb.CreateAndInsertWithJobDbJobsWithTxn(txn, nil, node.DeepCopyNilKeys()) require.NoError(t, err) } txn.Commit() diff --git a/internal/scheduler/scheduling/scheduling_algo_test.go b/internal/scheduler/scheduling/scheduling_algo_test.go index b69ab010244..4355086020c 100644 --- a/internal/scheduler/scheduling/scheduling_algo_test.go +++ b/internal/scheduler/scheduling/scheduling_algo_test.go @@ -638,10 +638,10 @@ func BenchmarkNodeDbConstruction(b *testing.B) { numNodes := int(math.Pow10(e)) b.Run(fmt.Sprintf("%d nodes", numNodes), func(b *testing.B) { jobs := testfixtures.N1Cpu4GiJobs("queue-alice", testfixtures.PriorityClass0, 32*numNodes) - nodes := testfixtures.N32CpuNodes(numNodes, testfixtures.TestPriorities) + nodes := testfixtures.ItN32CpuNodes(numNodes, testfixtures.TestPriorities) for i, node := range nodes { for j := 32 * i; j < 32*(i+1); j++ { - jobs[j] = jobs[j].WithNewRun("executor-01", node.Id, node.Name, node.Pool, jobs[j].PriorityClass().Priority) + jobs[j] = jobs[j].WithNewRun("executor-01", node.GetId(), node.GetName(), node.GetPool(), jobs[j].PriorityClass().Priority) } } armadaslices.Shuffle(jobs) @@ -661,12 +661,6 @@ func BenchmarkNodeDbConstruction(b *testing.B) { require.NoError(b, err) b.StartTimer() - nodeFactory := internaltypes.NewNodeFactory( - schedulingConfig.IndexedTaints, - schedulingConfig.IndexedNodeLabels, - testfixtures.TestResourceListFactory, - ) - nodeDb, err := nodedb.NewNodeDb( schedulingConfig.PriorityClasses, schedulingConfig.IndexedResources, @@ -679,9 +673,7 @@ func BenchmarkNodeDbConstruction(b *testing.B) { dbNodes := []*internaltypes.Node{} for _, node := range nodes { - dbNode, err := nodeFactory.FromSchedulerObjectsNode(node) - require.NoError(b, err) - dbNodes = append(dbNodes, dbNode) + dbNodes = append(dbNodes, node.DeepCopyNilKeys()) } err = algo.populateNodeDb(nodeDb, jobs, []*jobdb.Job{}, dbNodes) diff --git a/internal/scheduler/testfixtures/testfixtures.go b/internal/scheduler/testfixtures/testfixtures.go index 5c66f044119..10469f318fc 100644 --- a/internal/scheduler/testfixtures/testfixtures.go +++ b/internal/scheduler/testfixtures/testfixtures.go @@ -290,13 +290,6 @@ func WithMaxQueueLookbackConfig(maxQueueLookback uint, config schedulerconfigura return config } -func WithUsedResourcesNodes(p int32, rl schedulerobjects.ResourceList, nodes []*schedulerobjects.Node) []*schedulerobjects.Node { - for _, node := range nodes { - schedulerobjects.AllocatableByPriorityAndResourceType(node.AllocatableByPriorityAndResource).MarkAllocated(p, rl) - } - return nodes -} - func ItWithUsedResourcesNodes(p int32, rl internaltypes.ResourceList, nodes []*internaltypes.Node) []*internaltypes.Node { for _, node := range nodes { internaltypes.MarkAllocated(node.AllocatableByPriority, p, rl) @@ -304,17 +297,6 @@ func ItWithUsedResourcesNodes(p int32, rl internaltypes.ResourceList, nodes []*i return nodes } -func WithLabelsNodes(labels map[string]string, nodes []*schedulerobjects.Node) []*schedulerobjects.Node { - for _, node := range nodes { - if node.Labels == nil { - node.Labels = maps.Clone(labels) - } else { - maps.Copy(node.Labels, labels) - } - } - return nodes -} - func ItWithNodeTypeNodes(nodeType *internaltypes.NodeType, nodes []*internaltypes.Node) []*internaltypes.Node { result := make([]*internaltypes.Node, len(nodes)) for i, node := range nodes { @@ -777,14 +759,6 @@ func TestCluster() []*schedulerobjects.Node { } } -func N32CpuNodes(n int, priorities []int32) []*schedulerobjects.Node { - rv := make([]*schedulerobjects.Node, n) - for i := 0; i < n; i++ { - rv[i] = Test32CpuNode(priorities) - } - return rv -} - func ItN32CpuNodes(n int, priorities []int32) []*internaltypes.Node { rv := make([]*internaltypes.Node, n) for i := 0; i < n; i++ { @@ -793,14 +767,6 @@ func ItN32CpuNodes(n int, priorities []int32) []*internaltypes.Node { return rv } -func NTainted32CpuNodes(n int, priorities []int32) []*schedulerobjects.Node { - rv := make([]*schedulerobjects.Node, n) - for i := 0; i < n; i++ { - rv[i] = TestTainted32CpuNode(priorities) - } - return rv -} - func ItNTainted32CpuNodes(n int, priorities []int32) []*internaltypes.Node { rv := make([]*internaltypes.Node, n) for i := 0; i < n; i++ { @@ -809,14 +775,6 @@ func ItNTainted32CpuNodes(n int, priorities []int32) []*internaltypes.Node { return rv } -func N8GpuNodes(n int, priorities []int32) []*schedulerobjects.Node { - rv := make([]*schedulerobjects.Node, n) - for i := 0; i < n; i++ { - rv[i] = Test8GpuNode(priorities) - } - return rv -} - func ItN8GpuNodes(n int, priorities []int32) []*internaltypes.Node { rv := make([]*internaltypes.Node, n) for i := 0; i < n; i++ { @@ -1131,3 +1089,21 @@ func GetTestSupportedResourceTypes() []schedulerconfiguration.ResourceType { {Name: "nvidia.com/gpu", Resolution: resource.MustParse("1m")}, } } + +func Cpu(cpu string) internaltypes.ResourceList { + return CpuMemGpu(cpu, "0", "0") +} + +func CpuMem(cpu string, memory string) internaltypes.ResourceList { + return CpuMemGpu(cpu, memory, "0") +} + +func CpuMemGpu(cpu string, memory string, gpu string) internaltypes.ResourceList { + return TestResourceListFactory.FromNodeProto( + map[string]resource.Quantity{ + "cpu": resource.MustParse(cpu), + "memory": resource.MustParse(memory), + "nvidia.com/gpu": resource.MustParse(gpu), + }, + ) +} From 63b92025524134479e13247f43096a7d57d30116 Mon Sep 17 00:00:00 2001 From: Rob Smith <34475852+robertdavidsmith@users.noreply.github.com> Date: Fri, 3 Jan 2025 12:53:14 +0000 Subject: [PATCH 05/17] Scheduler: refactor testfixtures (#4115) Signed-off-by: Robert Smith --- .../testfixtures/testfixtures.go | 24 --- internal/scheduler/metrics_test.go | 8 +- internal/scheduler/nodedb/nodedb_test.go | 5 +- .../scheduler/scheduling/context/job_test.go | 3 +- .../scheduling/preemption_description_test.go | 23 ++- .../scheduling/scheduling_algo_test.go | 171 ++++++++++++----- internal/scheduler/submitcheck_test.go | 4 +- .../scheduler/testfixtures/testfixtures.go | 180 ++---------------- 8 files changed, 166 insertions(+), 252 deletions(-) delete mode 100644 internal/scheduler/internaltypes/testfixtures/testfixtures.go diff --git a/internal/scheduler/internaltypes/testfixtures/testfixtures.go b/internal/scheduler/internaltypes/testfixtures/testfixtures.go deleted file mode 100644 index 9c8414a747d..00000000000 --- a/internal/scheduler/internaltypes/testfixtures/testfixtures.go +++ /dev/null @@ -1,24 +0,0 @@ -package testfixtures - -import ( - "github.com/armadaproject/armada/internal/scheduler/internaltypes" -) - -func TestSimpleNode(id string) *internaltypes.Node { - return internaltypes.CreateNode( - id, - nil, - 0, - "", - "", - "", - nil, - nil, - internaltypes.ResourceList{}, - nil, - nil, - nil, - nil, - nil, - nil) -} diff --git a/internal/scheduler/metrics_test.go b/internal/scheduler/metrics_test.go index b67c29598ab..c3bb11cd738 100644 --- a/internal/scheduler/metrics_test.go +++ b/internal/scheduler/metrics_test.go @@ -515,7 +515,13 @@ func createExecutor(clusterName string, nodes ...*schedulerobjects.Node) *schedu } func createNode(nodeType string) *schedulerobjects.Node { - node := testfixtures.Test32CpuNode([]int32{}) + node := testfixtures.TestSchedulerObjectsNode( + []int32{}, + map[string]resource.Quantity{ + "cpu": resource.MustParse("32"), + "memory": resource.MustParse("256Gi"), + }, + ) node.ReportingNodeType = nodeType node.StateByJobRunId = map[string]schedulerobjects.JobRunState{} node.ResourceUsageByQueueAndPool = []*schedulerobjects.PoolQueueResource{} diff --git a/internal/scheduler/nodedb/nodedb_test.go b/internal/scheduler/nodedb/nodedb_test.go index 84eb691f41b..3a913ad834d 100644 --- a/internal/scheduler/nodedb/nodedb_test.go +++ b/internal/scheduler/nodedb/nodedb_test.go @@ -14,7 +14,6 @@ import ( "github.com/armadaproject/armada/internal/common/util" schedulerconfig "github.com/armadaproject/armada/internal/scheduler/configuration" "github.com/armadaproject/armada/internal/scheduler/internaltypes" - ittestfixtures "github.com/armadaproject/armada/internal/scheduler/internaltypes/testfixtures" "github.com/armadaproject/armada/internal/scheduler/jobdb" "github.com/armadaproject/armada/internal/scheduler/schedulerobjects" "github.com/armadaproject/armada/internal/scheduler/scheduling/context" @@ -74,7 +73,7 @@ func TestSelectNodeForPod_NodeIdLabel_Success(t *testing.T) { jctxs := context.JobSchedulingContextsFromJobs(jobs) for _, jctx := range jctxs { txn := db.Txn(false) - jctx.SetAssignedNode(ittestfixtures.TestSimpleNode(nodeId)) + jctx.SetAssignedNode(testfixtures.TestSimpleNode(nodeId)) node, err := db.SelectNodeForJobWithTxn(txn, jctx) txn.Abort() require.NoError(t, err) @@ -99,7 +98,7 @@ func TestSelectNodeForPod_NodeIdLabel_Failure(t *testing.T) { jctxs := context.JobSchedulingContextsFromJobs(jobs) for _, jctx := range jctxs { txn := db.Txn(false) - jctx.SetAssignedNode(ittestfixtures.TestSimpleNode("non-existent node")) + jctx.SetAssignedNode(testfixtures.TestSimpleNode("non-existent node")) node, err := db.SelectNodeForJobWithTxn(txn, jctx) txn.Abort() if !assert.NoError(t, err) { diff --git a/internal/scheduler/scheduling/context/job_test.go b/internal/scheduler/scheduling/context/job_test.go index 42999746039..77242139f01 100644 --- a/internal/scheduler/scheduling/context/job_test.go +++ b/internal/scheduler/scheduling/context/job_test.go @@ -6,7 +6,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/armadaproject/armada/internal/scheduler/configuration" - ittestfixtures "github.com/armadaproject/armada/internal/scheduler/internaltypes/testfixtures" "github.com/armadaproject/armada/internal/scheduler/testfixtures" ) @@ -23,7 +22,7 @@ func TestJobSchedulingContext_SetAssignedNode(t *testing.T) { assert.Empty(t, jctx.GetAssignedNodeId()) assert.Empty(t, jctx.AdditionalNodeSelectors) - n := ittestfixtures.TestSimpleNode("node1") + n := testfixtures.TestSimpleNode("node1") jctx.SetAssignedNode(n) assert.Equal(t, n, jctx.GetAssignedNode()) assert.Equal(t, "node1", jctx.GetAssignedNodeId()) diff --git a/internal/scheduler/scheduling/preemption_description_test.go b/internal/scheduler/scheduling/preemption_description_test.go index 22482677e78..c0ba13a325a 100644 --- a/internal/scheduler/scheduling/preemption_description_test.go +++ b/internal/scheduler/scheduling/preemption_description_test.go @@ -7,7 +7,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - ittestfixtures "github.com/armadaproject/armada/internal/scheduler/internaltypes/testfixtures" "github.com/armadaproject/armada/internal/scheduler/jobdb" "github.com/armadaproject/armada/internal/scheduler/schedulerobjects" "github.com/armadaproject/armada/internal/scheduler/scheduling/context" @@ -40,25 +39,25 @@ func TestPopulatePreemptionDescriptions(t *testing.T) { "unknown cause - basic job": { preemptedJobContext: &context.JobSchedulingContext{ JobId: "job-1", - AssignedNode: ittestfixtures.TestSimpleNode("node-3"), + AssignedNode: testfixtures.TestSimpleNode("node-3"), Job: makeJob(t, "job-1", false), }, expectedPreemptedJobContext: &context.JobSchedulingContext{ JobId: "job-1", - AssignedNode: ittestfixtures.TestSimpleNode("node-3"), + AssignedNode: testfixtures.TestSimpleNode("node-3"), Job: makeJob(t, "job-1", false), - PreemptionDescription: fmt.Sprintf(unknownPreemptionCause, ittestfixtures.TestSimpleNode("node-3").SummaryString()), + PreemptionDescription: fmt.Sprintf(unknownPreemptionCause, testfixtures.TestSimpleNode("node-3").SummaryString()), }, }, "unknown cause - gang job": { preemptedJobContext: &context.JobSchedulingContext{ JobId: "job-1", - AssignedNode: ittestfixtures.TestSimpleNode("node-3"), + AssignedNode: testfixtures.TestSimpleNode("node-3"), Job: makeJob(t, "job-1", true), }, expectedPreemptedJobContext: &context.JobSchedulingContext{ JobId: "job-1", - AssignedNode: ittestfixtures.TestSimpleNode("node-3"), + AssignedNode: testfixtures.TestSimpleNode("node-3"), Job: makeJob(t, "job-1", true), PreemptionDescription: unknownGangPreemptionCause, }, @@ -66,34 +65,34 @@ func TestPopulatePreemptionDescriptions(t *testing.T) { "urgency preemption - single preempting job": { preemptedJobContext: &context.JobSchedulingContext{ JobId: "job-1", - AssignedNode: ittestfixtures.TestSimpleNode("node-1"), + AssignedNode: testfixtures.TestSimpleNode("node-1"), }, expectedPreemptedJobContext: &context.JobSchedulingContext{ JobId: "job-1", - AssignedNode: ittestfixtures.TestSimpleNode("node-1"), + AssignedNode: testfixtures.TestSimpleNode("node-1"), PreemptionDescription: fmt.Sprintf(urgencyPreemptionTemplate, "job-2"), }, }, "urgency preemption - multiple preempting jobs": { preemptedJobContext: &context.JobSchedulingContext{ JobId: "job-1", - AssignedNode: ittestfixtures.TestSimpleNode("node-2"), + AssignedNode: testfixtures.TestSimpleNode("node-2"), }, expectedPreemptedJobContext: &context.JobSchedulingContext{ JobId: "job-1", - AssignedNode: ittestfixtures.TestSimpleNode("node-2"), + AssignedNode: testfixtures.TestSimpleNode("node-2"), PreemptionDescription: fmt.Sprintf(urgencyPreemptionMultiJobTemplate, "job-3,job-4"), }, }, "fairshare": { preemptedJobContext: &context.JobSchedulingContext{ JobId: "job-1", - AssignedNode: ittestfixtures.TestSimpleNode("node-4"), + AssignedNode: testfixtures.TestSimpleNode("node-4"), PreemptingJobId: "job-7", }, expectedPreemptedJobContext: &context.JobSchedulingContext{ JobId: "job-1", - AssignedNode: ittestfixtures.TestSimpleNode("node-4"), + AssignedNode: testfixtures.TestSimpleNode("node-4"), PreemptingJobId: "job-7", PreemptionDescription: fmt.Sprintf(fairSharePreemptionTemplate, "job-7"), }, diff --git a/internal/scheduler/scheduling/scheduling_algo_test.go b/internal/scheduler/scheduling/scheduling_algo_test.go index 4355086020c..f5dfdf51a90 100644 --- a/internal/scheduler/scheduling/scheduling_algo_test.go +++ b/internal/scheduler/scheduling/scheduling_algo_test.go @@ -10,6 +10,7 @@ import ( "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" "golang.org/x/exp/slices" + v1 "k8s.io/api/core/v1" k8sResource "k8s.io/apimachinery/pkg/api/resource" clock "k8s.io/utils/clock/testing" @@ -64,8 +65,8 @@ func TestSchedule(t *testing.T) { "scheduling": { schedulingConfig: testfixtures.TestSchedulingConfig(), executors: []*schedulerobjects.Executor{ - testfixtures.Test1Node32CoreExecutor("executor1"), - testfixtures.Test1Node32CoreExecutor("executor2"), + test1Node32CoreExecutor("executor1"), + test1Node32CoreExecutor("executor2"), }, queues: []*api.Queue{testfixtures.MakeTestQueue()}, queuedJobs: testfixtures.N16Cpu128GiJobs(testfixtures.TestQueue, testfixtures.PriorityClass3, 10), @@ -75,8 +76,8 @@ func TestSchedule(t *testing.T) { "scheduling - home away": { schedulingConfig: testfixtures.TestSchedulingConfig(), executors: []*schedulerobjects.Executor{ - testfixtures.MakeTestExecutorWithNodes("executor-1", - testfixtures.WithLargeNodeTaint(testfixtures.TestNodeWithPool(testfixtures.TestPool))), + makeTestExecutorWithNodes("executor-1", + withLargeNodeTaint(testNodeWithPool(testfixtures.TestPool))), }, queues: []*api.Queue{testfixtures.MakeTestQueue()}, queuedJobs: testfixtures.WithPools(testfixtures.N16Cpu128GiJobs(testfixtures.TestQueue, testfixtures.PriorityClass4PreemptibleAway, 10), []string{testfixtures.TestPool}), @@ -86,8 +87,8 @@ func TestSchedule(t *testing.T) { "scheduling - cross pool - home away": { schedulingConfig: multiPoolSchedulingConfig, executors: []*schedulerobjects.Executor{ - testfixtures.MakeTestExecutorWithNodes("executor-1", - testfixtures.WithLargeNodeTaint(testfixtures.TestNodeWithPool(testfixtures.TestPool2))), + makeTestExecutorWithNodes("executor-1", + withLargeNodeTaint(testNodeWithPool(testfixtures.TestPool2))), }, queues: []*api.Queue{testfixtures.MakeTestQueue()}, queuedJobs: testfixtures.WithPools(testfixtures.N16Cpu128GiJobs(testfixtures.TestQueue, testfixtures.PriorityClass4PreemptibleAway, 10), []string{testfixtures.TestPool, testfixtures.AwayPool}), @@ -97,8 +98,8 @@ func TestSchedule(t *testing.T) { "scheduling - mixed pool clusters": { schedulingConfig: testfixtures.TestSchedulingConfigWithPools([]configuration.PoolConfig{{Name: "pool-1"}, {Name: "pool-2"}}), executors: []*schedulerobjects.Executor{ - testfixtures.MakeTestExecutor("executor-1", "pool-1", "pool-2"), - testfixtures.MakeTestExecutor("executor-2", "pool-1"), + makeTestExecutor("executor-1", "pool-1", "pool-2"), + makeTestExecutor("executor-2", "pool-1"), }, queues: []*api.Queue{testfixtures.MakeTestQueue()}, queuedJobs: testfixtures.WithPools(testfixtures.N16Cpu128GiJobs(testfixtures.TestQueue, testfixtures.PriorityClass3, 10), []string{"pool-1", "pool-2"}), @@ -108,8 +109,8 @@ func TestSchedule(t *testing.T) { "Fair share": { schedulingConfig: testfixtures.TestSchedulingConfig(), executors: []*schedulerobjects.Executor{ - testfixtures.Test1Node32CoreExecutor("executor1"), - testfixtures.Test1Node32CoreExecutor("executor2"), + test1Node32CoreExecutor("executor1"), + test1Node32CoreExecutor("executor2"), }, queues: []*api.Queue{ { @@ -130,8 +131,8 @@ func TestSchedule(t *testing.T) { "do not schedule onto stale executors": { schedulingConfig: testfixtures.TestSchedulingConfig(), executors: []*schedulerobjects.Executor{ - testfixtures.Test1Node32CoreExecutor("executor1"), - testfixtures.WithLastUpdateTimeExecutor(testfixtures.BaseTime.Add(-1*time.Hour), testfixtures.Test1Node32CoreExecutor("executor2")), + test1Node32CoreExecutor("executor1"), + withLastUpdateTimeExecutor(testfixtures.BaseTime.Add(-1*time.Hour), test1Node32CoreExecutor("executor2")), }, queues: []*api.Queue{testfixtures.MakeTestQueue()}, queuedJobs: testfixtures.N16Cpu128GiJobs(testfixtures.TestQueue, testfixtures.PriorityClass3, 10), @@ -140,8 +141,8 @@ func TestSchedule(t *testing.T) { "schedule onto executors with some unacknowledged jobs": { schedulingConfig: testfixtures.WithMaxUnacknowledgedJobsPerExecutorConfig(16, testfixtures.TestSchedulingConfig()), executors: []*schedulerobjects.Executor{ - testfixtures.Test1Node32CoreExecutor("executor1"), - testfixtures.Test1Node32CoreExecutor("executor2"), + test1Node32CoreExecutor("executor1"), + test1Node32CoreExecutor("executor2"), }, queues: []*api.Queue{testfixtures.MakeTestQueue()}, queuedJobs: testfixtures.N1Cpu4GiJobs(testfixtures.TestQueue, testfixtures.PriorityClass3, 48), @@ -158,8 +159,8 @@ func TestSchedule(t *testing.T) { "do not schedule onto executors with too many unacknowledged jobs": { schedulingConfig: testfixtures.WithMaxUnacknowledgedJobsPerExecutorConfig(15, testfixtures.TestSchedulingConfig()), executors: []*schedulerobjects.Executor{ - testfixtures.Test1Node32CoreExecutor("executor1"), - testfixtures.Test1Node32CoreExecutor("executor2"), + test1Node32CoreExecutor("executor1"), + test1Node32CoreExecutor("executor2"), }, queues: []*api.Queue{testfixtures.MakeTestQueue()}, queuedJobs: testfixtures.N1Cpu4GiJobs(testfixtures.TestQueue, testfixtures.PriorityClass3, 48), @@ -176,8 +177,8 @@ func TestSchedule(t *testing.T) { "one executor full": { schedulingConfig: testfixtures.TestSchedulingConfig(), executors: []*schedulerobjects.Executor{ - testfixtures.Test1Node32CoreExecutor("executor1"), - testfixtures.Test1Node32CoreExecutor("executor2"), + test1Node32CoreExecutor("executor1"), + test1Node32CoreExecutor("executor2"), }, queues: []*api.Queue{testfixtures.MakeTestQueue()}, queuedJobs: testfixtures.N16Cpu128GiJobs(testfixtures.TestQueue, testfixtures.PriorityClass3, 10), @@ -199,8 +200,8 @@ func TestSchedule(t *testing.T) { testfixtures.TestSchedulingConfig(), ), executors: []*schedulerobjects.Executor{ - testfixtures.Test1Node32CoreExecutor("executor1"), - testfixtures.Test1Node32CoreExecutor("executor2"), + test1Node32CoreExecutor("executor1"), + test1Node32CoreExecutor("executor2"), }, queues: []*api.Queue{testfixtures.MakeTestQueue()}, queuedJobs: testfixtures.N16Cpu128GiJobs(testfixtures.TestQueue, testfixtures.PriorityClass3, 10), @@ -221,8 +222,8 @@ func TestSchedule(t *testing.T) { testfixtures.TestSchedulingConfig(), ), executors: []*schedulerobjects.Executor{ - testfixtures.Test1Node32CoreExecutor("executor1"), - testfixtures.Test1Node32CoreExecutor("executor2"), + test1Node32CoreExecutor("executor1"), + test1Node32CoreExecutor("executor2"), }, queues: []*api.Queue{testfixtures.MakeTestQueue()}, queuedJobs: testfixtures.N16Cpu128GiJobs(testfixtures.TestQueue, testfixtures.PriorityClass3, 10), @@ -239,8 +240,8 @@ func TestSchedule(t *testing.T) { "no queued jobs": { schedulingConfig: testfixtures.TestSchedulingConfig(), executors: []*schedulerobjects.Executor{ - testfixtures.Test1Node32CoreExecutor("executor1"), - testfixtures.Test1Node32CoreExecutor("executor2"), + test1Node32CoreExecutor("executor1"), + test1Node32CoreExecutor("executor2"), }, queues: []*api.Queue{testfixtures.MakeTestQueue()}, }, @@ -257,7 +258,7 @@ func TestSchedule(t *testing.T) { }, testfixtures.TestSchedulingConfig(), ), - executors: []*schedulerobjects.Executor{testfixtures.Test1Node32CoreExecutor("executor1")}, + executors: []*schedulerobjects.Executor{test1Node32CoreExecutor("executor1")}, queues: []*api.Queue{testfixtures.MakeTestQueue()}, queuedJobs: []*jobdb.Job{ // Submit the next job with a per-queue priority number (i.e., 1) that is larger @@ -278,7 +279,7 @@ func TestSchedule(t *testing.T) { }, "urgency-based preemption within a single queue": { schedulingConfig: testfixtures.TestSchedulingConfig(), - executors: []*schedulerobjects.Executor{testfixtures.Test1Node32CoreExecutor("executor1")}, + executors: []*schedulerobjects.Executor{test1Node32CoreExecutor("executor1")}, queues: []*api.Queue{{Name: "A"}}, queuedJobs: testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass1, 2), scheduledJobsByExecutorIndexAndNodeIndex: map[int]map[int]scheduledJobs{ @@ -298,7 +299,7 @@ func TestSchedule(t *testing.T) { }, "urgency-based preemption between queues": { schedulingConfig: testfixtures.TestSchedulingConfig(), - executors: []*schedulerobjects.Executor{testfixtures.Test1Node32CoreExecutor("executor1")}, + executors: []*schedulerobjects.Executor{test1Node32CoreExecutor("executor1")}, queues: []*api.Queue{{Name: "A"}, {Name: "B"}}, queuedJobs: testfixtures.N16Cpu128GiJobs("B", testfixtures.PriorityClass1, 2), scheduledJobsByExecutorIndexAndNodeIndex: map[int]map[int]scheduledJobs{ @@ -318,7 +319,7 @@ func TestSchedule(t *testing.T) { }, "preemption to fair share": { schedulingConfig: testfixtures.TestSchedulingConfig(), - executors: []*schedulerobjects.Executor{testfixtures.Test1Node32CoreExecutor("executor1")}, + executors: []*schedulerobjects.Executor{test1Node32CoreExecutor("executor1")}, queues: []*api.Queue{{Name: "A", PriorityFactor: 0.01}, {Name: "B", PriorityFactor: 0.01}}, queuedJobs: testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 2), scheduledJobsByExecutorIndexAndNodeIndex: map[int]map[int]scheduledJobs{ @@ -338,7 +339,7 @@ func TestSchedule(t *testing.T) { }, "gang scheduling successful": { schedulingConfig: testfixtures.TestSchedulingConfig(), - executors: []*schedulerobjects.Executor{testfixtures.Test1Node32CoreExecutor("executor1")}, + executors: []*schedulerobjects.Executor{test1Node32CoreExecutor("executor1")}, queues: []*api.Queue{{Name: "A", PriorityFactor: 0.01}}, queuedJobs: testfixtures.WithGangAnnotationsJobs(testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 2)), expectedScheduledIndices: []int{0, 1}, @@ -346,8 +347,8 @@ func TestSchedule(t *testing.T) { "gang scheduling successful - mixed pool clusters": { schedulingConfig: testfixtures.TestSchedulingConfigWithPools([]configuration.PoolConfig{{Name: "pool-1"}, {Name: "pool-2"}}), executors: []*schedulerobjects.Executor{ - testfixtures.MakeTestExecutor("executor1", "pool-1", "pool-2"), - testfixtures.MakeTestExecutor("executor2", "pool-1"), + makeTestExecutor("executor1", "pool-1", "pool-2"), + makeTestExecutor("executor2", "pool-1"), }, queues: []*api.Queue{{Name: "A", PriorityFactor: 0.01}}, queuedJobs: testfixtures.WithPools(testfixtures.WithNodeUniformityGangAnnotationsJobs(testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 3), testfixtures.PoolNameLabel), []string{"pool-1"}), @@ -357,22 +358,22 @@ func TestSchedule(t *testing.T) { "not scheduling a gang that does not fit on any executor": { schedulingConfig: testfixtures.TestSchedulingConfig(), executors: []*schedulerobjects.Executor{ - testfixtures.Test1Node32CoreExecutor("executor1"), - testfixtures.Test1Node32CoreExecutor("executor2"), + test1Node32CoreExecutor("executor1"), + test1Node32CoreExecutor("executor2"), }, queues: []*api.Queue{{Name: "A", PriorityFactor: 0.01}}, queuedJobs: testfixtures.WithNodeUniformityGangAnnotationsJobs(testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 3), testfixtures.ClusterNameLabel), }, "not scheduling a gang that does not fit on any pool": { schedulingConfig: testfixtures.TestSchedulingConfigWithPools([]configuration.PoolConfig{{Name: "pool-1"}, {Name: "pool-2"}}), - executors: []*schedulerobjects.Executor{testfixtures.MakeTestExecutor("executor1", "pool-1", "pool-2")}, + executors: []*schedulerobjects.Executor{makeTestExecutor("executor1", "pool-1", "pool-2")}, queues: []*api.Queue{{Name: "A", PriorityFactor: 0.01}}, queuedJobs: testfixtures.WithPools(testfixtures.WithNodeUniformityGangAnnotationsJobs(testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 3), testfixtures.ClusterNameLabel), []string{"pool-1", "pool-2"}), }, "urgency-based gang preemption": { schedulingConfig: testfixtures.TestSchedulingConfig(), executors: []*schedulerobjects.Executor{ - testfixtures.Test1Node32CoreExecutor("executor1"), + test1Node32CoreExecutor("executor1"), }, queues: []*api.Queue{{Name: "queue1", PriorityFactor: 0.01}, {Name: "queue2", PriorityFactor: 0.01}}, queuedJobs: testfixtures.N16Cpu128GiJobs("queue2", testfixtures.PriorityClass1, 1), @@ -393,7 +394,7 @@ func TestSchedule(t *testing.T) { }, "preemption to fair share evicting a gang": { schedulingConfig: testfixtures.TestSchedulingConfig(), - executors: []*schedulerobjects.Executor{testfixtures.Test1Node32CoreExecutor("executor1")}, + executors: []*schedulerobjects.Executor{test1Node32CoreExecutor("executor1")}, queues: []*api.Queue{{Name: "queue1", PriorityFactor: 0.01}, {Name: "queue2", PriorityFactor: 0.01}}, queuedJobs: testfixtures.N32Cpu256GiJobs("queue2", testfixtures.PriorityClass0, 1), scheduledJobsByExecutorIndexAndNodeIndex: map[int]map[int]scheduledJobs{ @@ -414,8 +415,8 @@ func TestSchedule(t *testing.T) { "Schedule gang job over multiple executors": { schedulingConfig: testfixtures.TestSchedulingConfig(), executors: []*schedulerobjects.Executor{ - testfixtures.Test1Node32CoreExecutor("executor1"), - testfixtures.Test1Node32CoreExecutor("executor2"), + test1Node32CoreExecutor("executor1"), + test1Node32CoreExecutor("executor2"), }, queues: []*api.Queue{testfixtures.MakeTestQueue()}, queuedJobs: testfixtures.WithGangAnnotationsJobs(testfixtures.N16Cpu128GiJobs(testfixtures.TestQueue, testfixtures.PriorityClass0, 4)), @@ -424,8 +425,8 @@ func TestSchedule(t *testing.T) { "scheduling from paused queue": { schedulingConfig: testfixtures.TestSchedulingConfig(), executors: []*schedulerobjects.Executor{ - testfixtures.Test1Node32CoreExecutor("executor1"), - testfixtures.Test1Node32CoreExecutor("executor2"), + test1Node32CoreExecutor("executor1"), + test1Node32CoreExecutor("executor2"), }, queues: []*api.Queue{testfixtures.MakeTestQueueCordoned()}, queuedJobs: testfixtures.N16Cpu128GiJobs(testfixtures.TestQueue, testfixtures.PriorityClass3, 10), @@ -434,8 +435,8 @@ func TestSchedule(t *testing.T) { "multi-queue scheduling": { schedulingConfig: testfixtures.TestSchedulingConfig(), executors: []*schedulerobjects.Executor{ - testfixtures.Test1Node32CoreExecutor("executor1"), - testfixtures.Test1Node32CoreExecutor("executor2"), + test1Node32CoreExecutor("executor1"), + test1Node32CoreExecutor("executor2"), }, queues: []*api.Queue{testfixtures.MakeTestQueue(), testfixtures.MakeTestQueue2()}, queuedJobs: append( @@ -447,8 +448,8 @@ func TestSchedule(t *testing.T) { "multi-queue scheduling with paused and non-paused queue": { schedulingConfig: testfixtures.TestSchedulingConfig(), executors: []*schedulerobjects.Executor{ - testfixtures.Test1Node32CoreExecutor("executor1"), - testfixtures.Test1Node32CoreExecutor("executor2"), + test1Node32CoreExecutor("executor1"), + test1Node32CoreExecutor("executor2"), }, queues: []*api.Queue{testfixtures.MakeTestQueueCordoned(), testfixtures.MakeTestQueue()}, queuedJobs: append( @@ -460,10 +461,10 @@ func TestSchedule(t *testing.T) { "multi-queue scheduling with paused and non-paused queue large": { schedulingConfig: testfixtures.TestSchedulingConfig(), executors: []*schedulerobjects.Executor{ - testfixtures.Test1Node32CoreExecutor("executor1"), - testfixtures.Test1Node32CoreExecutor("executor2"), - testfixtures.Test1Node32CoreExecutor("executor3"), - testfixtures.Test1Node32CoreExecutor("executor4"), + test1Node32CoreExecutor("executor1"), + test1Node32CoreExecutor("executor2"), + test1Node32CoreExecutor("executor3"), + test1Node32CoreExecutor("executor4"), }, queues: []*api.Queue{testfixtures.MakeTestQueueCordoned(), testfixtures.MakeTestQueue()}, queuedJobs: append( @@ -719,3 +720,77 @@ func makeResourceList(resourceName string, value string) internaltypes.ResourceL }, ) } + +func makeTestExecutorWithNodes(executorId string, nodes ...*schedulerobjects.Node) *schedulerobjects.Executor { + for _, node := range nodes { + node.Name = fmt.Sprintf("%s-node", executorId) + node.Executor = executorId + } + + return &schedulerobjects.Executor{ + Id: executorId, + Pool: testfixtures.TestPool, + Nodes: nodes, + LastUpdateTime: testfixtures.BaseTime, + } +} + +func test1Node32CoreExecutor(executorId string) *schedulerobjects.Executor { + node := test32CpuNode(testfixtures.TestPriorities) + node.Name = fmt.Sprintf("%s-node", executorId) + node.Executor = executorId + node.Labels[testfixtures.ClusterNameLabel] = executorId + return &schedulerobjects.Executor{ + Id: executorId, + Pool: testfixtures.TestPool, + Nodes: []*schedulerobjects.Node{node}, + LastUpdateTime: testfixtures.BaseTime, + } +} + +func makeTestExecutor(executorId string, nodePools ...string) *schedulerobjects.Executor { + nodes := []*schedulerobjects.Node{} + + for _, nodePool := range nodePools { + node := test32CpuNode(testfixtures.TestPriorities) + node.Name = fmt.Sprintf("%s-node", executorId) + node.Executor = executorId + node.Pool = nodePool + node.Labels[testfixtures.PoolNameLabel] = nodePool + nodes = append(nodes, node) + } + + return &schedulerobjects.Executor{ + Id: executorId, + Pool: testfixtures.TestPool, + Nodes: nodes, + LastUpdateTime: testfixtures.BaseTime, + } +} + +func withLastUpdateTimeExecutor(lastUpdateTime time.Time, executor *schedulerobjects.Executor) *schedulerobjects.Executor { + executor.LastUpdateTime = lastUpdateTime + return executor +} + +func testNodeWithPool(pool string) *schedulerobjects.Node { + node := test32CpuNode(testfixtures.TestPriorities) + node.Pool = pool + node.Labels[testfixtures.PoolNameLabel] = pool + return node +} + +func withLargeNodeTaint(node *schedulerobjects.Node) *schedulerobjects.Node { + node.Taints = append(node.Taints, v1.Taint{Key: "largeJobsOnly", Value: "true", Effect: v1.TaintEffectNoSchedule}) + return node +} + +func test32CpuNode(priorities []int32) *schedulerobjects.Node { + return testfixtures.TestSchedulerObjectsNode( + priorities, + map[string]k8sResource.Quantity{ + "cpu": k8sResource.MustParse("32"), + "memory": k8sResource.MustParse("256Gi"), + }, + ) +} diff --git a/internal/scheduler/submitcheck_test.go b/internal/scheduler/submitcheck_test.go index d5d355a9e5d..71d0bf6b6cc 100644 --- a/internal/scheduler/submitcheck_test.go +++ b/internal/scheduler/submitcheck_test.go @@ -329,7 +329,7 @@ func Executor(nodes ...*schedulerobjects.Node) *schedulerobjects.Executor { } func GpuNode(pool string) *schedulerobjects.Node { - node := testfixtures.TestNode( + node := testfixtures.TestSchedulerObjectsNode( testfixtures.TestPriorities, map[string]resource.Quantity{ "cpu": resource.MustParse("30"), @@ -348,7 +348,7 @@ func GpuNode(pool string) *schedulerobjects.Node { } func SmallNode(pool string) *schedulerobjects.Node { - node := testfixtures.TestNode( + node := testfixtures.TestSchedulerObjectsNode( testfixtures.TestPriorities, map[string]resource.Quantity{ "cpu": resource.MustParse("2"), diff --git a/internal/scheduler/testfixtures/testfixtures.go b/internal/scheduler/testfixtures/testfixtures.go index 10469f318fc..12d4c2d164a 100644 --- a/internal/scheduler/testfixtures/testfixtures.go +++ b/internal/scheduler/testfixtures/testfixtures.go @@ -700,65 +700,6 @@ func TestUnitReqs() *schedulerobjects.PodRequirements { } } -func TestCluster() []*schedulerobjects.Node { - return []*schedulerobjects.Node{ - { - Id: "node1", - Pool: TestPool, - AllocatableByPriorityAndResource: map[int32]schedulerobjects.ResourceList{ - 0: {Resources: map[string]resource.Quantity{"cpu": resource.MustParse("1"), "memory": resource.MustParse("1Gi")}}, - 1: {Resources: map[string]resource.Quantity{"cpu": resource.MustParse("2"), "memory": resource.MustParse("2Gi")}}, - 2: {Resources: map[string]resource.Quantity{"cpu": resource.MustParse("3"), "memory": resource.MustParse("3Gi")}}, - }, - TotalResources: schedulerobjects.ResourceList{ - Resources: map[string]resource.Quantity{ - "cpu": resource.MustParse("3"), - "memory": resource.MustParse("3Gi"), - }, - }, - Labels: map[string]string{ - TestHostnameLabel: "node1", - }, - }, - { - Id: "node2", - Pool: TestPool, - AllocatableByPriorityAndResource: map[int32]schedulerobjects.ResourceList{ - 0: {Resources: map[string]resource.Quantity{"cpu": resource.MustParse("4"), "memory": resource.MustParse("4Gi")}}, - 1: {Resources: map[string]resource.Quantity{"cpu": resource.MustParse("5"), "memory": resource.MustParse("5Gi")}}, - 2: {Resources: map[string]resource.Quantity{"cpu": resource.MustParse("6"), "memory": resource.MustParse("6Gi")}}, - }, - TotalResources: schedulerobjects.ResourceList{ - Resources: map[string]resource.Quantity{ - "cpu": resource.MustParse("6"), - "memory": resource.MustParse("6Gi"), - }, - }, - Labels: map[string]string{ - TestHostnameLabel: "node2", - }, - }, - { - Id: "node3", - Pool: TestPool, - AllocatableByPriorityAndResource: map[int32]schedulerobjects.ResourceList{ - 0: {Resources: map[string]resource.Quantity{"cpu": resource.MustParse("7"), "memory": resource.MustParse("7Gi")}}, - 1: {Resources: map[string]resource.Quantity{"cpu": resource.MustParse("8"), "memory": resource.MustParse("8Gi")}}, - 2: {Resources: map[string]resource.Quantity{"cpu": resource.MustParse("9"), "memory": resource.MustParse("9Gi")}}, - }, - TotalResources: schedulerobjects.ResourceList{ - Resources: map[string]resource.Quantity{ - "cpu": resource.MustParse("9"), - "memory": resource.MustParse("9Gi"), - }, - }, - Labels: map[string]string{ - TestHostnameLabel: "node3", - }, - }, - } -} - func ItN32CpuNodes(n int, priorities []int32) []*internaltypes.Node { rv := make([]*internaltypes.Node, n) for i := 0; i < n; i++ { @@ -787,7 +728,7 @@ func SingleQueuePriorityOne(name string) []*api.Queue { return []*api.Queue{{Name: name, PriorityFactor: 1.0}} } -func TestNode(priorities []int32, resources map[string]resource.Quantity) *schedulerobjects.Node { +func TestSchedulerObjectsNode(priorities []int32, resources map[string]resource.Quantity) *schedulerobjects.Node { id := uuid.NewString() return &schedulerobjects.Node{ Id: id, @@ -807,6 +748,25 @@ func TestNode(priorities []int32, resources map[string]resource.Quantity) *sched } } +func TestSimpleNode(id string) *internaltypes.Node { + return internaltypes.CreateNode( + id, + nil, + 0, + "", + "", + "", + nil, + nil, + internaltypes.ResourceList{}, + nil, + nil, + nil, + nil, + nil, + nil) +} + func ItTestNode(priorities []int32, resources map[string]resource.Quantity) *internaltypes.Node { rl := TestNodeFactory.ResourceListFactory().FromNodeProto(resources) id := uuid.NewString() @@ -825,16 +785,6 @@ func ItTestNode(priorities []int32, resources map[string]resource.Quantity) *int internaltypes.NewAllocatableByPriorityAndResourceType(priorities, rl)) } -func Test32CpuNode(priorities []int32) *schedulerobjects.Node { - return TestNode( - priorities, - map[string]resource.Quantity{ - "cpu": resource.MustParse("32"), - "memory": resource.MustParse("256Gi"), - }, - ) -} - func ItTest32CpuNode(priorities []int32) *internaltypes.Node { return ItTestNode( priorities, @@ -845,19 +795,6 @@ func ItTest32CpuNode(priorities []int32) *internaltypes.Node { ) } -func TestTainted32CpuNode(priorities []int32) *schedulerobjects.Node { - node := Test32CpuNode(priorities) - node.Taints = []v1.Taint{ - { - Key: "largeJobsOnly", - Value: "true", - Effect: v1.TaintEffectNoSchedule, - }, - } - node.Labels["largeJobsOnly"] = "true" - return node -} - func ItTestTainted32CpuNode(priorities []int32) *internaltypes.Node { node := ItTest32CpuNode(priorities) @@ -878,19 +815,6 @@ func ItTestTainted32CpuNode(priorities []int32) *internaltypes.Node { return node } -func Test8GpuNode(priorities []int32) *schedulerobjects.Node { - node := TestNode( - priorities, - map[string]resource.Quantity{ - "cpu": resource.MustParse("64"), - "memory": resource.MustParse("1024Gi"), - "nvidia.com/gpu": resource.MustParse("8"), - }, - ) - node.Labels["gpu"] = "true" - return node -} - func ItTest8GpuNode(priorities []int32) *internaltypes.Node { node := ItTestNode( priorities, @@ -906,70 +830,6 @@ func ItTest8GpuNode(priorities []int32) *internaltypes.Node { )[0] } -func WithLastUpdateTimeExecutor(lastUpdateTime time.Time, executor *schedulerobjects.Executor) *schedulerobjects.Executor { - executor.LastUpdateTime = lastUpdateTime - return executor -} - -func TestNodeWithPool(pool string) *schedulerobjects.Node { - node := Test32CpuNode(TestPriorities) - node.Pool = pool - node.Labels[PoolNameLabel] = pool - return node -} - -func WithLargeNodeTaint(node *schedulerobjects.Node) *schedulerobjects.Node { - node.Taints = append(node.Taints, v1.Taint{Key: "largeJobsOnly", Value: "true", Effect: v1.TaintEffectNoSchedule}) - return node -} - -func MakeTestExecutorWithNodes(executorId string, nodes ...*schedulerobjects.Node) *schedulerobjects.Executor { - for _, node := range nodes { - node.Name = fmt.Sprintf("%s-node", executorId) - node.Executor = executorId - } - - return &schedulerobjects.Executor{ - Id: executorId, - Pool: TestPool, - Nodes: nodes, - LastUpdateTime: BaseTime, - } -} - -func Test1Node32CoreExecutor(executorId string) *schedulerobjects.Executor { - node := Test32CpuNode(TestPriorities) - node.Name = fmt.Sprintf("%s-node", executorId) - node.Executor = executorId - node.Labels[ClusterNameLabel] = executorId - return &schedulerobjects.Executor{ - Id: executorId, - Pool: TestPool, - Nodes: []*schedulerobjects.Node{node}, - LastUpdateTime: BaseTime, - } -} - -func MakeTestExecutor(executorId string, nodePools ...string) *schedulerobjects.Executor { - nodes := []*schedulerobjects.Node{} - - for _, nodePool := range nodePools { - node := Test32CpuNode(TestPriorities) - node.Name = fmt.Sprintf("%s-node", executorId) - node.Executor = executorId - node.Pool = nodePool - node.Labels[PoolNameLabel] = nodePool - nodes = append(nodes, node) - } - - return &schedulerobjects.Executor{ - Id: executorId, - Pool: TestPool, - Nodes: nodes, - LastUpdateTime: BaseTime, - } -} - func MakeTestQueue() *api.Queue { return &api.Queue{ Name: TestQueue, From 6899a5fa4d2250dd5deb4a9456570da6bc51e9fa Mon Sep 17 00:00:00 2001 From: Rob Smith <34475852+robertdavidsmith@users.noreply.github.com> Date: Fri, 3 Jan 2025 15:16:52 +0000 Subject: [PATCH 06/17] Scheduler: tidy queue_scheduler_test (#4117) Signed-off-by: Robert Smith --- .../internaltypes/resource_list_map_util.go | 10 ------ .../resource_list_map_util_test.go | 32 ------------------- .../scheduling/queue_scheduler_test.go | 29 ++++------------- 3 files changed, 6 insertions(+), 65 deletions(-) diff --git a/internal/scheduler/internaltypes/resource_list_map_util.go b/internal/scheduler/internaltypes/resource_list_map_util.go index a26a1a80f6d..443f172bd64 100644 --- a/internal/scheduler/internaltypes/resource_list_map_util.go +++ b/internal/scheduler/internaltypes/resource_list_map_util.go @@ -2,8 +2,6 @@ package internaltypes import ( "strings" - - "github.com/armadaproject/armada/internal/scheduler/schedulerobjects" ) func RlMapToString(m map[string]ResourceList) string { @@ -40,14 +38,6 @@ func RlMapHasNegativeValues(m map[string]ResourceList) bool { return false } -func RlMapFromJobSchedulerObjects(m schedulerobjects.QuantityByTAndResourceType[string], rlFactory *ResourceListFactory) map[string]ResourceList { - result := map[string]ResourceList{} - for k, v := range m { - result[k] = rlFactory.FromJobResourceListIgnoreUnknown(v.Resources) - } - return result -} - func RlMapRemoveZeros(m map[string]ResourceList) map[string]ResourceList { result := map[string]ResourceList{} for k, v := range m { diff --git a/internal/scheduler/internaltypes/resource_list_map_util_test.go b/internal/scheduler/internaltypes/resource_list_map_util_test.go index 5e2b61f729f..7f6fb74b9b6 100644 --- a/internal/scheduler/internaltypes/resource_list_map_util_test.go +++ b/internal/scheduler/internaltypes/resource_list_map_util_test.go @@ -4,9 +4,6 @@ import ( "testing" "github.com/stretchr/testify/assert" - "k8s.io/apimachinery/pkg/api/resource" - - "github.com/armadaproject/armada/internal/scheduler/schedulerobjects" ) func TestRlMapSumValues(t *testing.T) { @@ -36,35 +33,6 @@ func TestRlMapHasNegativeValues(t *testing.T) { assert.False(t, RlMapHasNegativeValues(testMapEmpty(factory))) } -func TestRlMapFromJobSchedulerObjects(t *testing.T) { - factory := testFactory() - - input := make(schedulerobjects.QuantityByTAndResourceType[string]) - input.AddResourceList("priorityClass1", - schedulerobjects.ResourceList{ - Resources: map[string]resource.Quantity{ - "cpu": resource.MustParse("1"), - "memory": resource.MustParse("1Ki"), - }, - }, - ) - input.AddResourceList("priorityClass2", - schedulerobjects.ResourceList{ - Resources: map[string]resource.Quantity{ - "cpu": resource.MustParse("2"), - "memory": resource.MustParse("2Ki"), - }, - }, - ) - - expected := map[string]ResourceList{ - "priorityClass1": testResourceList(factory, "1", "1Ki"), - "priorityClass2": testResourceList(factory, "2", "2Ki"), - } - - assert.Equal(t, expected, RlMapFromJobSchedulerObjects(input, factory)) -} - func TestRlMapRemoveZeros(t *testing.T) { factory := testFactory() diff --git a/internal/scheduler/scheduling/queue_scheduler_test.go b/internal/scheduler/scheduling/queue_scheduler_test.go index 36b23d6a51b..9aaca4cdb78 100644 --- a/internal/scheduler/scheduling/queue_scheduler_test.go +++ b/internal/scheduler/scheduling/queue_scheduler_test.go @@ -11,7 +11,6 @@ import ( "golang.org/x/exp/slices" "golang.org/x/time/rate" v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" "github.com/armadaproject/armada/internal/common/armadacontext" armadaslices "github.com/armadaproject/armada/internal/common/slices" @@ -20,7 +19,6 @@ import ( "github.com/armadaproject/armada/internal/scheduler/internaltypes" "github.com/armadaproject/armada/internal/scheduler/jobdb" "github.com/armadaproject/armada/internal/scheduler/nodedb" - "github.com/armadaproject/armada/internal/scheduler/schedulerobjects" schedulerconstraints "github.com/armadaproject/armada/internal/scheduler/scheduling/constraints" "github.com/armadaproject/armada/internal/scheduler/scheduling/context" "github.com/armadaproject/armada/internal/scheduler/scheduling/fairness" @@ -32,13 +30,10 @@ import ( func TestQueueScheduler(t *testing.T) { tests := map[string]struct { SchedulingConfig configuration.SchedulingConfig - // Total resources across all clusters. - // Set to the total resources across all nodes if not provided. - TotalResources schedulerobjects.ResourceList // Queues Queues []*api.Queue // Initial resource usage for all queues. - InitialAllocatedByQueueAndPriorityClass map[string]schedulerobjects.QuantityByTAndResourceType[string] + InitialAllocatedByQueueAndPriorityClass map[string]map[string]internaltypes.ResourceList // Nodes to be considered by the scheduler. Nodes []*internaltypes.Node // Jobs to try scheduling. @@ -263,13 +258,9 @@ func TestQueueScheduler(t *testing.T) { testfixtures.N1Cpu4GiJobs("B", testfixtures.PriorityClass0, 32), ), Queues: []*api.Queue{{Name: "A", PriorityFactor: 1.0}, {Name: "B", PriorityFactor: 1.0}}, - InitialAllocatedByQueueAndPriorityClass: map[string]schedulerobjects.QuantityByTAndResourceType[string]{ + InitialAllocatedByQueueAndPriorityClass: map[string]map[string]internaltypes.ResourceList{ "A": { - testfixtures.PriorityClass0: schedulerobjects.ResourceList{ - Resources: map[string]resource.Quantity{ - "cpu": resource.MustParse("100"), - }, - }, + testfixtures.PriorityClass0: testfixtures.Cpu("100"), }, }, ExpectedScheduledIndices: testfixtures.IntRange(32, 63), @@ -476,12 +467,8 @@ func TestQueueScheduler(t *testing.T) { require.NoError(t, err) } txn.Commit() - if tc.TotalResources.Resources == nil { - // Default to NodeDb total. - tc.TotalResources = schedulerobjects.ResourceList{ - Resources: nodeDb.TotalKubernetesResources().ToMap(), - } - } + + totalResources := nodeDb.TotalKubernetesResources() queueNameToQueue := map[string]*api.Queue{} for _, q := range tc.Queues { @@ -500,7 +487,6 @@ func TestQueueScheduler(t *testing.T) { context.JobSchedulingContextsFromJobs(tc.Jobs), ) - totalResources := testfixtures.TestResourceListFactory.FromJobResourceListIgnoreUnknown(tc.TotalResources.Resources) fairnessCostProvider, err := fairness.NewDominantResourceFairness( totalResources, tc.SchedulingConfig, @@ -519,10 +505,7 @@ func TestQueueScheduler(t *testing.T) { weight := 1.0 / float64(q.PriorityFactor) err := sctx.AddQueueSchedulingContext( q.Name, weight, - internaltypes.RlMapFromJobSchedulerObjects( - tc.InitialAllocatedByQueueAndPriorityClass[q.Name], - testfixtures.TestResourceListFactory, - ), + tc.InitialAllocatedByQueueAndPriorityClass[q.Name], internaltypes.ResourceList{}, internaltypes.ResourceList{}, rate.NewLimiter( From 26c69c67f031afb3e4f64a8218328b88d037a15f Mon Sep 17 00:00:00 2001 From: Rob Smith <34475852+robertdavidsmith@users.noreply.github.com> Date: Fri, 3 Jan 2025 15:46:23 +0000 Subject: [PATCH 07/17] Scheduler: testfixture method renaming (#4119) Signed-off-by: Robert Smith --- internal/scheduler/nodedb/nodedb_test.go | 82 ++--- .../scheduler/nodedb/nodeiteration_test.go | 318 +++++++++--------- .../scheduling/gang_scheduler_test.go | 60 ++-- ..._driven_preempting_queue_scheduler_test.go | 16 +- .../preempting_queue_scheduler_test.go | 122 +++---- .../scheduling/queue_scheduler_test.go | 82 ++--- .../scheduling/scheduling_algo_test.go | 2 +- .../scheduler/testfixtures/testfixtures.go | 34 +- 8 files changed, 358 insertions(+), 358 deletions(-) diff --git a/internal/scheduler/nodedb/nodedb_test.go b/internal/scheduler/nodedb/nodedb_test.go index 3a913ad834d..368fdae6277 100644 --- a/internal/scheduler/nodedb/nodedb_test.go +++ b/internal/scheduler/nodedb/nodedb_test.go @@ -35,7 +35,7 @@ func TestTotalResources(t *testing.T) { expected := testfixtures.TestNodeFactory.ResourceListFactory().MakeAllZero() // Upserting nodes for the first time should increase the resource count. - nodes := testfixtures.ItN32CpuNodes(2, testfixtures.TestPriorities) + nodes := testfixtures.N32CpuNodes(2, testfixtures.TestPriorities) for _, node := range nodes { expected = expected.Add(node.GetTotalResources()) } @@ -49,7 +49,7 @@ func TestTotalResources(t *testing.T) { assert.True(t, expected.Equal(nodeDb.TotalKubernetesResources())) // Upserting new nodes should increase the resource count. - nodes = testfixtures.ItN8GpuNodes(3, testfixtures.TestPriorities) + nodes = testfixtures.N8GpuNodes(3, testfixtures.TestPriorities) for _, node := range nodes { expected = expected.Add(node.GetTotalResources()) } @@ -64,7 +64,7 @@ func TestTotalResources(t *testing.T) { } func TestSelectNodeForPod_NodeIdLabel_Success(t *testing.T) { - nodes := testfixtures.ItN32CpuNodes(2, testfixtures.TestPriorities) + nodes := testfixtures.N32CpuNodes(2, testfixtures.TestPriorities) nodeId := nodes[1].GetId() require.NotEmpty(t, nodeId) db, err := newNodeDbWithNodes(nodes) @@ -89,7 +89,7 @@ func TestSelectNodeForPod_NodeIdLabel_Success(t *testing.T) { } func TestSelectNodeForPod_NodeIdLabel_Failure(t *testing.T) { - nodes := testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities) + nodes := testfixtures.N32CpuNodes(1, testfixtures.TestPriorities) nodeId := nodes[0].GetId() require.NotEmpty(t, nodeId) db, err := newNodeDbWithNodes(nodes) @@ -114,7 +114,7 @@ func TestSelectNodeForPod_NodeIdLabel_Failure(t *testing.T) { } func TestNodeBindingEvictionUnbinding(t *testing.T) { - node := testfixtures.ItTest8GpuNode(testfixtures.TestPriorities) + node := testfixtures.Test8GpuNode(testfixtures.TestPriorities) nodeDb, err := newNodeDbWithNodes([]*internaltypes.Node{node}) require.NoError(t, err) entry, err := nodeDb.GetNode(node.GetId()) @@ -278,7 +278,7 @@ func TestEviction(t *testing.T) { testfixtures.Test1Cpu4GiJob("queue-alice", testfixtures.PriorityClass0), testfixtures.Test1Cpu4GiJob("queue-alice", testfixtures.PriorityClass3), } - node := testfixtures.ItTest32CpuNode(testfixtures.TestPriorities) + node := testfixtures.Test32CpuNode(testfixtures.TestPriorities) require.NoError(t, err) err = nodeDb.CreateAndInsertWithJobDbJobsWithTxn(txn, jobs, node) txn.Commit() @@ -308,22 +308,22 @@ func TestScheduleIndividually(t *testing.T) { ExpectSuccess []bool }{ "all jobs fit": { - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Jobs: testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 32), ExpectSuccess: testfixtures.Repeat(true, 32), }, "not all jobs fit": { - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Jobs: testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 33), ExpectSuccess: append(testfixtures.Repeat(true, 32), testfixtures.Repeat(false, 1)...), }, "unavailable resource": { - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Jobs: testfixtures.N1GpuJobs("A", testfixtures.PriorityClass0, 1), ExpectSuccess: testfixtures.Repeat(false, 1), }, "unsupported resource": { - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Jobs: testfixtures.WithRequestsJobs( schedulerobjects.ResourceList{ Resources: map[string]resource.Quantity{ @@ -335,7 +335,7 @@ func TestScheduleIndividually(t *testing.T) { ExpectSuccess: testfixtures.Repeat(true, 1), // we ignore unknown resource types on jobs, should never happen in practice anyway as these should fail earlier. }, "preemption": { - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Jobs: append( append( testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 32), @@ -346,7 +346,7 @@ func TestScheduleIndividually(t *testing.T) { ExpectSuccess: append(testfixtures.Repeat(true, 64), testfixtures.Repeat(false, 32)...), }, "taints/tolerations": { - Nodes: testfixtures.ItNTainted32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.NTainted32CpuNodes(1, testfixtures.TestPriorities), Jobs: append( append( testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1), @@ -357,9 +357,9 @@ func TestScheduleIndividually(t *testing.T) { ExpectSuccess: []bool{false, false, true}, }, "node selector": { - Nodes: append(testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: append(testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), testfixtures.TestNodeFactory.AddLabels( - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), map[string]string{ "key": "value", }, @@ -374,7 +374,7 @@ func TestScheduleIndividually(t *testing.T) { }, "node selector with mismatched value": { Nodes: testfixtures.TestNodeFactory.AddLabels( - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), map[string]string{ "key": "value", }, @@ -388,7 +388,7 @@ func TestScheduleIndividually(t *testing.T) { ExpectSuccess: testfixtures.Repeat(false, 1), }, "node selector with missing label": { - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Jobs: testfixtures.WithNodeSelectorJobs( map[string]string{ "this label does not exist": "value", @@ -399,9 +399,9 @@ func TestScheduleIndividually(t *testing.T) { }, "node affinity": { Nodes: append( - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), testfixtures.TestNodeFactory.AddLabels( - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), map[string]string{ "key": "value", }, @@ -482,18 +482,18 @@ func TestScheduleMany(t *testing.T) { }{ // Attempts to schedule 32. All jobs get scheduled. "simple success": { - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Jobs: [][]*jobdb.Job{gangSuccess}, ExpectSuccess: []bool{true}, }, // Attempts to schedule 33 jobs. The overall result fails. "simple failure with min cardinality": { - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Jobs: [][]*jobdb.Job{gangFailure}, ExpectSuccess: []bool{false}, }, "correct rollback": { - Nodes: testfixtures.ItN32CpuNodes(2, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(2, testfixtures.TestPriorities), Jobs: [][]*jobdb.Job{ gangSuccess, gangFailure, @@ -502,7 +502,7 @@ func TestScheduleMany(t *testing.T) { ExpectSuccess: []bool{true, false, true}, }, "varying job size": { - Nodes: testfixtures.ItN32CpuNodes(2, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(2, testfixtures.TestPriorities), Jobs: [][]*jobdb.Job{ append( testfixtures.N32Cpu256GiJobs("A", testfixtures.PriorityClass0, 1), @@ -556,7 +556,7 @@ func TestAwayNodeTypes(t *testing.T) { require.NoError(t, err) nodeDbTxn := nodeDb.Txn(true) - node := testfixtures.ItTest32CpuNode([]int32{29000, 30000}) + node := testfixtures.Test32CpuNode([]int32{29000, 30000}) node = testfixtures.TestNodeFactory.AddTaints( []*internaltypes.Node{node}, []v1.Taint{ @@ -740,15 +740,15 @@ func benchmarkUpsert(nodes []*internaltypes.Node, b *testing.B) { } func BenchmarkUpsert1(b *testing.B) { - benchmarkUpsert(testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), b) + benchmarkUpsert(testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), b) } func BenchmarkUpsert1000(b *testing.B) { - benchmarkUpsert(testfixtures.ItN32CpuNodes(1000, testfixtures.TestPriorities), b) + benchmarkUpsert(testfixtures.N32CpuNodes(1000, testfixtures.TestPriorities), b) } func BenchmarkUpsert100000(b *testing.B) { - benchmarkUpsert(testfixtures.ItN32CpuNodes(100000, testfixtures.TestPriorities), b) + benchmarkUpsert(testfixtures.N32CpuNodes(100000, testfixtures.TestPriorities), b) } func benchmarkScheduleMany(b *testing.B, nodes []*internaltypes.Node, jobs []*jobdb.Job) { @@ -782,7 +782,7 @@ func benchmarkScheduleMany(b *testing.B, nodes []*internaltypes.Node, jobs []*jo func BenchmarkScheduleMany10CpuNodes320SmallJobs(b *testing.B) { benchmarkScheduleMany( b, - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 320), ) } @@ -790,7 +790,7 @@ func BenchmarkScheduleMany10CpuNodes320SmallJobs(b *testing.B) { func BenchmarkScheduleMany10CpuNodes640SmallJobs(b *testing.B) { benchmarkScheduleMany( b, - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 640), ) } @@ -798,7 +798,7 @@ func BenchmarkScheduleMany10CpuNodes640SmallJobs(b *testing.B) { func BenchmarkScheduleMany100CpuNodes3200SmallJobs(b *testing.B) { benchmarkScheduleMany( b, - testfixtures.ItN32CpuNodes(100, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(100, testfixtures.TestPriorities), testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 3200), ) } @@ -806,7 +806,7 @@ func BenchmarkScheduleMany100CpuNodes3200SmallJobs(b *testing.B) { func BenchmarkScheduleMany100CpuNodes6400SmallJobs(b *testing.B) { benchmarkScheduleMany( b, - testfixtures.ItN32CpuNodes(100, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(100, testfixtures.TestPriorities), testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 6400), ) } @@ -814,7 +814,7 @@ func BenchmarkScheduleMany100CpuNodes6400SmallJobs(b *testing.B) { func BenchmarkScheduleMany1000CpuNodes32000SmallJobs(b *testing.B) { benchmarkScheduleMany( b, - testfixtures.ItN32CpuNodes(1000, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1000, testfixtures.TestPriorities), testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 32000), ) } @@ -822,7 +822,7 @@ func BenchmarkScheduleMany1000CpuNodes32000SmallJobs(b *testing.B) { func BenchmarkScheduleMany1000CpuNodes64000SmallJobs(b *testing.B) { benchmarkScheduleMany( b, - testfixtures.ItN32CpuNodes(1000, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1000, testfixtures.TestPriorities), testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 64000), ) } @@ -830,10 +830,10 @@ func BenchmarkScheduleMany1000CpuNodes64000SmallJobs(b *testing.B) { func BenchmarkScheduleMany100CpuNodes1CpuUnused(b *testing.B) { benchmarkScheduleMany( b, - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 0, testfixtures.Cpu("31"), - testfixtures.ItN32CpuNodes(100, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(100, testfixtures.TestPriorities), ), testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 100), ) @@ -842,10 +842,10 @@ func BenchmarkScheduleMany100CpuNodes1CpuUnused(b *testing.B) { func BenchmarkScheduleMany1000CpuNodes1CpuUnused(b *testing.B) { benchmarkScheduleMany( b, - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 0, testfixtures.Cpu("31"), - testfixtures.ItN32CpuNodes(1000, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1000, testfixtures.TestPriorities), ), testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1000), ) @@ -854,10 +854,10 @@ func BenchmarkScheduleMany1000CpuNodes1CpuUnused(b *testing.B) { func BenchmarkScheduleMany10000CpuNodes1CpuUnused(b *testing.B) { benchmarkScheduleMany( b, - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 0, testfixtures.Cpu("31"), - testfixtures.ItN32CpuNodes(10000, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(10000, testfixtures.TestPriorities), ), testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 10000), ) @@ -865,9 +865,9 @@ func BenchmarkScheduleMany10000CpuNodes1CpuUnused(b *testing.B) { func BenchmarkScheduleManyResourceConstrained(b *testing.B) { nodes := append(append( - testfixtures.ItN32CpuNodes(500, testfixtures.TestPriorities), - testfixtures.ItN8GpuNodes(1, testfixtures.TestPriorities)...), - testfixtures.ItN32CpuNodes(499, testfixtures.TestPriorities)..., + testfixtures.N32CpuNodes(500, testfixtures.TestPriorities), + testfixtures.N8GpuNodes(1, testfixtures.TestPriorities)...), + testfixtures.N32CpuNodes(499, testfixtures.TestPriorities)..., ) benchmarkScheduleMany( b, diff --git a/internal/scheduler/nodedb/nodeiteration_test.go b/internal/scheduler/nodedb/nodeiteration_test.go index 27f9d901329..f70b1dce23a 100644 --- a/internal/scheduler/nodedb/nodeiteration_test.go +++ b/internal/scheduler/nodedb/nodeiteration_test.go @@ -23,13 +23,13 @@ func TestNodesIterator(t *testing.T) { Nodes []*internaltypes.Node }{ "1 node": { - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), }, "0 nodes": { - Nodes: testfixtures.ItN32CpuNodes(0, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(0, testfixtures.TestPriorities), }, "3 nodes": { - Nodes: testfixtures.ItN32CpuNodes(3, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(3, testfixtures.TestPriorities), }, } for name, tc := range tests { @@ -85,17 +85,17 @@ func TestNodeTypeIterator(t *testing.T) { }{ "only yield nodes of the right nodeType": { nodes: armadaslices.Concatenate( - testfixtures.ItWithNodeTypeNodes( + testfixtures.WithNodeTypeNodes( nodeTypeA, - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), ), - testfixtures.ItWithNodeTypeNodes( + testfixtures.WithNodeTypeNodes( nodeTypeB, - testfixtures.ItN32CpuNodes(2, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(2, testfixtures.TestPriorities), ), - testfixtures.ItWithNodeTypeNodes( + testfixtures.WithNodeTypeNodes( nodeTypeA, - testfixtures.ItN32CpuNodes(3, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(3, testfixtures.TestPriorities), ), ), nodeTypeId: nodeTypeA.GetId(), @@ -107,23 +107,23 @@ func TestNodeTypeIterator(t *testing.T) { ), }, "filter nodes with insufficient resources and return in increasing order": { - nodes: testfixtures.ItWithNodeTypeNodes( + nodes: testfixtures.WithNodeTypeNodes( nodeTypeA, armadaslices.Concatenate( - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 0, testfixtures.Cpu("15"), - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), ), - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 0, testfixtures.Cpu("16"), - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), ), - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 0, testfixtures.Cpu("17"), - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), ), ), ), @@ -133,53 +133,53 @@ func TestNodeTypeIterator(t *testing.T) { expected: []int{1, 0}, }, "filter nodes with insufficient resources at priority and return in increasing order": { - nodes: testfixtures.ItWithNodeTypeNodes( + nodes: testfixtures.WithNodeTypeNodes( nodeTypeA, armadaslices.Concatenate( - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 0, testfixtures.Cpu("15"), - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), ), - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 0, testfixtures.Cpu("16"), - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), ), - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 0, testfixtures.Cpu("17"), - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), ), - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 1, testfixtures.Cpu("15"), - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), ), - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 1, testfixtures.Cpu("16"), - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), ), - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 1, testfixtures.Cpu("17"), - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), ), - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 2, testfixtures.Cpu("15"), - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), ), - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 2, testfixtures.Cpu("16"), - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), ), - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 2, testfixtures.Cpu("17"), - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), ), ), ), @@ -189,53 +189,53 @@ func TestNodeTypeIterator(t *testing.T) { expected: []int{4, 7, 3, 6, 0, 1, 2}, }, "nested ordering": { - nodes: testfixtures.ItWithNodeTypeNodes( + nodes: testfixtures.WithNodeTypeNodes( nodeTypeA, armadaslices.Concatenate( - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 0, testfixtures.CpuMem("15", "1Gi"), - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), ), - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 0, testfixtures.CpuMem("15", "2Gi"), - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), ), - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 0, testfixtures.CpuMem("15", "129Gi"), - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), ), - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 0, testfixtures.CpuMem("15", "130Gi"), - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), ), - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 0, testfixtures.CpuMem("15", "131Gi"), - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), ), - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 0, testfixtures.CpuMem("16", "130Gi"), - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), ), - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 0, testfixtures.CpuMem("16", "128Gi"), - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), ), - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 0, testfixtures.CpuMem("16", "129Gi"), - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), ), - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 0, testfixtures.Cpu("17"), - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), ), ), ), @@ -245,58 +245,58 @@ func TestNodeTypeIterator(t *testing.T) { expected: []int{6, 1, 0}, }, "double-nested ordering": { - nodes: testfixtures.ItWithNodeTypeNodes( + nodes: testfixtures.WithNodeTypeNodes( nodeTypeA, armadaslices.Concatenate( - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 0, testfixtures.CpuMem("31", "1Gi"), - testfixtures.ItN8GpuNodes(1, testfixtures.TestPriorities), + testfixtures.N8GpuNodes(1, testfixtures.TestPriorities), ), - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 0, testfixtures.CpuMemGpu("31", "1Gi", "1"), - testfixtures.ItN8GpuNodes(1, testfixtures.TestPriorities), + testfixtures.N8GpuNodes(1, testfixtures.TestPriorities), ), - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 0, testfixtures.CpuMemGpu("31", "1Gi", "2"), - testfixtures.ItN8GpuNodes(1, testfixtures.TestPriorities), + testfixtures.N8GpuNodes(1, testfixtures.TestPriorities), ), - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 0, testfixtures.CpuMemGpu("31", "1Gi", "5"), - testfixtures.ItN8GpuNodes(1, testfixtures.TestPriorities), + testfixtures.N8GpuNodes(1, testfixtures.TestPriorities), ), - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 0, testfixtures.CpuMem("31", "2Gi"), - testfixtures.ItN8GpuNodes(1, testfixtures.TestPriorities), + testfixtures.N8GpuNodes(1, testfixtures.TestPriorities), ), - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 0, testfixtures.CpuMemGpu("31", "2Gi", "1"), - testfixtures.ItN8GpuNodes(1, testfixtures.TestPriorities), + testfixtures.N8GpuNodes(1, testfixtures.TestPriorities), ), - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 0, testfixtures.CpuMem("32", "514Gi"), - testfixtures.ItN8GpuNodes(1, testfixtures.TestPriorities), + testfixtures.N8GpuNodes(1, testfixtures.TestPriorities), ), - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 0, testfixtures.CpuMem("32", "512Gi"), - testfixtures.ItN8GpuNodes(1, testfixtures.TestPriorities), + testfixtures.N8GpuNodes(1, testfixtures.TestPriorities), ), - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 0, testfixtures.CpuMem("32", "513Gi"), - testfixtures.ItN8GpuNodes(1, testfixtures.TestPriorities), + testfixtures.N8GpuNodes(1, testfixtures.TestPriorities), ), - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 0, testfixtures.Cpu("33"), - testfixtures.ItN8GpuNodes(1, testfixtures.TestPriorities), + testfixtures.N8GpuNodes(1, testfixtures.TestPriorities), ), ), ), @@ -315,7 +315,7 @@ func TestNodeTypeIterator(t *testing.T) { for i, node := range tc.nodes { // Set monotonically increasing node IDs to ensure nodes appear in predictable order. newNodeId := fmt.Sprintf("%d", i) - entry := testfixtures.ItWithIdNodes(newNodeId, []*internaltypes.Node{node})[0] + entry := testfixtures.WithIdNodes(newNodeId, []*internaltypes.Node{node})[0] nodeDb.AddNodeToDb(entry) entries[i] = entry @@ -388,17 +388,17 @@ func TestNodeTypesIterator(t *testing.T) { }{ "only yield nodes of the right nodeType": { nodes: armadaslices.Concatenate( - testfixtures.ItWithNodeTypeNodes( + testfixtures.WithNodeTypeNodes( nodeTypeA, - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), ), - testfixtures.ItWithNodeTypeNodes( + testfixtures.WithNodeTypeNodes( nodeTypeB, - testfixtures.ItN32CpuNodes(2, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(2, testfixtures.TestPriorities), ), - testfixtures.ItWithNodeTypeNodes( + testfixtures.WithNodeTypeNodes( nodeTypeC, - testfixtures.ItN32CpuNodes(3, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(3, testfixtures.TestPriorities), ), ), nodeTypeIds: []uint64{nodeTypeA.GetId(), nodeTypeC.GetId()}, @@ -411,35 +411,35 @@ func TestNodeTypesIterator(t *testing.T) { }, "filter nodes with insufficient resources and return in increasing order": { nodes: armadaslices.Concatenate( - testfixtures.ItWithNodeTypeNodes( + testfixtures.WithNodeTypeNodes( nodeTypeA, - testfixtures.ItWithUsedResourcesNodes(0, + testfixtures.WithUsedResourcesNodes(0, testfixtures.Cpu("15"), - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), ), ), - testfixtures.ItWithNodeTypeNodes( + testfixtures.WithNodeTypeNodes( nodeTypeB, - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 0, testfixtures.Cpu("16"), - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), ), ), - testfixtures.ItWithNodeTypeNodes( + testfixtures.WithNodeTypeNodes( nodeTypeC, - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 0, testfixtures.Cpu("17"), - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), ), ), - testfixtures.ItWithNodeTypeNodes( + testfixtures.WithNodeTypeNodes( nodeTypeD, - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 0, testfixtures.Cpu("14"), - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), ), ), ), @@ -449,53 +449,53 @@ func TestNodeTypesIterator(t *testing.T) { expected: []int{1, 0}, }, "filter nodes with insufficient resources at priority and return in increasing order": { - nodes: testfixtures.ItWithNodeTypeNodes( + nodes: testfixtures.WithNodeTypeNodes( nodeTypeA, armadaslices.Concatenate( - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 0, testfixtures.Cpu("15"), - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), ), - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 0, testfixtures.Cpu("16"), - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), ), - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 0, testfixtures.Cpu("17"), - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), ), - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 1, testfixtures.Cpu("15"), - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), ), - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 1, testfixtures.Cpu("16"), - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), ), - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 1, testfixtures.Cpu("17"), - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), ), - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 2, testfixtures.Cpu("15"), - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), ), - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 2, testfixtures.Cpu("16"), - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), ), - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 2, testfixtures.Cpu("17"), - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), ), ), ), @@ -505,53 +505,53 @@ func TestNodeTypesIterator(t *testing.T) { expected: []int{4, 7, 3, 6, 0, 1, 2}, }, "nested ordering": { - nodes: testfixtures.ItWithNodeTypeNodes( + nodes: testfixtures.WithNodeTypeNodes( nodeTypeA, armadaslices.Concatenate( - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 0, testfixtures.CpuMem("15", "1Gi"), - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), ), - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 0, testfixtures.CpuMem("15", "2Gi"), - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), ), - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 0, testfixtures.CpuMem("15", "129Gi"), - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), ), - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 0, testfixtures.CpuMem("15", "130Gi"), - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), ), - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 0, testfixtures.CpuMem("15", "131Gi"), - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), ), - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 0, testfixtures.CpuMem("16", "130Gi"), - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), ), - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 0, testfixtures.CpuMem("16", "128Gi"), - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), ), - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 0, testfixtures.CpuMem("16", "129Gi"), - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), ), - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 0, testfixtures.Cpu("17"), - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), ), ), ), @@ -562,68 +562,68 @@ func TestNodeTypesIterator(t *testing.T) { }, "double-nested ordering": { nodes: armadaslices.Concatenate( - testfixtures.ItWithNodeTypeNodes( + testfixtures.WithNodeTypeNodes( nodeTypeA, armadaslices.Concatenate( - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 0, testfixtures.CpuMem("31", "1Gi"), - testfixtures.ItN8GpuNodes(1, testfixtures.TestPriorities), + testfixtures.N8GpuNodes(1, testfixtures.TestPriorities), ), - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 0, testfixtures.CpuMemGpu("31", "1Gi", "1"), - testfixtures.ItN8GpuNodes(1, testfixtures.TestPriorities), + testfixtures.N8GpuNodes(1, testfixtures.TestPriorities), ), - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 0, testfixtures.CpuMemGpu("31", "1Gi", "2"), - testfixtures.ItN8GpuNodes(1, testfixtures.TestPriorities), + testfixtures.N8GpuNodes(1, testfixtures.TestPriorities), ), - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 0, testfixtures.CpuMemGpu("31", "1Gi", "5"), - testfixtures.ItN8GpuNodes(1, testfixtures.TestPriorities), + testfixtures.N8GpuNodes(1, testfixtures.TestPriorities), ), ), ), - testfixtures.ItWithNodeTypeNodes( + testfixtures.WithNodeTypeNodes( nodeTypeB, armadaslices.Concatenate( - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 0, testfixtures.CpuMem("31", "2Gi"), - testfixtures.ItN8GpuNodes(1, testfixtures.TestPriorities), + testfixtures.N8GpuNodes(1, testfixtures.TestPriorities), ), - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 0, testfixtures.CpuMemGpu("31", "2Gi", "1"), - testfixtures.ItN8GpuNodes(1, testfixtures.TestPriorities), + testfixtures.N8GpuNodes(1, testfixtures.TestPriorities), ), - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 0, testfixtures.CpuMem("32", "514Gi"), - testfixtures.ItN8GpuNodes(1, testfixtures.TestPriorities), + testfixtures.N8GpuNodes(1, testfixtures.TestPriorities), ), - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 0, testfixtures.CpuMem("32", "512Gi"), - testfixtures.ItN8GpuNodes(1, testfixtures.TestPriorities), + testfixtures.N8GpuNodes(1, testfixtures.TestPriorities), ), ), ), - testfixtures.ItWithNodeTypeNodes( + testfixtures.WithNodeTypeNodes( nodeTypeC, armadaslices.Concatenate( - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 0, testfixtures.CpuMem("32", "513Gi"), - testfixtures.ItN8GpuNodes(1, testfixtures.TestPriorities), + testfixtures.N8GpuNodes(1, testfixtures.TestPriorities), ), - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 0, testfixtures.Cpu("33"), - testfixtures.ItN8GpuNodes(1, testfixtures.TestPriorities), + testfixtures.N8GpuNodes(1, testfixtures.TestPriorities), ), ), ), @@ -643,8 +643,8 @@ func TestNodeTypesIterator(t *testing.T) { for i, node := range tc.nodes { // Set monotonically increasing node IDs to ensure nodes appear in predictable order. nodeId := fmt.Sprintf("%d", i) - entry := testfixtures.ItWithIdNodes(nodeId, []*internaltypes.Node{node})[0] - entry = testfixtures.ItWithIndexNode(uint64(i), entry) + entry := testfixtures.WithIdNodes(nodeId, []*internaltypes.Node{node})[0] + entry = testfixtures.WithIndexNode(uint64(i), entry) require.NoError(t, err) @@ -704,11 +704,11 @@ func BenchmarkNodeTypeIterator(b *testing.B) { 2, 2100, 2200, 2300, 2400, 2500, 2600, 2700, 2800, 2900, 3, 4, 5, 6, 7, 8, 9, } - nodes := testfixtures.ItN32CpuNodes(numNodes, testfixtures.TestPriorities) + nodes := testfixtures.N32CpuNodes(numNodes, testfixtures.TestPriorities) for i, node := range nodes { var q resource.Quantity q.SetMilli(allocatedMilliCpus[i%len(allocatedMilliCpus)]) - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( testfixtures.TestPriorities[len(testfixtures.TestPriorities)-1], testfixtures.TestResourceListFactory.FromJobResourceListIgnoreUnknown(map[string]resource.Quantity{"cpu": q}), []*internaltypes.Node{node}, diff --git a/internal/scheduler/scheduling/gang_scheduler_test.go b/internal/scheduler/scheduling/gang_scheduler_test.go index f21338d20ca..53b5c9121af 100644 --- a/internal/scheduler/scheduling/gang_scheduler_test.go +++ b/internal/scheduler/scheduling/gang_scheduler_test.go @@ -54,7 +54,7 @@ func TestGangScheduler(t *testing.T) { }{ "simple success": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Gangs: [][]*jobdb.Job{ testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 32)), }, @@ -65,7 +65,7 @@ func TestGangScheduler(t *testing.T) { }, "simple failure": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Gangs: [][]*jobdb.Job{ testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 33)), }, @@ -76,7 +76,7 @@ func TestGangScheduler(t *testing.T) { }, "one success and one failure": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Gangs: [][]*jobdb.Job{ testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 32)), testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1)), @@ -88,7 +88,7 @@ func TestGangScheduler(t *testing.T) { }, "multiple nodes": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.ItN32CpuNodes(2, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(2, testfixtures.TestPriorities), Gangs: [][]*jobdb.Job{ testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 64)), }, @@ -103,7 +103,7 @@ func TestGangScheduler(t *testing.T) { cfg.ExperimentalFloatingResources = testfixtures.TestFloatingResourceConfig return cfg }(), - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Gangs: [][]*jobdb.Job{ // we have 10 of test-floating-resource so only the first of these two jobs should fit addFloatingResourceRequest("6", testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1))), @@ -119,7 +119,7 @@ func TestGangScheduler(t *testing.T) { map[string]float64{"cpu": 0.5}, testfixtures.TestSchedulingConfig(), ), - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Gangs: [][]*jobdb.Job{ testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 8)), testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 16)), @@ -138,7 +138,7 @@ func TestGangScheduler(t *testing.T) { testfixtures.TestSchedulingConfig(), ), ), - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Gangs: [][]*jobdb.Job{ testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1)), testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1)), @@ -159,7 +159,7 @@ func TestGangScheduler(t *testing.T) { testfixtures.TestSchedulingConfig(), ), ), - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Gangs: [][]*jobdb.Job{ testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1)), testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1)), @@ -182,7 +182,7 @@ func TestGangScheduler(t *testing.T) { }, testfixtures.TestSchedulingConfig(), ), - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Gangs: [][]*jobdb.Job{ testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1)), testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 2)), @@ -206,7 +206,7 @@ func TestGangScheduler(t *testing.T) { }, testfixtures.TestSchedulingConfig(), ), - Nodes: testfixtures.ItN32CpuNodes(3, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(3, testfixtures.TestPriorities), Gangs: [][]*jobdb.Job{ testfixtures.WithGangAnnotationsJobs(testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 1)), testfixtures.WithGangAnnotationsJobs(testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 1)), @@ -228,7 +228,7 @@ func TestGangScheduler(t *testing.T) { }, testfixtures.TestSchedulingConfig(), ), - Nodes: testfixtures.ItN32CpuNodes(3, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(3, testfixtures.TestPriorities), Gangs: [][]*jobdb.Job{ testfixtures.WithGangAnnotationsJobs(testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 1)), testfixtures.WithGangAnnotationsJobs(testfixtures.N16Cpu128GiJobs("A", testfixtures.PriorityClass0, 1)), @@ -250,15 +250,15 @@ func TestGangScheduler(t *testing.T) { testfixtures.TestSchedulingConfig(), ), Nodes: armadaslices.Concatenate( - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 0, testfixtures.CpuMemGpu("31.5", "512Gi", "8"), - testfixtures.ItN8GpuNodes(1, testfixtures.TestPriorities), + testfixtures.N8GpuNodes(1, testfixtures.TestPriorities), ), - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 0, testfixtures.Cpu("32"), - testfixtures.ItN8GpuNodes(1, testfixtures.TestPriorities), + testfixtures.N8GpuNodes(1, testfixtures.TestPriorities), ), ), Gangs: [][]*jobdb.Job{ @@ -272,7 +272,7 @@ func TestGangScheduler(t *testing.T) { "NodeUniformityLabel set but not indexed": { SchedulingConfig: testfixtures.TestSchedulingConfig(), Nodes: testfixtures.TestNodeFactory.AddLabels( - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), map[string]string{"foo": "foov"}, ), Gangs: [][]*jobdb.Job{ @@ -292,7 +292,7 @@ func TestGangScheduler(t *testing.T) { []string{"foo", "bar"}, testfixtures.TestSchedulingConfig(), ), - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Gangs: [][]*jobdb.Job{ testfixtures.WithGangAnnotationsJobs( testfixtures.WithNodeUniformityLabelAnnotationJobs( @@ -312,11 +312,11 @@ func TestGangScheduler(t *testing.T) { ), Nodes: armadaslices.Concatenate( testfixtures.TestNodeFactory.AddLabels( - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), map[string]string{"foo": "foov1"}, ), testfixtures.TestNodeFactory.AddLabels( - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), map[string]string{"foo": "foov2"}, ), ), @@ -337,22 +337,22 @@ func TestGangScheduler(t *testing.T) { ), Nodes: armadaslices.Concatenate( testfixtures.TestNodeFactory.AddLabels( - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 0, testfixtures.Cpu("1"), - testfixtures.ItN32CpuNodes(2, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(2, testfixtures.TestPriorities), ), map[string]string{"foo": "foov1"}, ), testfixtures.TestNodeFactory.AddLabels( - testfixtures.ItN32CpuNodes(2, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(2, testfixtures.TestPriorities), map[string]string{"foo": "foov2"}, ), testfixtures.TestNodeFactory.AddLabels( - testfixtures.ItWithUsedResourcesNodes( + testfixtures.WithUsedResourcesNodes( 0, testfixtures.Cpu("1"), - testfixtures.ItN32CpuNodes(2, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(2, testfixtures.TestPriorities), ), map[string]string{"foo": "foov3"}, ), @@ -376,11 +376,11 @@ func TestGangScheduler(t *testing.T) { ), Nodes: append( testfixtures.TestNodeFactory.AddLabels( - testfixtures.ItN32CpuNodes(2, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(2, testfixtures.TestPriorities), map[string]string{"my-cool-node-uniformity": "a"}, ), testfixtures.TestNodeFactory.AddLabels( - testfixtures.ItN32CpuNodes(3, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(3, testfixtures.TestPriorities), map[string]string{"my-cool-node-uniformity": "b"}, )..., ), @@ -445,7 +445,7 @@ func TestGangScheduler(t *testing.T) { return config }(), Nodes: testfixtures.TestNodeFactory.AddTaints( - testfixtures.ItN8GpuNodes(1, []int32{29000, 30000}), + testfixtures.N8GpuNodes(1, []int32{29000, 30000}), []v1.Taint{ {Key: "taint-a", Value: "true", Effect: v1.TaintEffectNoSchedule}, {Key: "taint-b", Value: "true", Effect: v1.TaintEffectNoSchedule}, @@ -494,7 +494,7 @@ func TestGangScheduler(t *testing.T) { return config }(), Nodes: testfixtures.TestNodeFactory.AddTaints( - testfixtures.ItN32CpuNodes(1, []int32{29000, 30000}), + testfixtures.N32CpuNodes(1, []int32{29000, 30000}), []v1.Taint{ {Key: "taint-a", Value: "true", Effect: v1.TaintEffectNoSchedule}, }, @@ -518,7 +518,7 @@ func TestGangScheduler(t *testing.T) { cfg.MaximumPerQueueSchedulingRate = 2 return cfg }(), - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Gangs: [][]*jobdb.Job{ testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 2)), testfixtures.WithGangAnnotationsJobs(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1)), @@ -531,7 +531,7 @@ func TestGangScheduler(t *testing.T) { }, "Hitting globally applicable constraint makes job scheduling key unfeasible": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Gangs: [][]*jobdb.Job{ testfixtures.WithGangAnnotationsJobs(testfixtures.N1GpuJobs("A", testfixtures.PriorityClass0, 1)), testfixtures.WithGangAnnotationsJobs(testfixtures.N1GpuJobs("B", testfixtures.PriorityClass0, 1)), diff --git a/internal/scheduler/scheduling/market_driven_preempting_queue_scheduler_test.go b/internal/scheduler/scheduling/market_driven_preempting_queue_scheduler_test.go index 4c5e8e1257a..e2363a19b37 100644 --- a/internal/scheduler/scheduling/market_driven_preempting_queue_scheduler_test.go +++ b/internal/scheduler/scheduling/market_driven_preempting_queue_scheduler_test.go @@ -54,7 +54,7 @@ func TestMarketDrivenPreemptingQueueScheduler(t *testing.T) { }{ "three users, highest price jobs from single queue get on": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -78,7 +78,7 @@ func TestMarketDrivenPreemptingQueueScheduler(t *testing.T) { }, "three users, highest price jobs between queues get on": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -114,7 +114,7 @@ func TestMarketDrivenPreemptingQueueScheduler(t *testing.T) { }, "Two users, no preemption if price lower": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -134,7 +134,7 @@ func TestMarketDrivenPreemptingQueueScheduler(t *testing.T) { }, "Two users, preemption if price higher": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -162,7 +162,7 @@ func TestMarketDrivenPreemptingQueueScheduler(t *testing.T) { }, "Two users, partial preemption if price higher": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -193,7 +193,7 @@ func TestMarketDrivenPreemptingQueueScheduler(t *testing.T) { }, "Self Preemption If Price Is Higher": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -224,7 +224,7 @@ func TestMarketDrivenPreemptingQueueScheduler(t *testing.T) { }, "Two Users. Self preemption plus cross user preemption": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -254,7 +254,7 @@ func TestMarketDrivenPreemptingQueueScheduler(t *testing.T) { }, "gang preemption": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ diff --git a/internal/scheduler/scheduling/preempting_queue_scheduler_test.go b/internal/scheduler/scheduling/preempting_queue_scheduler_test.go index 775a5b66ebf..88d70e37257 100644 --- a/internal/scheduler/scheduling/preempting_queue_scheduler_test.go +++ b/internal/scheduler/scheduling/preempting_queue_scheduler_test.go @@ -57,7 +57,7 @@ func TestEvictOversubscribed(t *testing.T) { err = nodeDb.CreateAndInsertWithJobDbJobsWithTxn( nodeDbTxn, jobs, - testfixtures.ItTest32CpuNode(priorities), + testfixtures.Test32CpuNode(priorities), ) require.NoError(t, err) @@ -110,7 +110,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { }{ "balancing three queues": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -166,7 +166,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { }, "balancing two queues weighted": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -204,7 +204,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { }, "balancing two queues weighted with inactive queues": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -244,7 +244,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { }, "don't prempt jobs where we don't know the queue": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -264,7 +264,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { }, "avoid preemption when not improving fairness": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -287,7 +287,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { }, "avoid preemption when not improving fairness reverse queue naming": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -310,7 +310,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { }, "preemption when improving fairness": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.ItN32CpuNodes(2, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(2, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -341,7 +341,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { }, "reschedule onto same node": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.ItN32CpuNodes(2, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(2, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -368,7 +368,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { }, "reschedule onto same node reverse order": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.ItN32CpuNodes(2, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(2, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -395,7 +395,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { }, "urgency-based preemption stability": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -432,7 +432,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { }, "avoid urgency-based preemption when possible": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.ItN32CpuNodes(2, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(2, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -458,7 +458,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { }, "preempt in order of priority": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.ItN32CpuNodes(2, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(2, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -497,7 +497,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { }, "avoid urgency-based preemption when possible cross-queue": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.ItN32CpuNodes(3, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(3, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -549,7 +549,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { }, "gang preemption": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.ItN32CpuNodes(2, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(2, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { // Fill half of node 1 and half of node 2. @@ -596,7 +596,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { "gang preemption avoid cascading preemption": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.ItN32CpuNodes(3, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(3, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { // Schedule a gang spanning nodes 1 and 2. @@ -642,7 +642,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { }, "rescheduled jobs don't count towards global scheduling rate limit": { SchedulingConfig: testfixtures.WithGlobalSchedulingRateLimiterConfig(2, 5, testfixtures.TestSchedulingConfig()), - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -667,7 +667,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { }, "MaximumSchedulingRate": { SchedulingConfig: testfixtures.WithGlobalSchedulingRateLimiterConfig(2, 4, testfixtures.TestSchedulingConfig()), - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -703,7 +703,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { }, "rescheduled jobs don't count towards maxQueueLookback": { SchedulingConfig: testfixtures.WithMaxLookbackPerQueueConfig(5, testfixtures.TestSchedulingConfig()), - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -733,7 +733,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { }, testfixtures.TestSchedulingConfig(), ), - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -758,7 +758,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { }, "priority class preemption two classes": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -788,7 +788,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { }, "priority class preemption cross-queue": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -819,7 +819,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { }, "priority class preemption not scheduled": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -840,7 +840,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { // - 17 jobs should be preempted to make space; all the priority zero jobs and a single priority 1 job "priority class preemption through multiple levels": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -906,7 +906,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { //}, "priority class preemption four classes": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -948,7 +948,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { }, testfixtures.TestSchedulingConfig(), ), - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -984,7 +984,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { }, testfixtures.TestSchedulingConfig(), ), - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -1052,7 +1052,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { map[string]float64{"cpu": 15.0 / 64.0}, testfixtures.TestSchedulingConfig(), ), - Nodes: testfixtures.ItN32CpuNodes(2, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(2, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -1099,7 +1099,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { }, "Queued jobs are not preempted cross queue": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -1119,7 +1119,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { }, "Queued jobs are not preempted cross queue with some scheduled": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -1140,7 +1140,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { }, "Queued jobs are not preempted cross queue with non-preemptible jobs": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -1160,7 +1160,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { }, "Queued jobs are not preempted cross queue multiple rounds": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -1188,7 +1188,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { }, "Oversubscribed eviction does not evict non-preemptible": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.ItN32CpuNodes(2, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(2, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -1226,7 +1226,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { }, "Cordoning prevents scheduling new jobs but not re-scheduling running jobs": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -1259,7 +1259,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { 1.0, testfixtures.TestSchedulingConfig(), ), - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -1295,7 +1295,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { 0.5, testfixtures.TestSchedulingConfig(), ), - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -1331,7 +1331,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { 0.5, testfixtures.TestSchedulingConfig(), ), - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -1375,7 +1375,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { 1.0, testfixtures.TestSchedulingConfig(), ), - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -1411,7 +1411,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { 1.0, testfixtures.TestSchedulingConfig(), ), - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -1439,7 +1439,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { }, "DominantResourceFairness": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -1462,18 +1462,18 @@ func TestPreemptingQueueScheduler(t *testing.T) { SchedulingConfig: testfixtures.TestSchedulingConfig(), Nodes: armadaslices.Concatenate( testfixtures.TestNodeFactory.AddLabels( - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), map[string]string{"key": "val1"}, ), testfixtures.TestNodeFactory.AddLabels( - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), map[string]string{"key": "val2"}, ), testfixtures.TestNodeFactory.AddLabels( - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), map[string]string{"key": "val3"}, ), - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), ), Rounds: []SchedulingRound{ { @@ -1507,7 +1507,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { // Schedule job from queue C. // This does not prevent re-scheduling. SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.ItN32CpuNodes(2, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(2, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -1544,7 +1544,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { // Try to schedule a job from queue B. This fails as the node is cordoned. // This should not prevent re-scheduling jobs in queue A. SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Rounds: []SchedulingRound{ { JobsByQueue: map[string][]*jobdb.Job{ @@ -1576,7 +1576,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { return config }(), Nodes: testfixtures.TestNodeFactory.AddTaints( - testfixtures.ItN8GpuNodes(2, []int32{29000, 30000}), + testfixtures.N8GpuNodes(2, []int32{29000, 30000}), []v1.Taint{{Key: "gpu", Value: "true", Effect: v1.TaintEffectNoSchedule}}, ), Rounds: []SchedulingRound{ @@ -1633,7 +1633,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { return config }(), Nodes: testfixtures.TestNodeFactory.AddTaints( - testfixtures.ItN8GpuNodes(2, []int32{29000, 30000}), + testfixtures.N8GpuNodes(2, []int32{29000, 30000}), []v1.Taint{{Key: "gpu", Value: "true", Effect: v1.TaintEffectNoSchedule}}, ), Rounds: []SchedulingRound{ @@ -1698,10 +1698,10 @@ func TestPreemptingQueueScheduler(t *testing.T) { Nodes: func() []*internaltypes.Node { priorities := []int32{29000, 30000} gpuNodes := testfixtures.TestNodeFactory.AddTaints( - testfixtures.ItN8GpuNodes(1, priorities), + testfixtures.N8GpuNodes(1, priorities), []v1.Taint{{Key: "gpu", Value: "true", Effect: v1.TaintEffectNoSchedule}}, ) - return append(testfixtures.ItN32CpuNodes(1, priorities), gpuNodes...) + return append(testfixtures.N32CpuNodes(1, priorities), gpuNodes...) }(), Rounds: []SchedulingRound{ { @@ -1759,10 +1759,10 @@ func TestPreemptingQueueScheduler(t *testing.T) { Nodes: func() []*internaltypes.Node { priorities := []int32{29000, 30000} gpuNodes := testfixtures.TestNodeFactory.AddTaints( - testfixtures.ItN8GpuNodes(1, priorities), + testfixtures.N8GpuNodes(1, priorities), []v1.Taint{{Key: "gpu", Value: "true", Effect: v1.TaintEffectNoSchedule}}, ) - return append(testfixtures.ItN32CpuNodes(1, priorities), gpuNodes...) + return append(testfixtures.N32CpuNodes(1, priorities), gpuNodes...) }(), Rounds: []SchedulingRound{ { @@ -1831,7 +1831,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { return config }(), Nodes: testfixtures.TestNodeFactory.AddTaints( - testfixtures.ItN32CpuNodes(1, []int32{29000, 28000, 30000}), + testfixtures.N32CpuNodes(1, []int32{29000, 28000, 30000}), []v1.Taint{{Key: "gpu", Value: "true", Effect: v1.TaintEffectNoSchedule}}, ), Rounds: []SchedulingRound{ @@ -2259,7 +2259,7 @@ func BenchmarkPreemptingQueueScheduler(b *testing.B) { }{ "1 node 1 queue 320 jobs": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), JobFunc: testfixtures.N1Cpu4GiJobs, NumQueues: 1, NumJobsPerQueue: 320, @@ -2268,7 +2268,7 @@ func BenchmarkPreemptingQueueScheduler(b *testing.B) { }, "1 node 10 queues 320 jobs": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), JobFunc: testfixtures.N1Cpu4GiJobs, NumQueues: 10, NumJobsPerQueue: 320, @@ -2277,7 +2277,7 @@ func BenchmarkPreemptingQueueScheduler(b *testing.B) { }, "10 nodes 1 queue 3200 jobs": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.ItN32CpuNodes(10, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(10, testfixtures.TestPriorities), JobFunc: testfixtures.N1Cpu4GiJobs, NumQueues: 1, NumJobsPerQueue: 3200, @@ -2286,7 +2286,7 @@ func BenchmarkPreemptingQueueScheduler(b *testing.B) { }, "10 nodes 10 queues 3200 jobs": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.ItN32CpuNodes(10, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(10, testfixtures.TestPriorities), JobFunc: testfixtures.N1Cpu4GiJobs, NumQueues: 10, NumJobsPerQueue: 3200, @@ -2295,7 +2295,7 @@ func BenchmarkPreemptingQueueScheduler(b *testing.B) { }, "100 nodes 1 queue 32000 jobs": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.ItN32CpuNodes(100, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(100, testfixtures.TestPriorities), JobFunc: testfixtures.N1Cpu4GiJobs, NumQueues: 1, NumJobsPerQueue: 32000, @@ -2304,7 +2304,7 @@ func BenchmarkPreemptingQueueScheduler(b *testing.B) { }, "100 nodes 10 queues 32000 jobs": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.ItN32CpuNodes(100, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(100, testfixtures.TestPriorities), JobFunc: testfixtures.N1Cpu4GiJobs, NumQueues: 10, NumJobsPerQueue: 32000, @@ -2313,7 +2313,7 @@ func BenchmarkPreemptingQueueScheduler(b *testing.B) { }, "1000 nodes 1 queue 320000 jobs": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.ItN32CpuNodes(1000, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1000, testfixtures.TestPriorities), JobFunc: testfixtures.N1Cpu4GiJobs, NumQueues: 1, NumJobsPerQueue: 320000, @@ -2322,7 +2322,7 @@ func BenchmarkPreemptingQueueScheduler(b *testing.B) { }, "1000 nodes 10 queues 320000 jobs": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.ItN32CpuNodes(1000, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1000, testfixtures.TestPriorities), JobFunc: testfixtures.N1Cpu4GiJobs, NumQueues: 1, NumJobsPerQueue: 32000, diff --git a/internal/scheduler/scheduling/queue_scheduler_test.go b/internal/scheduler/scheduling/queue_scheduler_test.go index 9aaca4cdb78..14e0454386a 100644 --- a/internal/scheduler/scheduling/queue_scheduler_test.go +++ b/internal/scheduler/scheduling/queue_scheduler_test.go @@ -46,48 +46,48 @@ func TestQueueScheduler(t *testing.T) { "simple success": { SchedulingConfig: testfixtures.TestSchedulingConfig(), Queues: testfixtures.SingleQueuePriorityOne("A"), - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Jobs: testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 32), ExpectedScheduledIndices: testfixtures.IntRange(0, 31), }, "simple failure": { SchedulingConfig: testfixtures.TestSchedulingConfig(), Queues: testfixtures.SingleQueuePriorityOne("A"), - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Jobs: testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 33), ExpectedScheduledIndices: testfixtures.IntRange(0, 31), }, "multiple nodes": { SchedulingConfig: testfixtures.TestSchedulingConfig(), Queues: testfixtures.SingleQueuePriorityOne("A"), - Nodes: testfixtures.ItN32CpuNodes(2, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(2, testfixtures.TestPriorities), Jobs: testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 64), ExpectedScheduledIndices: testfixtures.IntRange(0, 63), }, "preempt lower-priority jobs": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Jobs: armadaslices.Concatenate(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1), testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass1, 1)), Queues: testfixtures.SingleQueuePriorityOne("A"), ExpectedScheduledIndices: testfixtures.IntRange(0, 1), }, "no preemption of higher-priority jobs": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Jobs: armadaslices.Concatenate(testfixtures.N32Cpu256GiJobs("A", testfixtures.PriorityClass1, 1), testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1)), Queues: testfixtures.SingleQueuePriorityOne("A"), ExpectedScheduledIndices: testfixtures.IntRange(0, 0), }, "unschedulable jobs do not block schedulable jobs": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Jobs: armadaslices.Concatenate(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1), testfixtures.N32Cpu256GiJobs("A", testfixtures.PriorityClass0, 10), testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1)), Queues: testfixtures.SingleQueuePriorityOne("A"), ExpectedScheduledIndices: []int{0, 11}, }, "MaximumSchedulingBurst": { SchedulingConfig: testfixtures.WithGlobalSchedulingRateLimiterConfig(10, 2, testfixtures.TestSchedulingConfig()), - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Jobs: armadaslices.Concatenate( testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1), testfixtures.N32Cpu256GiJobs("A", testfixtures.PriorityClass0, 10), @@ -99,7 +99,7 @@ func TestQueueScheduler(t *testing.T) { }, "MaximumPerQueueSchedulingBurst": { SchedulingConfig: testfixtures.WithPerQueueSchedulingLimiterConfig(10, 2, testfixtures.TestSchedulingConfig()), - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Jobs: armadaslices.Concatenate( testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1), testfixtures.N32Cpu256GiJobs("A", testfixtures.PriorityClass0, 10), @@ -112,7 +112,7 @@ func TestQueueScheduler(t *testing.T) { }, "MaximumSchedulingBurst is not exceeded by gangs": { SchedulingConfig: testfixtures.WithGlobalSchedulingRateLimiterConfig(10, 2, testfixtures.TestSchedulingConfig()), - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Jobs: armadaslices.Concatenate( testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1), testfixtures.N32Cpu256GiJobs("A", testfixtures.PriorityClass0, 1), @@ -126,7 +126,7 @@ func TestQueueScheduler(t *testing.T) { }, "MaximumPerQueueSchedulingBurst is not exceeded by gangs": { SchedulingConfig: testfixtures.WithPerQueueSchedulingLimiterConfig(10, 2, testfixtures.TestSchedulingConfig()), - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Jobs: armadaslices.Concatenate( testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1), testfixtures.N32Cpu256GiJobs("A", testfixtures.PriorityClass0, 1), @@ -144,7 +144,7 @@ func TestQueueScheduler(t *testing.T) { testfixtures.TestSchedulingConfig(), ), Queues: testfixtures.SingleQueuePriorityOne("A"), - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Jobs: testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 32), ExpectedScheduledIndices: testfixtures.IntRange(0, 16), ExpectedNeverAttemptedIndices: testfixtures.IntRange(17, 31), @@ -160,7 +160,7 @@ func TestQueueScheduler(t *testing.T) { testfixtures.TestSchedulingConfig(), ), Queues: testfixtures.SingleQueuePriorityOne("A"), - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Jobs: armadaslices.Concatenate( testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1), testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 2), @@ -196,20 +196,20 @@ func TestQueueScheduler(t *testing.T) { }, }, }, - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Jobs: testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 32), ExpectedScheduledIndices: testfixtures.IntRange(0, 15), }, "fairness two queues": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Jobs: armadaslices.Concatenate(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 32), testfixtures.N1Cpu4GiJobs("B", testfixtures.PriorityClass0, 32)), Queues: []*api.Queue{{Name: "A", PriorityFactor: 1.0}, {Name: "B", PriorityFactor: 1.0}}, ExpectedScheduledIndices: armadaslices.Concatenate(testfixtures.IntRange(0, 15), testfixtures.IntRange(32, 47)), }, "fairness three queues": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Jobs: armadaslices.Concatenate( testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 32), testfixtures.N1Cpu4GiJobs("B", testfixtures.PriorityClass0, 32), @@ -224,7 +224,7 @@ func TestQueueScheduler(t *testing.T) { }, "weighted fairness two queues": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.ItN32CpuNodes(3, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(3, testfixtures.TestPriorities), Jobs: armadaslices.Concatenate( testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 96), testfixtures.N1Cpu4GiJobs("B", testfixtures.PriorityClass0, 96), @@ -237,7 +237,7 @@ func TestQueueScheduler(t *testing.T) { }, "weighted fairness three queues": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.ItN32CpuNodes(3, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(3, testfixtures.TestPriorities), Jobs: armadaslices.Concatenate( testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 96), testfixtures.N1Cpu4GiJobs("B", testfixtures.PriorityClass0, 96), @@ -252,7 +252,7 @@ func TestQueueScheduler(t *testing.T) { }, "fairness two queues with initial allocation": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Jobs: armadaslices.Concatenate( testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 32), testfixtures.N1Cpu4GiJobs("B", testfixtures.PriorityClass0, 32), @@ -267,10 +267,10 @@ func TestQueueScheduler(t *testing.T) { }, "node with no available capacity": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.ItWithUsedResourcesNodes( + Nodes: testfixtures.WithUsedResourcesNodes( 0, testfixtures.Cpu("32"), - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), ), Jobs: testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1), Queues: testfixtures.SingleQueuePriorityOne("A"), @@ -278,10 +278,10 @@ func TestQueueScheduler(t *testing.T) { }, "node with some available capacity": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.ItWithUsedResourcesNodes( + Nodes: testfixtures.WithUsedResourcesNodes( 0, testfixtures.Cpu("31"), - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), ), Jobs: testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 2), Queues: testfixtures.SingleQueuePriorityOne("A"), @@ -289,10 +289,10 @@ func TestQueueScheduler(t *testing.T) { }, "preempt used resources of lower-priority jobs": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.ItWithUsedResourcesNodes( + Nodes: testfixtures.WithUsedResourcesNodes( 0, testfixtures.Cpu("32"), - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), ), Jobs: testfixtures.N32Cpu256GiJobs("A", testfixtures.PriorityClass1, 1), Queues: testfixtures.SingleQueuePriorityOne("A"), @@ -300,14 +300,14 @@ func TestQueueScheduler(t *testing.T) { }, "respect taints": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.ItNTainted32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.NTainted32CpuNodes(1, testfixtures.TestPriorities), Jobs: armadaslices.Concatenate(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1), testfixtures.N32Cpu256GiJobs("A", testfixtures.PriorityClass0, 1)), Queues: testfixtures.SingleQueuePriorityOne("A"), ExpectedScheduledIndices: []int{1}, }, "taints and tolerations": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.ItNTainted32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.NTainted32CpuNodes(1, testfixtures.TestPriorities), Jobs: armadaslices.Concatenate(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1), testfixtures.N32Cpu256GiJobs("A", testfixtures.PriorityClass0, 1)), Queues: testfixtures.SingleQueuePriorityOne("A"), ExpectedScheduledIndices: []int{1}, @@ -315,9 +315,9 @@ func TestQueueScheduler(t *testing.T) { "Node selector": { SchedulingConfig: testfixtures.TestSchedulingConfig(), Nodes: armadaslices.Concatenate( - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), testfixtures.TestNodeFactory.AddLabels( - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), map[string]string{"foo": "foo"}, ), ), @@ -327,7 +327,7 @@ func TestQueueScheduler(t *testing.T) { }, "taints and tolerations (indexed)": { SchedulingConfig: testfixtures.WithIndexedTaintsConfig([]string{"largeJobsOnly"}, testfixtures.TestSchedulingConfig()), - Nodes: testfixtures.ItNTainted32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.NTainted32CpuNodes(1, testfixtures.TestPriorities), Jobs: armadaslices.Concatenate(testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1), testfixtures.N32Cpu256GiJobs("A", testfixtures.PriorityClass0, 1)), Queues: testfixtures.SingleQueuePriorityOne("A"), ExpectedScheduledIndices: []int{1}, @@ -335,9 +335,9 @@ func TestQueueScheduler(t *testing.T) { "Node selector (indexed)": { SchedulingConfig: testfixtures.WithIndexedNodeLabelsConfig([]string{"foo"}, testfixtures.TestSchedulingConfig()), Nodes: append( - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), testfixtures.TestNodeFactory.AddLabels( - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), map[string]string{"foo": "foo"}, )..., ), @@ -347,7 +347,7 @@ func TestQueueScheduler(t *testing.T) { }, "MaxQueueLookback": { SchedulingConfig: testfixtures.WithMaxQueueLookbackConfig(3, testfixtures.TestSchedulingConfig()), - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Jobs: armadaslices.Concatenate( testfixtures.N1Cpu4GiJobs("A", testfixtures.PriorityClass0, 1), testfixtures.N32Cpu256GiJobs("A", testfixtures.PriorityClass0, 3), @@ -359,14 +359,14 @@ func TestQueueScheduler(t *testing.T) { }, "gang success": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.ItN32CpuNodes(2, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(2, testfixtures.TestPriorities), Jobs: testfixtures.WithGangAnnotationsJobs(testfixtures.N32Cpu256GiJobs("A", testfixtures.PriorityClass0, 2)), Queues: testfixtures.SingleQueuePriorityOne("A"), ExpectedScheduledIndices: []int{0, 1}, }, "non-consecutive gang success": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.ItN32CpuNodes(3, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(3, testfixtures.TestPriorities), Jobs: armadaslices.Concatenate( testfixtures.WithAnnotationsJobs(map[string]string{ armadaconfiguration.GangIdAnnotation: "my-gang", @@ -385,14 +385,14 @@ func TestQueueScheduler(t *testing.T) { }, "gang failure": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.ItN32CpuNodes(2, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(2, testfixtures.TestPriorities), Jobs: testfixtures.WithGangAnnotationsJobs(testfixtures.N32Cpu256GiJobs("A", testfixtures.PriorityClass0, 3)), Queues: testfixtures.SingleQueuePriorityOne("A"), ExpectedScheduledIndices: nil, }, "non-consecutive gang failure": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.ItN32CpuNodes(2, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(2, testfixtures.TestPriorities), Jobs: armadaslices.Concatenate( testfixtures.WithAnnotationsJobs(map[string]string{ armadaconfiguration.GangIdAnnotation: "my-gang", @@ -411,7 +411,7 @@ func TestQueueScheduler(t *testing.T) { }, "job priority": { SchedulingConfig: testfixtures.TestSchedulingConfig(), - Nodes: testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + Nodes: testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), Jobs: armadaslices.Concatenate( testfixtures.WithPriorityJobs(10, testfixtures.N32Cpu256GiJobs("A", testfixtures.PriorityClass0, 1)), testfixtures.WithPriorityJobs(1, testfixtures.N32Cpu256GiJobs("A", testfixtures.PriorityClass0, 1)), @@ -424,18 +424,18 @@ func TestQueueScheduler(t *testing.T) { SchedulingConfig: testfixtures.TestSchedulingConfig(), Nodes: armadaslices.Concatenate( testfixtures.TestNodeFactory.AddLabels( - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), map[string]string{"key": "val1"}, ), testfixtures.TestNodeFactory.AddLabels( - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), map[string]string{"key": "val2"}, ), testfixtures.TestNodeFactory.AddLabels( - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), map[string]string{"key": "val3"}, ), - testfixtures.ItN32CpuNodes(1, testfixtures.TestPriorities), + testfixtures.N32CpuNodes(1, testfixtures.TestPriorities), ), Jobs: armadaslices.Concatenate( testfixtures.WithNodeAffinityJobs( diff --git a/internal/scheduler/scheduling/scheduling_algo_test.go b/internal/scheduler/scheduling/scheduling_algo_test.go index f5dfdf51a90..8ea52c14b7e 100644 --- a/internal/scheduler/scheduling/scheduling_algo_test.go +++ b/internal/scheduler/scheduling/scheduling_algo_test.go @@ -639,7 +639,7 @@ func BenchmarkNodeDbConstruction(b *testing.B) { numNodes := int(math.Pow10(e)) b.Run(fmt.Sprintf("%d nodes", numNodes), func(b *testing.B) { jobs := testfixtures.N1Cpu4GiJobs("queue-alice", testfixtures.PriorityClass0, 32*numNodes) - nodes := testfixtures.ItN32CpuNodes(numNodes, testfixtures.TestPriorities) + nodes := testfixtures.N32CpuNodes(numNodes, testfixtures.TestPriorities) for i, node := range nodes { for j := 32 * i; j < 32*(i+1); j++ { jobs[j] = jobs[j].WithNewRun("executor-01", node.GetId(), node.GetName(), node.GetPool(), jobs[j].PriorityClass().Priority) diff --git a/internal/scheduler/testfixtures/testfixtures.go b/internal/scheduler/testfixtures/testfixtures.go index 12d4c2d164a..902e5a1054d 100644 --- a/internal/scheduler/testfixtures/testfixtures.go +++ b/internal/scheduler/testfixtures/testfixtures.go @@ -290,14 +290,14 @@ func WithMaxQueueLookbackConfig(maxQueueLookback uint, config schedulerconfigura return config } -func ItWithUsedResourcesNodes(p int32, rl internaltypes.ResourceList, nodes []*internaltypes.Node) []*internaltypes.Node { +func WithUsedResourcesNodes(p int32, rl internaltypes.ResourceList, nodes []*internaltypes.Node) []*internaltypes.Node { for _, node := range nodes { internaltypes.MarkAllocated(node.AllocatableByPriority, p, rl) } return nodes } -func ItWithNodeTypeNodes(nodeType *internaltypes.NodeType, nodes []*internaltypes.Node) []*internaltypes.Node { +func WithNodeTypeNodes(nodeType *internaltypes.NodeType, nodes []*internaltypes.Node) []*internaltypes.Node { result := make([]*internaltypes.Node, len(nodes)) for i, node := range nodes { result[i] = internaltypes.CreateNode(node.GetId(), @@ -319,7 +319,7 @@ func ItWithNodeTypeNodes(nodeType *internaltypes.NodeType, nodes []*internaltype return result } -func ItWithIdNodes(nodeId string, nodes []*internaltypes.Node) []*internaltypes.Node { +func WithIdNodes(nodeId string, nodes []*internaltypes.Node) []*internaltypes.Node { result := make([]*internaltypes.Node, len(nodes)) for i, node := range nodes { result[i] = internaltypes.CreateNode(nodeId, @@ -342,7 +342,7 @@ func ItWithIdNodes(nodeId string, nodes []*internaltypes.Node) []*internaltypes. return result } -func ItWithIndexNode(idx uint64, node *internaltypes.Node) *internaltypes.Node { +func WithIndexNode(idx uint64, node *internaltypes.Node) *internaltypes.Node { return internaltypes.CreateNode(node.GetId(), node.GetNodeType(), idx, @@ -700,26 +700,26 @@ func TestUnitReqs() *schedulerobjects.PodRequirements { } } -func ItN32CpuNodes(n int, priorities []int32) []*internaltypes.Node { +func N32CpuNodes(n int, priorities []int32) []*internaltypes.Node { rv := make([]*internaltypes.Node, n) for i := 0; i < n; i++ { - rv[i] = ItTest32CpuNode(priorities) + rv[i] = Test32CpuNode(priorities) } return rv } -func ItNTainted32CpuNodes(n int, priorities []int32) []*internaltypes.Node { +func NTainted32CpuNodes(n int, priorities []int32) []*internaltypes.Node { rv := make([]*internaltypes.Node, n) for i := 0; i < n; i++ { - rv[i] = ItTestTainted32CpuNode(priorities) + rv[i] = TestTainted32CpuNode(priorities) } return rv } -func ItN8GpuNodes(n int, priorities []int32) []*internaltypes.Node { +func N8GpuNodes(n int, priorities []int32) []*internaltypes.Node { rv := make([]*internaltypes.Node, n) for i := 0; i < n; i++ { - rv[i] = ItTest8GpuNode(priorities) + rv[i] = Test8GpuNode(priorities) } return rv } @@ -767,7 +767,7 @@ func TestSimpleNode(id string) *internaltypes.Node { nil) } -func ItTestNode(priorities []int32, resources map[string]resource.Quantity) *internaltypes.Node { +func TestNode(priorities []int32, resources map[string]resource.Quantity) *internaltypes.Node { rl := TestNodeFactory.ResourceListFactory().FromNodeProto(resources) id := uuid.NewString() return TestNodeFactory.CreateNodeAndType(id, @@ -785,8 +785,8 @@ func ItTestNode(priorities []int32, resources map[string]resource.Quantity) *int internaltypes.NewAllocatableByPriorityAndResourceType(priorities, rl)) } -func ItTest32CpuNode(priorities []int32) *internaltypes.Node { - return ItTestNode( +func Test32CpuNode(priorities []int32) *internaltypes.Node { + return TestNode( priorities, map[string]resource.Quantity{ "cpu": resource.MustParse("32"), @@ -795,8 +795,8 @@ func ItTest32CpuNode(priorities []int32) *internaltypes.Node { ) } -func ItTestTainted32CpuNode(priorities []int32) *internaltypes.Node { - node := ItTest32CpuNode(priorities) +func TestTainted32CpuNode(priorities []int32) *internaltypes.Node { + node := Test32CpuNode(priorities) node = TestNodeFactory.AddTaints([]*internaltypes.Node{node}, []v1.Taint{ @@ -815,8 +815,8 @@ func ItTestTainted32CpuNode(priorities []int32) *internaltypes.Node { return node } -func ItTest8GpuNode(priorities []int32) *internaltypes.Node { - node := ItTestNode( +func Test8GpuNode(priorities []int32) *internaltypes.Node { + node := TestNode( priorities, map[string]resource.Quantity{ "cpu": resource.MustParse("64"), From 1ddd71bcd50530eaa792b8af3d6eb5dd8f96e506 Mon Sep 17 00:00:00 2001 From: Maurice Yap Date: Fri, 3 Jan 2025 16:24:11 +0000 Subject: [PATCH 08/17] Use virtualized table for job sets table (#4116) We [previously](https://github.com/armadaproject/armada/commit/a57e7f2622a4efb8e70b24415ec75046ecd5824d) made the job sets table a normal, non-virtualized table. This was to simplify the logic, remove the third-party library, `react-virtualized`, which is no longer well-maintained, and done on the assumption that the page would not have to display enough rows of data to cause it any performance-related trouble. We have observed in some environments that the latter assumption is not correct, and have noticed significant performance degradation with very large datasets. This commit makes the table a virtualised one again, but using the well-maintained Virtuoso library. It also stops truncating the job set ID as users have indicated it is often useful to see the whole thing, even if it is long. --- internal/lookout/ui/package.json | 2 +- .../src/components/job-sets/JobSetTable.tsx | 165 ++++++++++-------- .../lookoutV2/JobStateCountChip.tsx | 6 +- internal/lookout/ui/yarn.lock | 10 +- 4 files changed, 103 insertions(+), 80 deletions(-) diff --git a/internal/lookout/ui/package.json b/internal/lookout/ui/package.json index 7878909b465..1dc8345ff74 100644 --- a/internal/lookout/ui/package.json +++ b/internal/lookout/ui/package.json @@ -29,7 +29,6 @@ "@mui/icons-material": "^6.1.10", "@mui/lab": "^6.0.0-beta.18", "@mui/material": "^6.1.10", - "@re-dev/react-truncate": "^0.4.3", "@tanstack/react-query": "^5.62.3", "@tanstack/react-table": "^8.7.0", "date-fns": "^2.29.3", @@ -45,6 +44,7 @@ "react": "^18", "react-dom": "^18", "react-router-dom": "6.14.2", + "react-virtuoso": "^4.12.3", "semver": "6.3.1", "tough-cookie": "^4.1.3", "use-debounce": "^10.0.0", diff --git a/internal/lookout/ui/src/components/job-sets/JobSetTable.tsx b/internal/lookout/ui/src/components/job-sets/JobSetTable.tsx index 59755ff4e68..0dbe358363b 100644 --- a/internal/lookout/ui/src/components/job-sets/JobSetTable.tsx +++ b/internal/lookout/ui/src/components/job-sets/JobSetTable.tsx @@ -1,4 +1,4 @@ -import { useCallback, useEffect, useState } from "react" +import { forwardRef, useCallback, useEffect, useState } from "react" import { Alert, @@ -15,7 +15,7 @@ import { TableSortLabel, } from "@mui/material" import { visuallyHidden } from "@mui/utils" -import { Truncate } from "@re-dev/react-truncate" +import { TableVirtuoso, TableComponents } from "react-virtuoso" import { JobState, jobStateColors, jobStateIcons } from "../../models/lookoutV2Models" import { JobSet } from "../../services/JobService" @@ -31,7 +31,33 @@ const JOB_STATES_TO_DISPLAY = [ [JobState.Cancelled, "jobsCancelled"], ] as const -const MinWidthTableCell = styled(TableCell)({ width: "0%", textWrap: "nowrap" }) +interface TableContext { + selectedJobSetIds: Set +} + +const NoWrapTableCell = styled(TableCell)({ textWrap: "nowrap" }) +const MinWidthTableCell = styled(NoWrapTableCell)({ width: "0%" }) +const MaxWidthTableCell = styled(TableCell)({ width: "100%" }) + +const StyledTable = styled(Table)({ + borderCollapse: "separate", +}) + +const TableHeaderRowCell = styled(TableCell)(({ theme }) => ({ background: theme.palette.background.paper })) + +const TableHeaderRowMinWidthCell = styled(MinWidthTableCell)(({ theme }) => ({ + background: theme.palette.background.paper, +})) + +const VirtuosoTableComponents: TableComponents = { + Scroller: forwardRef((props, ref) => ), + Table: (props) => , + TableHead: forwardRef((props, ref) => ), + TableRow: ({ context, ...props }) => ( + + ), + TableBody: forwardRef((props, ref) => ), +} interface JobSetTableProps { queue: string @@ -89,82 +115,77 @@ export default function JobSetTable({ } return ( - - - - + + data={jobSets} + components={VirtuosoTableComponents} + context={{ selectedJobSetIds: new Set(selectedJobSets.keys()) }} + fixedHeaderContent={() => ( + + + (checked ? onSelectAllClick() : onDeselectAllClick())} + /> + + Job Set + + onOrderChange(!newestFirst)}> + Submission time + + {newestFirst ? "sorted descending" : "sorted ascending"} + + + + {JOB_STATES_TO_DISPLAY.map(([jobState]) => { + const Icon = jobStateIcons[jobState] + return ( + + + {formatJobState(jobState)} + + + + ) + })} + + )} + itemContent={(jobSetIndex, { jobSetId, latestSubmissionTime, ...jobSetRest }) => { + const rowSelected = selectedJobSets.has(jobSetId) + return ( + <> (checked ? onSelectAllClick() : onDeselectAllClick())} + checked={rowSelected} + onChange={(_, checked) => + shiftKeyPressed ? onShiftSelectJobSet(jobSetIndex, checked) : onSelectJobSet(jobSetIndex, checked) + } /> - Job Set - - onOrderChange(!newestFirst)} - > - Submission time - - {newestFirst ? "sorted descending" : "sorted ascending"} - - - - {JOB_STATES_TO_DISPLAY.map(([jobState]) => { - const Icon = jobStateIcons[jobState] - return ( - - - {formatJobState(jobState)} - - - - ) - })} - - - - {jobSets.map(({ jobSetId, latestSubmissionTime, ...jobSetRest }, jobSetIndex) => { - const rowSelected = selectedJobSets.has(jobSetId) - return ( - - - - shiftKeyPressed ? onShiftSelectJobSet(jobSetIndex, checked) : onSelectJobSet(jobSetIndex, checked) - } + {jobSetId} + {latestSubmissionTime} + {JOB_STATES_TO_DISPLAY.map(([jobState, jobSetKey]) => ( + +
+ onJobSetStateClick(jobSetIndex, jobState)} /> - - - {jobSetId} - - {latestSubmissionTime} - {JOB_STATES_TO_DISPLAY.map(([jobState, jobSetKey]) => ( - - onJobSetStateClick(jobSetIndex, jobState)} - /> - - ))} - - ) - })} - -
- + + + ))} + + ) + }} + /> ) } diff --git a/internal/lookout/ui/src/components/lookoutV2/JobStateCountChip.tsx b/internal/lookout/ui/src/components/lookoutV2/JobStateCountChip.tsx index 354f7aaa948..7c2868a0397 100644 --- a/internal/lookout/ui/src/components/lookoutV2/JobStateCountChip.tsx +++ b/internal/lookout/ui/src/components/lookoutV2/JobStateCountChip.tsx @@ -1,7 +1,9 @@ -import { Chip } from "@mui/material" +import { Chip, styled } from "@mui/material" import { JobState, jobStateColors } from "../../models/lookoutV2Models" +const StyledChip = styled(Chip)({ padding: "0 1ch" }) + export interface JobStateCountChipProps { state: JobState count: number @@ -11,7 +13,7 @@ export const JobStateCountChip = ({ state, count, onClick }: JobStateCountChipPr const label = count.toString() return count > 0 ? ( - + ) : ( <>{label} ) diff --git a/internal/lookout/ui/yarn.lock b/internal/lookout/ui/yarn.lock index 0c92373beac..6f7d6602b7f 100644 --- a/internal/lookout/ui/yarn.lock +++ b/internal/lookout/ui/yarn.lock @@ -988,11 +988,6 @@ resolved "https://registry.yarnpkg.com/@popperjs/core/-/core-2.11.8.tgz#6b79032e760a0899cd4204710beede972a3a185f" integrity sha512-P1st0aksCrn9sGZhp8GMYwBnQsbvAWsZAX44oXNNvLHGqAOcoVxmjZiohstwQ7SqKnbR47akdNi+uleWD8+g6A== -"@re-dev/react-truncate@^0.4.3": - version "0.4.3" - resolved "https://registry.yarnpkg.com/@re-dev/react-truncate/-/react-truncate-0.4.3.tgz#89ed2935a30f271a80f6af8fa66c9fe471baf742" - integrity sha512-dMtHgU/uOorC5gQNtJBjIf9Zv5+lRa11+0YSTEbfD1KCyKKb0bcCjVCEVyyKUSv0/vEmiG+twQIt2eVKETaRNA== - "@remix-run/router@1.7.2": version "1.7.2" resolved "https://registry.yarnpkg.com/@remix-run/router/-/router-1.7.2.tgz#cba1cf0a04bc04cb66027c51fa600e9cbc388bc8" @@ -4247,6 +4242,11 @@ react-transition-group@^4.4.5: loose-envify "^1.4.0" prop-types "^15.6.2" +react-virtuoso@^4.12.3: + version "4.12.3" + resolved "https://registry.yarnpkg.com/react-virtuoso/-/react-virtuoso-4.12.3.tgz#beecf0582b31058c5a6ed3ec58fc43fd780e5844" + integrity sha512-6X1p/sU7hecmjDZMAwN+r3go9EVjofKhwkUbVlL8lXhBZecPv9XVCkZ/kBPYOr0Mv0Vl5+Ziwgexg9Kh7+NNXQ== + react@^18: version "18.3.1" resolved "https://registry.yarnpkg.com/react/-/react-18.3.1.tgz#49ab892009c53933625bd16b2533fc754cab2891" From 3a3fc36d44f184d51cf64ee61e4dc0c66cea848c Mon Sep 17 00:00:00 2001 From: Rob Smith <34475852+robertdavidsmith@users.noreply.github.com> Date: Fri, 3 Jan 2025 17:06:34 +0000 Subject: [PATCH 09/17] Scheduler: fix panic in nodedb benchmark (#4120) Signed-off-by: Robert Smith --- internal/scheduler/nodedb/nodedb_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/scheduler/nodedb/nodedb_test.go b/internal/scheduler/nodedb/nodedb_test.go index 368fdae6277..e8258d140c8 100644 --- a/internal/scheduler/nodedb/nodedb_test.go +++ b/internal/scheduler/nodedb/nodedb_test.go @@ -727,7 +727,8 @@ func benchmarkUpsert(nodes []*internaltypes.Node, b *testing.B) { for i, node := range nodes { err = nodeDb.CreateAndInsertWithJobDbJobsWithTxn(txn, nil, node) require.NoError(b, err) - entry, err := nodeDb.GetNode(node.GetId()) + entry, err := nodeDb.GetNodeWithTxn(txn, node.GetId()) + require.NotNil(b, entry) require.NoError(b, err) entries[i] = entry } From 640df2c9040dad05532fcd7427551a602789335a Mon Sep 17 00:00:00 2001 From: Chris Martin Date: Mon, 6 Jan 2025 09:36:31 +0000 Subject: [PATCH 10/17] Update GRPC Middleware (#4112) * update pulsar mock * go mod tidy * import order * wip * wip * wip * supress logging * lint * wip --- go.mod | 3 +- go.sum | 25 +-- .../common/armadacontext/armada_context.go | 7 +- .../armadacontext/armada_context_test.go | 9 - internal/common/armadaerrors/errors_test.go | 2 +- internal/common/auth/common.go | 16 +- internal/common/auth/kubernetes_test.go | 4 +- internal/common/grpc/grpc.go | 158 ++++++++++-------- internal/common/logging/interceptors.go | 50 ------ internal/common/logging/interceptors_test.go | 70 -------- internal/common/requestid/interceptors.go | 3 +- .../common/requestid/interceptors_test.go | 2 +- internal/executor/service/lease_requester.go | 2 +- internal/scheduler/schedulerapp.go | 12 +- .../testsuite/eventwatcher/eventwatcher.go | 2 +- pkg/client/connection.go | 2 +- 16 files changed, 116 insertions(+), 251 deletions(-) delete mode 100644 internal/common/logging/interceptors.go delete mode 100644 internal/common/logging/interceptors_test.go diff --git a/go.mod b/go.mod index a650830b832..c2623ca95ce 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,6 @@ require ( github.com/golang/protobuf v1.5.4 github.com/google/go-cmp v0.6.0 // indirect github.com/google/uuid v1.6.0 - github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/grpc-ecosystem/grpc-gateway v1.16.0 github.com/hashicorp/go-memdb v1.3.4 @@ -72,6 +71,8 @@ require ( github.com/go-playground/validator/v10 v10.15.4 github.com/gogo/status v1.1.1 github.com/goreleaser/goreleaser v1.24.0 + github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 + github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.2.0 github.com/jackc/pgx/v5 v5.5.4 github.com/jessevdk/go-flags v1.5.0 github.com/magefile/mage v1.14.0 diff --git a/go.sum b/go.sum index ac75f1140ad..1491cc31df8 100644 --- a/go.sum +++ b/go.sum @@ -73,7 +73,6 @@ github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuP github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= -github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/immutable v0.4.3 h1:GYHcksoJ9K6HyAUpGxwZURrbTkXA0Dh4otXGqbhdrjA= github.com/benbjohnson/immutable v0.4.3/go.mod h1:qJIKKSmdqz1tVzNtst1DZzvaqOU1onk1rc03IeM3Owk= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -170,8 +169,6 @@ github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= -github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -208,7 +205,6 @@ github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91 github.com/go-playground/validator/v10 v10.15.4 h1:zMXza4EpOdooxPel5xDqXEdXG5r+WggpvnAKMsalBjs= github.com/go-playground/validator/v10 v10.15.4/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 h1:ZpnhV/YsD2/4cESfV5+Hoeu/iUR3ruzNvZ+yQfO03a0= @@ -298,8 +294,10 @@ github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 h1:pdN6V1QBWetyv/0+wjACpqVH+eVULgEjkurDLq3goeM= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= -github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 h1:UH//fgunKIs4JdUbpDl1VZCDaL56wXCB/5+wF6uHfaI= -github.com/grpc-ecosystem/go-grpc-middleware v1.4.0/go.mod h1:g5qyo/la0ALbONm6Vbp88Yd8NsDy6rZz+RcrMPxvld8= +github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 h1:qnpSQwGEnkcRpTqNOIR6bJbR0gAorgP9CSALpRcKoAA= +github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1/go.mod h1:lXGCsh6c22WGtjr+qGHj1otzZpV/1kwTMAqkwZsnWRU= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.2.0 h1:kQ0NI7W1B3HwiN5gAYtY+XFItDPbLBwYRxAqbFTyDes= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.2.0/go.mod h1:zrT2dxOAjNFPRGjTUe2Xmb4q4YdUwVvQFV6xiCSf+z0= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= @@ -369,7 +367,6 @@ github.com/klauspost/compress v1.13.1/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8 github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -462,7 +459,6 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= -github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pborman/getopt v0.0.0-20180729010549-6fdd0a2c7117/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= @@ -475,7 +471,6 @@ github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4/v4 v4.1.8 h1:ieHkV+i2BRzngO4Wd/3HGowuZStgq6QkPsD1eolNAO4= github.com/pierrec/lz4/v4 v4.1.8/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -521,7 +516,6 @@ github.com/shirou/gopsutil/v3 v3.23.12 h1:z90NtUkp3bMtmICZKpC4+WaknU1eXtp5vtbQ11 github.com/shirou/gopsutil/v3 v3.23.12/go.mod h1:1FrWgea594Jp7qmjHUUPlJDTPgcsb9mGnXDxavtikzM= github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= @@ -541,7 +535,6 @@ github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -609,16 +602,12 @@ go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 h1:+FNtrFTmVw0YZGpBGX56XDee331t6JAXeK2bcyhLOOc= go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5/go.mod h1:nmDLcffg48OtT/PSW0Hg7FvpRQsQh5OSqIylirxKC7o= -go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU= go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM= -go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= golang.org/x/crypto v0.0.0-20180723164146-c126467f60eb/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -712,7 +701,6 @@ golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -734,7 +722,6 @@ golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210819135213-f52c844e1c1c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -776,7 +763,6 @@ golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -832,7 +818,6 @@ google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvx google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= @@ -851,7 +836,6 @@ google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQ google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.67.1 h1:zWnc1Vrcno+lHZCOofnIMvycFcc0QRGIzm9dhnDX68E= google.golang.org/grpc v1.67.1/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA= @@ -898,7 +882,6 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/internal/common/armadacontext/armada_context.go b/internal/common/armadacontext/armada_context.go index 665c19c51f9..30fb2b466c9 100644 --- a/internal/common/armadacontext/armada_context.go +++ b/internal/common/armadacontext/armada_context.go @@ -4,7 +4,6 @@ import ( "context" "time" - "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus/ctxlogrus" "github.com/sirupsen/logrus" "golang.org/x/sync/errgroup" ) @@ -39,8 +38,10 @@ func FromGrpcCtx(ctx context.Context) *Context { if ok { return armadaCtx } - log := ctxlogrus.Extract(ctx) - return New(ctx, log) + logger := logrus.NewEntry(logrus.StandardLogger()). + WithField("user", ctx.Value("user")). + WithField("requestId", ctx.Value("requestId")) + return New(ctx, logger) } // New returns an armada context that encapsulates both a go context and a logger diff --git a/internal/common/armadacontext/armada_context_test.go b/internal/common/armadacontext/armada_context_test.go index 4cda401c1b1..ef7c84aa705 100644 --- a/internal/common/armadacontext/armada_context_test.go +++ b/internal/common/armadacontext/armada_context_test.go @@ -5,8 +5,6 @@ import ( "testing" "time" - "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus/ctxlogrus" - "github.com/sirupsen/logrus" "github.com/stretchr/testify/require" ) @@ -19,13 +17,6 @@ func TestNew(t *testing.T) { require.Equal(t, context.Background(), ctx.Context) } -func TestFromGrpcContext(t *testing.T) { - grpcCtx := ctxlogrus.ToContext(context.Background(), defaultLogger) - ctx := FromGrpcCtx(grpcCtx) - require.Equal(t, grpcCtx, ctx.Context) - require.Equal(t, defaultLogger, ctx.FieldLogger) -} - func TestBackground(t *testing.T) { ctx := Background() require.Equal(t, ctx.Context, context.Background()) diff --git a/internal/common/armadaerrors/errors_test.go b/internal/common/armadaerrors/errors_test.go index fa70cf43e32..da72a6458c0 100644 --- a/internal/common/armadaerrors/errors_test.go +++ b/internal/common/armadaerrors/errors_test.go @@ -9,7 +9,7 @@ import ( "time" "github.com/apache/pulsar-client-go/pulsar" - grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware" + grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware/v2" "github.com/pkg/errors" "github.com/redis/go-redis/v9" "github.com/stretchr/testify/assert" diff --git a/internal/common/auth/common.go b/internal/common/auth/common.go index f3bd468cffd..604487bdffd 100644 --- a/internal/common/auth/common.go +++ b/internal/common/auth/common.go @@ -4,11 +4,9 @@ import ( "context" "net/http" + "github.com/grpc-ecosystem/go-grpc-middleware/v2/metadata" "golang.org/x/exp/slices" - grpc_ctxtags "github.com/grpc-ecosystem/go-grpc-middleware/tags" - "github.com/grpc-ecosystem/go-grpc-middleware/util/metautils" - "github.com/armadaproject/armada/internal/common/util" ) @@ -120,16 +118,13 @@ type AuthService interface { // request context for logging purposes. func CreateGrpcMiddlewareAuthFunction(authService AuthService) func(ctx context.Context) (context.Context, error) { return func(ctx context.Context) (context.Context, error) { - authHeader := metautils.ExtractIncoming(ctx).Get("authorization") + authHeader := metadata.ExtractIncoming(ctx).Get("authorization") principal, err := authService.Authenticate(ctx, authHeader) if err != nil { return nil, err } - // record username for request logging - grpc_ctxtags.Extract(ctx).Set("user", principal.GetName()) - grpc_ctxtags.Extract(ctx).Set("authService", principal.GetAuthMethod()) - + ctx = context.WithValue(ctx, "user", principal.GetName()) return WithPrincipal(ctx, principal), nil } } @@ -150,11 +145,6 @@ func CreateHttpMiddlewareAuthFunction(authService AuthService) func(w http.Respo http.Error(w, "auth error:"+err.Error(), http.StatusInternalServerError) return nil, err } - - // record username for request logging - grpc_ctxtags.Extract(ctx).Set("user", principal.GetName()) - grpc_ctxtags.Extract(ctx).Set("authService", principal.GetAuthMethod()) - return WithPrincipal(ctx, principal), nil } } diff --git a/internal/common/auth/kubernetes_test.go b/internal/common/auth/kubernetes_test.go index 01da9d02b1b..d29ceb4c17b 100644 --- a/internal/common/auth/kubernetes_test.go +++ b/internal/common/auth/kubernetes_test.go @@ -9,7 +9,7 @@ import ( "testing" "time" - "github.com/grpc-ecosystem/go-grpc-middleware/util/metautils" + grpc_metadata "github.com/grpc-ecosystem/go-grpc-middleware/v2/metadata" "github.com/patrickmn/go-cache" "github.com/stretchr/testify/assert" authv1 "k8s.io/api/authentication/v1" @@ -151,7 +151,7 @@ func TestAuthenticateKubernetes(t *testing.T) { // Create authentication context payload := createKubernetesAuthPayload(testToken, testCA) ctx := context.Background() - metadata := metautils.ExtractIncoming(ctx) + metadata := grpc_metadata.ExtractIncoming(ctx) metadata.Set("authorization", payload) ctx = metadata.ToIncoming(ctx) diff --git a/internal/common/grpc/grpc.go b/internal/common/grpc/grpc.go index c4f858f3ba1..1139e04a427 100644 --- a/internal/common/grpc/grpc.go +++ b/internal/common/grpc/grpc.go @@ -1,6 +1,7 @@ package grpc import ( + "context" "crypto/tls" "fmt" "net" @@ -8,12 +9,12 @@ import ( "sync" "time" - grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware" - grpc_auth "github.com/grpc-ecosystem/go-grpc-middleware/auth" - grpc_logrus "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus" - grpc_recovery "github.com/grpc-ecosystem/go-grpc-middleware/recovery" - grpc_ctxtags "github.com/grpc-ecosystem/go-grpc-middleware/tags" - grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" + grpc_prometheus "github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus" + grpc_auth "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/auth" + grpc_logging "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/logging" + grpc_recovery "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/recovery" + + "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" "google.golang.org/grpc" "google.golang.org/grpc/codes" @@ -37,77 +38,47 @@ func CreateGrpcServer( keepaliveEnforcementPolicy keepalive.EnforcementPolicy, authServices []auth.AuthService, tlsConfig configuration.TlsConfig, - logrusOptions ...grpc_logrus.Option, + loggerOpts ...grpc_logging.Option, ) *grpc.Server { - // Logging, authentication, etc. are implemented via gRPC interceptors - // (i.e., via functions that are called before handling the actual request). - // There are separate interceptors for unary and streaming gRPC calls. - unaryInterceptors := []grpc.UnaryServerInterceptor{} - streamInterceptors := []grpc.StreamServerInterceptor{} - - // Automatically recover from panics - // NOTE This must be the first interceptor, so it can handle panics in any subsequently added interceptor - recovery := grpc_recovery.WithRecoveryHandler(panicRecoveryHandler) - unaryInterceptors = append(unaryInterceptors, grpc_recovery.UnaryServerInterceptor(recovery)) - streamInterceptors = append(streamInterceptors, grpc_recovery.StreamServerInterceptor(recovery)) - - // Logging (using logrus) - // By default, information contained in the request context is logged - // tagsExtractor pulls information out of the request payload (a protobuf) and stores it in - // the context, such that it is logged. - messageDefault := log.NewEntry(log.StandardLogger()) - tagsExtractor := grpc_ctxtags.WithFieldExtractor(grpc_ctxtags.CodeGenRequestFieldExtractor) - unaryInterceptors = append(unaryInterceptors, - grpc_ctxtags.UnaryServerInterceptor(tagsExtractor), - requestid.UnaryServerInterceptor(false), - armadaerrors.UnaryServerInterceptor(2000), - grpc_logrus.UnaryServerInterceptor(messageDefault, logrusOptions...), - ) - streamInterceptors = append(streamInterceptors, - grpc_ctxtags.StreamServerInterceptor(tagsExtractor), - requestid.StreamServerInterceptor(false), - armadaerrors.StreamServerInterceptor(2000), - grpc_logrus.StreamServerInterceptor(messageDefault, logrusOptions...), - ) - - // Authentication - // The provided authServices represents a list of services that can be used to authenticate - // the client (e.g., username/password and OpenId). authFunction is a combination of these. authFunction := auth.CreateGrpcMiddlewareAuthFunction(auth.NewMultiAuthService(authServices)) - unaryInterceptors = append(unaryInterceptors, grpc_auth.UnaryServerInterceptor(authFunction)) - streamInterceptors = append(streamInterceptors, grpc_auth.StreamServerInterceptor(authFunction)) + srvMetrics := setupPromMetrics() - // Prometheus timeseries collection integration - grpc_prometheus.EnableHandlingTimeHistogram() - unaryInterceptors = append(unaryInterceptors, grpc_prometheus.UnaryServerInterceptor) - streamInterceptors = append(streamInterceptors, grpc_prometheus.StreamServerInterceptor) + loggerOpts = append( + loggerOpts, + grpc_logging.WithLogOnEvents(grpc_logging.StartCall, grpc_logging.FinishCall)) - serverOptions := []grpc.ServerOption{ + return grpc.NewServer( grpc.KeepaliveParams(keepaliveParams), grpc.KeepaliveEnforcementPolicy(keepaliveEnforcementPolicy), - grpc.StreamInterceptor(grpc_middleware.ChainStreamServer(streamInterceptors...)), - grpc.UnaryInterceptor(grpc_middleware.ChainUnaryServer(unaryInterceptors...)), - } - - if tlsConfig.Enabled { - cachedCertificateService := certs.NewCachedCertificateService(tlsConfig.CertPath, tlsConfig.KeyPath, time.Minute) - go func() { - cachedCertificateService.Run(armadacontext.Background()) - }() - tlsCreds := credentials.NewTLS(&tls.Config{ - GetCertificate: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) { - cert := cachedCertificateService.GetCertificate() - if cert == nil { - return nil, fmt.Errorf("unexpectedly received nil from certificate cache") - } - return cert, nil - }, - }) - serverOptions = append(serverOptions, grpc.Creds(tlsCreds)) - } + setupTls(tlsConfig), + grpc.ChainUnaryInterceptor( + srvMetrics.UnaryServerInterceptor(), + requestid.UnaryServerInterceptor(false), + grpc_auth.UnaryServerInterceptor(authFunction), + grpc_logging.UnaryServerInterceptor(InterceptorLogger(), loggerOpts...), + armadaerrors.UnaryServerInterceptor(2000), + grpc_recovery.UnaryServerInterceptor(grpc_recovery.WithRecoveryHandler(panicRecoveryHandler)), + ), + grpc.ChainStreamInterceptor( + srvMetrics.StreamServerInterceptor(), + requestid.StreamServerInterceptor(false), + grpc_auth.StreamServerInterceptor(authFunction), + grpc_logging.StreamServerInterceptor(InterceptorLogger(), loggerOpts...), + armadaerrors.StreamServerInterceptor(2000), + grpc_recovery.StreamServerInterceptor(grpc_recovery.WithRecoveryHandler(panicRecoveryHandler)), + ), + ) +} - // Interceptors are registered at server creation - return grpc.NewServer(serverOptions...) +func setupPromMetrics() *grpc_prometheus.ServerMetrics { + srvMetrics := grpc_prometheus.NewServerMetrics( + grpc_prometheus.WithServerHandlingTimeHistogram( + grpc_prometheus.WithHistogramBuckets([]float64{0.001, 0.01, 0.1, 0.3, 0.6, 1, 3, 6, 9, 20, 30, 60, 90, 120}), + ), + ) + reg := prometheus.NewRegistry() + reg.MustRegister(srvMetrics) + return srvMetrics } // TODO We don't need this function. Just do this at the caller. @@ -143,8 +114,55 @@ func CreateShutdownHandler(ctx *armadacontext.Context, gracePeriod time.Duration } } +func setupTls(tlsConfig configuration.TlsConfig) grpc.ServerOption { + if !tlsConfig.Enabled { + return grpc.EmptyServerOption{} + } + + cachedCertificateService := certs.NewCachedCertificateService(tlsConfig.CertPath, tlsConfig.KeyPath, time.Minute) + go func() { + cachedCertificateService.Run(armadacontext.Background()) + }() + tlsCreds := credentials.NewTLS(&tls.Config{ + GetCertificate: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) { + cert := cachedCertificateService.GetCertificate() + if cert == nil { + return nil, fmt.Errorf("unexpectedly received nil from certificate cache") + } + return cert, nil + }, + }) + return grpc.Creds(tlsCreds) +} + // This function is called whenever a gRPC handler panics. func panicRecoveryHandler(p interface{}) (err error) { log.Errorf("Request triggered panic with cause %v \n%s", p, string(debug.Stack())) return status.Errorf(codes.Internal, "Internal server error caused by %v", p) } + +func InterceptorLogger() grpc_logging.Logger { + return grpc_logging.LoggerFunc(func(ctx context.Context, lvl grpc_logging.Level, msg string, fields ...any) { + logFields := make(map[string]any, len(fields)/2+2) + logFields["user"] = ctx.Value("user") + logFields["requestId"] = ctx.Value("requestId") + i := grpc_logging.Fields(fields).Iterator() + for i.Next() { + k, v := i.At() + logFields[k] = v + } + l := log.WithFields(logFields) + switch lvl { + case grpc_logging.LevelDebug: + l.Debug(msg) + case grpc_logging.LevelInfo: + l.Info(msg) + case grpc_logging.LevelWarn: + l.Warn(msg) + case grpc_logging.LevelError: + l.Error(msg) + default: + panic(fmt.Sprintf("unknown level %v", lvl)) + } + }) +} diff --git a/internal/common/logging/interceptors.go b/internal/common/logging/interceptors.go deleted file mode 100644 index 33b4ed48889..00000000000 --- a/internal/common/logging/interceptors.go +++ /dev/null @@ -1,50 +0,0 @@ -package logging - -import ( - "context" - "fmt" - - "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus/ctxlogrus" - "github.com/sirupsen/logrus" - "google.golang.org/grpc" - - "github.com/armadaproject/armada/internal/common/requestid" -) - -// UnaryServerInterceptor returns an interceptor that adds the request id as a -// field to the logrus logger embedded in the context. If an error occurs in the handler, -// it also adds a stack trace. -func UnaryServerInterceptor() grpc.UnaryServerInterceptor { - return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { - if id, ok := requestid.FromContext(ctx); ok { - ctxlogrus.AddFields(ctx, logrus.Fields{requestid.MetadataKey: id}) - } - rv, err := handler(ctx, req) - if err != nil { - // %+v prints a stack trace for pkg/errors errors - ctxlogrus.AddFields(ctx, logrus.Fields{"errorVerbose": fmt.Sprintf("%+v", err)}) - } - return rv, err - } -} - -// StreamServerInterceptor returns an interceptor that adds the request id as a -// field to the logrus logger embedded in the context. If an error occurs in the handler, -// it also adds a stack trace. -func StreamServerInterceptor() grpc.StreamServerInterceptor { - return func(srv interface{}, stream grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { - ctx := stream.Context() - if id, ok := requestid.FromContext(ctx); ok { - ctxlogrus.AddFields(ctx, logrus.Fields{requestid.MetadataKey: id}) - } - // The logrus logger only logs at the end of the call. As streaming calls may last a long time - // We also log here which will produce a log line at the start of the call - ctxlogrus.Extract(ctx).Infof("started streaming call") - err := handler(srv, stream) - if err != nil { - // %+v prints a stack trace for pkg/errors errors - ctxlogrus.AddFields(ctx, logrus.Fields{"errorVerbose": fmt.Sprintf("%+v", err)}) - } - return err - } -} diff --git a/internal/common/logging/interceptors_test.go b/internal/common/logging/interceptors_test.go deleted file mode 100644 index 2d3f76ff159..00000000000 --- a/internal/common/logging/interceptors_test.go +++ /dev/null @@ -1,70 +0,0 @@ -package logging - -import ( - "context" - "testing" - - grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware" - "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus/ctxlogrus" - "github.com/renstrom/shortuuid" - "github.com/sirupsen/logrus" - "github.com/stretchr/testify/require" - "google.golang.org/grpc" - "google.golang.org/grpc/metadata" - - "github.com/armadaproject/armada/internal/common/requestid" -) - -func TestUnaryServerInterceptor(t *testing.T) { - ctx := context.Background() - ctx = metadata.NewIncomingContext(ctx, metadata.New(map[string]string{})) - id := shortuuid.New() - ctx, ok := requestid.AddToIncomingContext(ctx, id) - require.True(t, ok, "error adding request id to context") - logger := logrus.New() - entry := logrus.NewEntry(logger) - ctx = ctxlogrus.ToContext(ctx, entry) - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - entry := ctxlogrus.Extract(ctx) - for _, field := range entry.Data { - if s, ok := field.(string); ok && s == id { - return nil, nil - } - } - t.Fatal("request id was not added as a logger field") - return nil, nil - } - - f := UnaryServerInterceptor() - _, err := f(ctx, nil, nil, handler) - require.NoError(t, err) -} - -func TestStreamServerInterceptor(t *testing.T) { - ctx := context.Background() - ctx = metadata.NewIncomingContext(ctx, metadata.New(map[string]string{})) - id := shortuuid.New() - ctx, ok := requestid.AddToIncomingContext(ctx, id) - require.True(t, ok, "error adding request id to context") - - logger := logrus.New() - entry := logrus.NewEntry(logger) - ctx = ctxlogrus.ToContext(ctx, entry) - stream := &grpc_middleware.WrappedServerStream{} - stream.WrappedContext = ctx - handler := func(srv interface{}, stream grpc.ServerStream) error { - ctx := stream.Context() - entry := ctxlogrus.Extract(ctx) - for _, field := range entry.Data { - if s, ok := field.(string); ok && s == id { - return nil - } - } - t.Fatal("request id was not added as a logger field") - return nil - } - - f := StreamServerInterceptor() - err := f(nil, stream, nil, handler) - require.NoError(t, err) -} diff --git a/internal/common/requestid/interceptors.go b/internal/common/requestid/interceptors.go index b5ee03cd058..bd7dd1e7202 100644 --- a/internal/common/requestid/interceptors.go +++ b/internal/common/requestid/interceptors.go @@ -3,7 +3,7 @@ package requestid import ( "context" - grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware" + grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware/v2" "github.com/renstrom/shortuuid" "google.golang.org/grpc" "google.golang.org/grpc/metadata" @@ -43,6 +43,7 @@ func FromContextOrMissing(ctx context.Context) string { // The second return value is true if the operation was successful. func AddToIncomingContext(ctx context.Context, id string) (context.Context, bool) { if md, ok := metadata.FromIncomingContext(ctx); ok { + ctx = context.WithValue(ctx, "requestId", id) md.Set(MetadataKey, id) return metadata.NewIncomingContext(ctx, md), true } diff --git a/internal/common/requestid/interceptors_test.go b/internal/common/requestid/interceptors_test.go index 30cbd63dffa..584bf8798f5 100644 --- a/internal/common/requestid/interceptors_test.go +++ b/internal/common/requestid/interceptors_test.go @@ -4,7 +4,7 @@ import ( "context" "testing" - grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware" + grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware/v2" "github.com/renstrom/shortuuid" "github.com/stretchr/testify/require" "google.golang.org/grpc" diff --git a/internal/executor/service/lease_requester.go b/internal/executor/service/lease_requester.go index c95f31b4725..c9451962b89 100644 --- a/internal/executor/service/lease_requester.go +++ b/internal/executor/service/lease_requester.go @@ -4,7 +4,7 @@ import ( "fmt" "io" - grpcretry "github.com/grpc-ecosystem/go-grpc-middleware/retry" + grpcretry "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/retry" "github.com/pkg/errors" log "github.com/sirupsen/logrus" "google.golang.org/grpc" diff --git a/internal/scheduler/schedulerapp.go b/internal/scheduler/schedulerapp.go index 7482c38ffa7..9bbd0a01b53 100644 --- a/internal/scheduler/schedulerapp.go +++ b/internal/scheduler/schedulerapp.go @@ -10,7 +10,7 @@ import ( "github.com/apache/pulsar-client-go/pulsar" "github.com/google/uuid" - grpc_logrus "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus" + grpc_logging "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/logging" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" @@ -183,7 +183,7 @@ func Run(config schedulerconfig.Configuration) error { if err != nil { return errors.WithMessage(err, "error creating auth services") } - grpcServer := grpcCommon.CreateGrpcServer(config.Grpc.KeepaliveParams, config.Grpc.KeepaliveEnforcementPolicy, authServices, config.Grpc.Tls, createLogrusLoggingOption()) + grpcServer := grpcCommon.CreateGrpcServer(config.Grpc.KeepaliveParams, config.Grpc.KeepaliveEnforcementPolicy, authServices, config.Grpc.Tls, createGrpcLoggingOption()) defer grpcServer.GracefulStop() lis, err := net.Listen("tcp", fmt.Sprintf(":%d", config.Grpc.Port)) if err != nil { @@ -369,13 +369,13 @@ func loadClusterConfig(ctx *armadacontext.Context) (*rest.Config, error) { // - We only care about failures, so lets only log failures // - We normally use these logs to work out who is calling us, however the Executor API is not public // and is only called by other Armada components -func createLogrusLoggingOption() grpc_logrus.Option { - return grpc_logrus.WithLevels(func(code codes.Code) log.Level { +func createGrpcLoggingOption() grpc_logging.Option { + return grpc_logging.WithLevels(func(code codes.Code) grpc_logging.Level { switch code { case codes.OK: - return log.TraceLevel + return grpc_logging.LevelDebug default: - return grpc_logrus.DefaultCodeToLevel(code) + return grpc_logging.DefaultServerCodeToLevel(code) } }) } diff --git a/internal/testsuite/eventwatcher/eventwatcher.go b/internal/testsuite/eventwatcher/eventwatcher.go index 51d619d4050..c39f1d100c0 100644 --- a/internal/testsuite/eventwatcher/eventwatcher.go +++ b/internal/testsuite/eventwatcher/eventwatcher.go @@ -12,7 +12,7 @@ import ( "strings" "time" - "github.com/grpc-ecosystem/go-grpc-middleware/util/backoffutils" + "github.com/grpc-ecosystem/go-grpc-middleware/v2/util/backoffutils" "github.com/pkg/errors" "golang.org/x/exp/maps" "golang.org/x/sync/errgroup" diff --git a/pkg/client/connection.go b/pkg/client/connection.go index 64e3b5dfed5..a9b086b93f6 100644 --- a/pkg/client/connection.go +++ b/pkg/client/connection.go @@ -6,7 +6,7 @@ import ( "strings" "time" - grpc_retry "github.com/grpc-ecosystem/go-grpc-middleware/retry" + grpc_retry "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/retry" "github.com/pkg/errors" "google.golang.org/grpc" "google.golang.org/grpc/credentials" From e611e13ec91abf3874df21f523d0e85c9c594f91 Mon Sep 17 00:00:00 2001 From: JamesMurkin Date: Mon, 6 Jan 2025 10:00:49 +0000 Subject: [PATCH 11/17] [executor] Remove duplicate tolerations section from deployment (#4114) tolerations was specified twice in the executor deployment definition, I've removed one of the places it is specified. I left the place that was most consistent with our other deployments --- deployment/executor/templates/deployment.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/deployment/executor/templates/deployment.yaml b/deployment/executor/templates/deployment.yaml index abd30ade56f..0e98fa9afa5 100644 --- a/deployment/executor/templates/deployment.yaml +++ b/deployment/executor/templates/deployment.yaml @@ -88,8 +88,6 @@ spec: {{- if .Values.priorityClassName }} priorityClassName: {{ .Values.priorityClassName }} {{- end }} - tolerations: - {{- toYaml .Values.tolerations | nindent 8 }} volumes: - name: user-config secret: From 6e5c1eb069027cbd2f7e4b21f2a8855b0447e420 Mon Sep 17 00:00:00 2001 From: Maurice Yap Date: Mon, 6 Jan 2025 10:46:55 +0000 Subject: [PATCH 12/17] Allow configuration of column order (#4118) In the lookout UI, allow the user to change the order of the columns of the jobs table through a drag-and-drop interface. This change replaces the existing column selector with a new column configuration dialog which extends the current functionality of the column selector with the ability to drag and drop columns into the user's desired order. The existing functionality which has been preserved is: - Hiding and showing columns (via a checkbox) - Adding annotation key columns - Removing and editing annotation key columns which have been added - Indicates when columns are for annotation keys The new column configuration dialog has the following additional functionalities: - Hides columns which are configured to be pinned (i.e. the selector column) - Hides columns which are grouped, and explains this to the user - When the input to add an annotation column is not visible on the screen, it displays a chip indicating its presence. Clicking on this chip will scroll to the input - Validates annotation key columns on input, making sure they have no leading or trailing whitespace, and that there is not already a column for the same annotation key This change also makes the buttons in the job action bar more visually consistent - using primary colours for actions and secondary colours for configuration changes. --- internal/lookout/ui/package.json | 3 + .../AddAnnotationColumnInput.tsx | 69 +++ .../EditAnnotationColumnInput.tsx | 69 +++ .../OrderableColumnListItem.tsx | 130 +++++ .../ColumnConfigurationDialog/index.tsx | 247 +++++++++ .../lookoutV2/ColumnSelect.module.css | 31 -- .../src/components/lookoutV2/ColumnSelect.tsx | 255 --------- .../lookoutV2/CustomViewPicker.module.css | 2 +- .../components/lookoutV2/CustomViewPicker.tsx | 8 +- .../lookoutV2/JobsTableActionBar.module.css | 23 +- .../lookoutV2/JobsTableActionBar.tsx | 145 +++--- .../lookoutV2/JobsTableContainer.test.tsx | 29 +- .../lookoutV2/JobsTableContainer.tsx | 68 ++- .../JobsTablePreferencesService.test.ts | 482 +++++++++++++++++- .../lookoutV2/JobsTablePreferencesService.ts | 38 +- .../lookout/ui/src/utils/jobsTableColumns.tsx | 10 +- internal/lookout/ui/yarn.lock | 41 +- 17 files changed, 1216 insertions(+), 434 deletions(-) create mode 100644 internal/lookout/ui/src/components/lookoutV2/ColumnConfigurationDialog/AddAnnotationColumnInput.tsx create mode 100644 internal/lookout/ui/src/components/lookoutV2/ColumnConfigurationDialog/EditAnnotationColumnInput.tsx create mode 100644 internal/lookout/ui/src/components/lookoutV2/ColumnConfigurationDialog/OrderableColumnListItem.tsx create mode 100644 internal/lookout/ui/src/components/lookoutV2/ColumnConfigurationDialog/index.tsx delete mode 100644 internal/lookout/ui/src/components/lookoutV2/ColumnSelect.module.css delete mode 100644 internal/lookout/ui/src/components/lookoutV2/ColumnSelect.tsx diff --git a/internal/lookout/ui/package.json b/internal/lookout/ui/package.json index 1dc8345ff74..a503197ced5 100644 --- a/internal/lookout/ui/package.json +++ b/internal/lookout/ui/package.json @@ -18,6 +18,9 @@ "fmt": "eslint './src/**/*.{js,ts,tsx}' --max-warnings 0 --fix" }, "dependencies": { + "@dnd-kit/core": "^6.3.1", + "@dnd-kit/modifiers": "^9.0.0", + "@dnd-kit/sortable": "^10.0.0", "@emotion/react": "^11.13.5", "@emotion/styled": "^11.13.5", "@fortawesome/fontawesome-common-types": "^6.7.1", diff --git a/internal/lookout/ui/src/components/lookoutV2/ColumnConfigurationDialog/AddAnnotationColumnInput.tsx b/internal/lookout/ui/src/components/lookoutV2/ColumnConfigurationDialog/AddAnnotationColumnInput.tsx new file mode 100644 index 00000000000..a98b1df957e --- /dev/null +++ b/internal/lookout/ui/src/components/lookoutV2/ColumnConfigurationDialog/AddAnnotationColumnInput.tsx @@ -0,0 +1,69 @@ +import { useState } from "react" + +import { AddCircle } from "@mui/icons-material" +import { FormControl, FormHelperText, IconButton, InputAdornment, InputLabel, OutlinedInput } from "@mui/material" + +const INPUT_LABEL_TEXT = "Annotation key" +const INPUT_ID = "annotation-key" + +export interface AddAnnotationColumnInputProps { + onCreate: (annotationKey: string) => void + existingAnnotationColumnKeysSet: Set +} + +export const AddAnnotationColumnInput = ({ + onCreate, + existingAnnotationColumnKeysSet, +}: AddAnnotationColumnInputProps) => { + const [value, setValue] = useState("") + + const isValueFilled = value.length !== 0 + + const valueHasNoLeadingTrailingWhitespace = value.trim() === value + const valueIsNew = !existingAnnotationColumnKeysSet.has(value) + const isValueValid = [valueHasNoLeadingTrailingWhitespace, valueIsNew].every(Boolean) + + const handleCreate = () => { + onCreate(value) + setValue("") + } + + return ( + + {INPUT_LABEL_TEXT} + { + setValue(target.value) + }} + onKeyDown={(event) => { + if (event.key === "Enter" && isValueFilled && isValueValid) { + handleCreate() + event.currentTarget.blur() + event.preventDefault() + } + }} + endAdornment={ + isValueFilled && isValueValid ? ( + + + + + + ) : undefined + } + /> + {!valueHasNoLeadingTrailingWhitespace && ( + The annotation key must not have leading or trailing whitespace. + )} + {!valueIsNew && A column for this annotation key already exists.} + + ) +} diff --git a/internal/lookout/ui/src/components/lookoutV2/ColumnConfigurationDialog/EditAnnotationColumnInput.tsx b/internal/lookout/ui/src/components/lookoutV2/ColumnConfigurationDialog/EditAnnotationColumnInput.tsx new file mode 100644 index 00000000000..d8edf03e2a2 --- /dev/null +++ b/internal/lookout/ui/src/components/lookoutV2/ColumnConfigurationDialog/EditAnnotationColumnInput.tsx @@ -0,0 +1,69 @@ +import { useState } from "react" + +import { Check } from "@mui/icons-material" +import { FormControl, FormHelperText, IconButton, InputAdornment, InputLabel, OutlinedInput } from "@mui/material" + +const INPUT_LABEL_TEXT = "Annotation key" +const INPUT_ID = "annotation-key" + +export interface EditAnnotationColumnInputProps { + onEdit: (annotationKey: string) => void + currentAnnotationKey: string + existingAnnotationColumnKeys: Set +} + +export const EditAnnotationColumnInput = ({ + onEdit, + existingAnnotationColumnKeys, + currentAnnotationKey, +}: EditAnnotationColumnInputProps) => { + const [value, setValue] = useState(currentAnnotationKey) + + const isValueFilled = value.length !== 0 + + const valueHasNoLeadingTrailingWhitespace = value.trim() === value + const valueIsUnique = !existingAnnotationColumnKeys.has(value) || value === currentAnnotationKey + const isValueValid = [valueHasNoLeadingTrailingWhitespace, valueIsUnique].every(Boolean) + + const handleEdit = () => { + onEdit(value) + } + + return ( + + {INPUT_LABEL_TEXT} + { + setValue(target.value) + }} + onKeyDown={(event) => { + event.stopPropagation() + if (event.key === "Enter" && isValueFilled && isValueValid) { + handleEdit() + } + }} + endAdornment={ + isValueFilled && isValueValid ? ( + + + + + + ) : undefined + } + /> + {!valueHasNoLeadingTrailingWhitespace && ( + The annotation key must not have leading or trailing whitespace. + )} + {!valueIsUnique && A column for this annotation key already exists.} + + ) +} diff --git a/internal/lookout/ui/src/components/lookoutV2/ColumnConfigurationDialog/OrderableColumnListItem.tsx b/internal/lookout/ui/src/components/lookoutV2/ColumnConfigurationDialog/OrderableColumnListItem.tsx new file mode 100644 index 00000000000..00bd3520392 --- /dev/null +++ b/internal/lookout/ui/src/components/lookoutV2/ColumnConfigurationDialog/OrderableColumnListItem.tsx @@ -0,0 +1,130 @@ +import { useState } from "react" + +import { useSortable } from "@dnd-kit/sortable" +import { CSS } from "@dnd-kit/utilities" +import { Cancel, Delete, DragHandle, Edit } from "@mui/icons-material" +import { + Checkbox, + IconButton, + ListItem, + ListItemButton, + ListItemIcon, + ListItemText, + Stack, + styled, +} from "@mui/material" + +import { EditAnnotationColumnInput } from "./EditAnnotationColumnInput" +import { SPACING } from "../../../styling/spacing" +import { + AnnotationColumnId, + fromAnnotationColId, + getColumnMetadata, + JobTableColumn, + toColId, +} from "../../../utils/jobsTableColumns" + +const GrabListItemIcon = styled(ListItemIcon)({ + cursor: "grab", + touchAction: "none", +}) + +const EditAnnotationColumnInputContainer = styled(Stack)({ + width: "100%", +}) + +export interface OrderableColumnListItemProps { + column: JobTableColumn + isVisible: boolean + onToggleVisibility: () => void + removeAnnotationColumn: () => void + editAnnotationColumn: (annotationKey: string) => void + existingAnnotationColumnKeysSet: Set +} + +export const OrderableColumnListItem = ({ + column, + isVisible, + onToggleVisibility, + removeAnnotationColumn, + editAnnotationColumn, + existingAnnotationColumnKeysSet, +}: OrderableColumnListItemProps) => { + const colId = toColId(column.id) + const colMetadata = getColumnMetadata(column) + const colIsAnnotation = colMetadata.annotation ?? false + const { attributes, listeners, setNodeRef, transform, transition } = useSortable({ + id: colId, + }) + + const [isEditing, setIsEditing] = useState(false) + + return ( + + { + setIsEditing(true) + }} + > + + + + + + + ) : undefined + } + ref={setNodeRef} + {...attributes} + {...listeners} + > + + + + {colIsAnnotation && isEditing ? ( + + { + editAnnotationColumn(annotationKey) + setIsEditing(false) + }} + currentAnnotationKey={fromAnnotationColId(colId as AnnotationColumnId)} + existingAnnotationColumnKeys={existingAnnotationColumnKeysSet} + /> +
+ setIsEditing(false)} title="Cancel changes"> + + +
+ + ) : ( + + + + + + + )} + + ) +} diff --git a/internal/lookout/ui/src/components/lookoutV2/ColumnConfigurationDialog/index.tsx b/internal/lookout/ui/src/components/lookoutV2/ColumnConfigurationDialog/index.tsx new file mode 100644 index 00000000000..cfd26aa003c --- /dev/null +++ b/internal/lookout/ui/src/components/lookoutV2/ColumnConfigurationDialog/index.tsx @@ -0,0 +1,247 @@ +import { PointerEvent, useCallback, useMemo, useRef, useState } from "react" + +import { DndContext, DragEndEvent, KeyboardSensor, PointerSensor, useSensor, useSensors } from "@dnd-kit/core" +import { restrictToWindowEdges, restrictToVerticalAxis } from "@dnd-kit/modifiers" +import { arrayMove, SortableContext } from "@dnd-kit/sortable" +import { ArrowDownward, Close } from "@mui/icons-material" +import { + Alert, + Chip, + Dialog, + DialogContent, + DialogContentText, + DialogTitle, + IconButton, + List, + Stack, + styled, + Typography, +} from "@mui/material" + +import { AddAnnotationColumnInput } from "./AddAnnotationColumnInput" +import { OrderableColumnListItem } from "./OrderableColumnListItem" +import { SPACING } from "../../../styling/spacing" +import { + AnnotationColumnId, + ColumnId, + fromAnnotationColId, + getColumnMetadata, + JobTableColumn, + PINNED_COLUMNS, + toColId, +} from "../../../utils/jobsTableColumns" + +const ScrollToAddAnnotationColumnChip = styled(Chip)({ + zIndex: 100, + + position: "absolute", + left: "50%", + bottom: 10, + transform: "translate(-50%, 0)", +}) + +const StyledDialogTitle = styled(DialogTitle)({ + paddingRight: 10, +}) + +const CloseIconButton = styled(IconButton)(({ theme }) => ({ + position: "absolute", + top: 8, + right: 8, + color: theme.palette.text.secondary, +})) + +class ColumnListPointerSensor extends PointerSensor { + static activators = [ + { + eventName: "onPointerDown", + handler: ({ nativeEvent: event }: PointerEvent) => { + // Block DnD event propagation if element has "data-no-dnd" attribute + let cur = event.target as HTMLElement + while (cur) { + if (cur.dataset && cur.dataset.noDnd) { + return false + } + cur = cur.parentElement as HTMLElement + } + return true + }, + }, + ] as (typeof PointerSensor)["activators"] +} + +export interface ColumnConfigurationDialogProps { + open: boolean + onClose: () => void + allColumns: JobTableColumn[] + groupedColumnIds: ColumnId[] + visibleColumnIds: ColumnId[] + columnOrderIds: ColumnId[] + setColumnOrder: (columnOrder: ColumnId[]) => void + toggleColumnVisibility: (columnId: ColumnId) => void + onAddAnnotationColumn: (annotationKey: string) => void + onRemoveAnnotationColumn: (colId: ColumnId) => void + onEditAnnotationColumn: (colId: ColumnId, annotationKey: string) => void +} + +export const ColumnConfigurationDialog = ({ + open, + onClose, + allColumns, + groupedColumnIds, + visibleColumnIds, + columnOrderIds, + setColumnOrder, + toggleColumnVisibility, + onAddAnnotationColumn, + onRemoveAnnotationColumn, + onEditAnnotationColumn, +}: ColumnConfigurationDialogProps) => { + const allColumnsById = useMemo( + () => + allColumns.reduce( + (acc, column) => { + acc[toColId(column.id)] = column + return acc + }, + {} as Record, + ), + [allColumns], + ) + + const orderedColumns = useMemo(() => { + const groupedColumnsSet = new Set(groupedColumnIds) + return columnOrderIds + .filter((id) => !groupedColumnsSet.has(id) && !PINNED_COLUMNS.includes(toColId(id)) && id in allColumnsById) + .map((id) => allColumnsById[id]) + }, [columnOrderIds, groupedColumnIds, allColumnsById]) + + const visibleColumnsSet = useMemo(() => new Set(visibleColumnIds), [visibleColumnIds]) + + const handleDragEnd = useCallback( + ({ active, over }: DragEndEvent) => { + if (over && active.id !== over.id) { + const oldIndex = columnOrderIds.indexOf(active.id as ColumnId) + const newIndex = columnOrderIds.indexOf(over.id as ColumnId) + setColumnOrder(arrayMove(columnOrderIds, oldIndex, newIndex)) + } + }, + [columnOrderIds, setColumnOrder], + ) + + const annotationColumnKeysSet = useMemo( + () => + new Set( + allColumns + .filter((column) => getColumnMetadata(column).annotation) + .map(({ id }) => fromAnnotationColId(id as AnnotationColumnId)), + ), + [allColumns], + ) + + const pointerSensor = useSensor(ColumnListPointerSensor) + const keyboardSensor = useSensor(KeyboardSensor) + const sensors = useSensors(pointerSensor, keyboardSensor) + + const [isAddAnnotationColumnContainerVisible, setIsAddAnnotationColumnContainerVisible] = useState(false) + const addAnnotationColumnContainerObserver = useRef() + const scrollIntoViewAddAnnotationColumnContainer = useRef<() => void>() + const addAnnotationColumnContainerRef = useCallback((node: HTMLDivElement) => { + if (addAnnotationColumnContainerObserver.current) { + addAnnotationColumnContainerObserver.current.disconnect() + } + + addAnnotationColumnContainerObserver.current = new IntersectionObserver(([entry]) => { + setIsAddAnnotationColumnContainerVisible(entry.isIntersecting) + }) + + if (node) { + addAnnotationColumnContainerObserver.current.observe(node) + scrollIntoViewAddAnnotationColumnContainer.current = () => { + node.scrollIntoView({ behavior: "smooth" }) + } + } + }, []) + + return ( + + {!isAddAnnotationColumnContainerVisible && ( + } + color="primary" + /> + )} + Column configuration + + + + + + + Select which columns to view, any additional annotation columns, and the order of the columns. + + {groupedColumnIds.length > 0 && ( + <> + + The following columns cannot be hidden or re-ordered because they are currently grouped: + + + {groupedColumnIds + .map((id) => allColumnsById[id]) + .map((column) => { + const { displayName } = getColumnMetadata(column) + return ( + + {displayName} + + ) + })} + + + )} + + Click and drag the columns into your desired order. + +
+ + toColId(id))}> + + {orderedColumns.map((column) => { + const colId = toColId(column.id) + return ( + toggleColumnVisibility(colId)} + removeAnnotationColumn={() => onRemoveAnnotationColumn(colId)} + editAnnotationColumn={(annotationKey) => onEditAnnotationColumn(colId, annotationKey)} + existingAnnotationColumnKeysSet={annotationColumnKeysSet} + /> + ) + })} + + + +
+
+ Add annotation column + + Annotations are metadata (key-value pairs) that you can add to your job. + + +
+ + + + ) +} diff --git a/internal/lookout/ui/src/components/lookoutV2/ColumnSelect.module.css b/internal/lookout/ui/src/components/lookoutV2/ColumnSelect.module.css deleted file mode 100644 index ca1a3fb7c05..00000000000 --- a/internal/lookout/ui/src/components/lookoutV2/ColumnSelect.module.css +++ /dev/null @@ -1,31 +0,0 @@ -.columnMenu { - display: flex; - flex-direction: row; -} - -.columnSelect { - flex: 1 0 auto; - max-width: 500px; - overflow-y: auto; -} - -.annotationSelectContainer { - flex: 0 0 250px; - padding-left: 8px; - padding-right: 8px; - padding-top: 8px; -} - -.addColumnButton { - margin: 8px; -} - -.addAnnotationButtons { - display: flex; - flex-direction: row; - justify-content: space-between; -} - -.addAnnotationAction { - margin: 8px; -} diff --git a/internal/lookout/ui/src/components/lookoutV2/ColumnSelect.tsx b/internal/lookout/ui/src/components/lookoutV2/ColumnSelect.tsx deleted file mode 100644 index d63001b1316..00000000000 --- a/internal/lookout/ui/src/components/lookoutV2/ColumnSelect.tsx +++ /dev/null @@ -1,255 +0,0 @@ -import { useRef, useState } from "react" - -import { ArrowDropDown, ArrowDropUp, Check, Delete, Edit } from "@mui/icons-material" -import { - Button, - Checkbox, - FormControl, - IconButton, - InputAdornment, - InputLabel, - ListItemText, - MenuItem, - OutlinedInput, - Popover, - TextField, - Typography, -} from "@mui/material" - -import styles from "./ColumnSelect.module.css" -import { ColumnId, getColumnMetadata, JobTableColumn, StandardColumnId, toColId } from "../../utils/jobsTableColumns" - -type ColumnSelectProps = { - selectableColumns: JobTableColumn[] - groupedColumns: ColumnId[] - visibleColumns: ColumnId[] - onAddAnnotation: (annotationKey: string) => void - onToggleColumn: (columnId: ColumnId) => void - onRemoveAnnotation: (columnId: ColumnId) => void - onEditAnnotation: (columnId: ColumnId, newDisplayName: string) => void -} - -export default function ColumnSelect({ - selectableColumns, - groupedColumns, - visibleColumns, - onAddAnnotation, - onToggleColumn, - onRemoveAnnotation, - onEditAnnotation, -}: ColumnSelectProps) { - const anchorEl = useRef(null) - const [isOpen, setIsOpen] = useState(false) - - const [creatingAnnotation, setCreatingAnnotation] = useState(false) - const [newAnnotationKey, setNewAnnotationKey] = useState("") - - const [currentlyEditing, setCurrentlyEditing] = useState(new Map()) - - function clearAddAnnotation() { - setCreatingAnnotation(false) - setNewAnnotationKey("") - } - - function saveNewAnnotation() { - onAddAnnotation(newAnnotationKey.trim()) - clearAddAnnotation() - } - - function edit(key: string, name: string) { - const newCurrentlyEditing = new Map(currentlyEditing) - newCurrentlyEditing.set(key, name) - setCurrentlyEditing(newCurrentlyEditing) - } - - function stopEditing(key: string) { - if (currentlyEditing.has(key)) { - const newCurrentlyEditing = new Map(currentlyEditing) - newCurrentlyEditing.delete(key) - setCurrentlyEditing(newCurrentlyEditing) - } - } - - return ( - <> - - - Columns - - setIsOpen(true)} - type={"button"} - label={"Columns"} - value={`${ - selectableColumns.filter((col) => (visibleColumns as string[]).includes(col.id ?? "")).length - } columns selected`} - endAdornment={{isOpen ? : }} - style={{ paddingRight: 5 }} - /> - setIsOpen(false)} - anchorEl={anchorEl.current} - anchorOrigin={{ - vertical: "bottom", - horizontal: "center", - }} - transformOrigin={{ - vertical: "top", - horizontal: "center", - }} - > -
-
- {selectableColumns.map((column) => { - const colId = toColId(column.id) - const colIsGrouped = groupedColumns.includes(colId) - const colIsVisible = visibleColumns.includes(colId) - const colMetadata = getColumnMetadata(column) - const colIsAnnotation = colMetadata.annotation ?? false - return ( - { - if (!colIsAnnotation) { - onToggleColumn(colId) - } - }} - key={colId} - value={colMetadata.displayName} - disabled={colIsGrouped || colId === StandardColumnId.Count} - > - onToggleColumn(colId)} /> - {colIsAnnotation ? ( - <> - {currentlyEditing.has(colId) ? ( - <> - edit(colId, e.target.value)} - style={{ - maxWidth: 350, - }} - fullWidth={true} - /> - { - if (currentlyEditing.has(colId)) { - onEditAnnotation(colId, currentlyEditing.get(colId) ?? "") - } - stopEditing(colId) - }} - > - - - - ) : ( - <> - - edit(colId, colMetadata.displayName)}> - - - - )} - { - stopEditing(colId) - onRemoveAnnotation(colId) - }} - > - - - - ) : ( - - )} - - ) - })} -
-
- - Click here to add an annotation column. - - - Annotations are metadata (key-value pairs) that you can add to your job. - -
- {creatingAnnotation ? ( - <> - { - setNewAnnotationKey(e.target.value) - }} - onKeyUp={(e) => { - if (e.key === "Enter") { - saveNewAnnotation() - } - }} - /> -
-
- -
-
- -
-
- - ) : ( - - )} -
-
-
- - - - ) -} diff --git a/internal/lookout/ui/src/components/lookoutV2/CustomViewPicker.module.css b/internal/lookout/ui/src/components/lookoutV2/CustomViewPicker.module.css index 96c2d026e9e..c5d3eb77661 100644 --- a/internal/lookout/ui/src/components/lookoutV2/CustomViewPicker.module.css +++ b/internal/lookout/ui/src/components/lookoutV2/CustomViewPicker.module.css @@ -1,4 +1,4 @@ -.settingsButton { +.customizeButton { min-width: 50px; display: flex; justify-content: center; diff --git a/internal/lookout/ui/src/components/lookoutV2/CustomViewPicker.tsx b/internal/lookout/ui/src/components/lookoutV2/CustomViewPicker.tsx index 93bee0ac871..f8608212581 100644 --- a/internal/lookout/ui/src/components/lookoutV2/CustomViewPicker.tsx +++ b/internal/lookout/ui/src/components/lookoutV2/CustomViewPicker.tsx @@ -1,6 +1,6 @@ import { useRef, useState } from "react" -import { Delete, Settings } from "@mui/icons-material" +import { DashboardCustomize, Delete } from "@mui/icons-material" import { Button, Popover, @@ -48,9 +48,9 @@ export const CustomViewPicker = ({ return ( <> -
- setIsOpen(!isOpen)}> - +
+ setIsOpen(!isOpen)}> +
void selectedItemFilters: JobFilter[][] + filtersActive: boolean customViews: string[] activeJobSets: boolean onActiveJobSetsChanged: (newVal: boolean) => void @@ -48,7 +51,10 @@ export const JobsTableActionBar = memo( allColumns, groupedColumns, visibleColumns, + columnOrder, + setColumnOrder, selectedItemFilters, + filtersActive, customViews, activeJobSets, onActiveJobSetsChanged, @@ -68,18 +74,38 @@ export const JobsTableActionBar = memo( onDeleteCustomView, onLoadCustomView, }: JobsTableActionBarProps) => { + const [columnConfigurationDialogOpen, setColumnConfigurationDialogOpen] = useState(false) const [cancelDialogOpen, setCancelDialogOpen] = useState(false) const [reprioritiseDialogOpen, setReprioritiseDialogOpen] = useState(false) - const openSnackbar = useCustomSnackbar() - const selectableColumns = useMemo(() => allColumns.filter((col) => col.enableHiding !== false), [allColumns]) + const numberSelectedColumns = useMemo(() => { + const visibleColumnsSet = new Set(visibleColumns) + return allColumns.filter((col) => { + const colId = toColId(col.id) + return !PINNED_COLUMNS.includes(colId) && visibleColumnsSet.has(colId) + }).length + }, [allColumns, visibleColumns]) const numSelectedItems = selectedItemFilters.length + const columnConfigurationDialogOpenOnClose = useCallback(() => setColumnConfigurationDialogOpen(false), []) const cancelDialogOnClose = useCallback(() => setCancelDialogOpen(false), []) const reprioritiseDialogOnClose = useCallback(() => setReprioritiseDialogOpen(false), []) return (
+ {cancelDialogOpen && ( )} - +
+ +
- - - - { - try { - onAddAnnotationColumn(annotationKey) - } catch (e) { - const err = e as Error - console.error(err.message) - openSnackbar(`Failed to create annotation column: ${err.message}`, "error") - } - }} - onRemoveAnnotation={(columnId) => { - try { - onRemoveAnnotationColumn(columnId) - } catch (e) { - const err = e as Error - console.error(err.message) - openSnackbar(`Failed to remove annotation column: ${err.message}`, "error") - } - }} - onEditAnnotation={(columnId, annotationKey) => { - try { - onEditAnnotationColumn(columnId, annotationKey) - } catch (e) { - const err = e as Error - console.error(err.message) - openSnackbar(`Failed to edit annotation column: ${err.message}`, "error") - } - }} - onToggleColumn={toggleColumnVisibility} - /> +
+ +
+
+ +
+
+ +
+
+ +
- - +
+ +
+
+ +
) diff --git a/internal/lookout/ui/src/containers/lookoutV2/JobsTableContainer.test.tsx b/internal/lookout/ui/src/containers/lookoutV2/JobsTableContainer.test.tsx index 019f61ac397..056d9e0145e 100644 --- a/internal/lookout/ui/src/containers/lookoutV2/JobsTableContainer.test.tsx +++ b/internal/lookout/ui/src/containers/lookoutV2/JobsTableContainer.test.tsx @@ -23,6 +23,12 @@ import FakeGetJobsService from "../../services/lookoutV2/mocks/FakeGetJobsServic import { FakeGetRunInfoService } from "../../services/lookoutV2/mocks/FakeGetRunInfoService" import FakeGroupJobsService from "../../services/lookoutV2/mocks/FakeGroupJobsService" +const intersectionObserverMock = () => ({ + observe: () => null, + disconnect: () => null, +}) +window.IntersectionObserver = vi.fn().mockImplementation(intersectionObserverMock) + vi.setConfig({ // This is quite a heavy component, and tests can timeout on a slower machine testTimeout: 30_000, @@ -177,15 +183,14 @@ describe("JobsTableContainer", () => { ["Job Set", "jobSet"], ["Queue", "queue"], ["State", "state"], - ])("should allow grouping by %s", async (displayString, groupKey) => { + ] as [string, keyof Job][])("should allow grouping by %s", async (displayString, groupKey) => { const jobs = [ ...makeTestJobs(5, "queue-1", "job-set-1", JobState.Queued), ...makeTestJobs(10, "queue-2", "job-set-2", JobState.Pending), ...makeTestJobs(15, "queue-3", "job-set-3", JobState.Running), ] - const jobObjKey = groupKey as keyof Job - const numUniqueForJobKey = new Set(jobs.map((j) => j[jobObjKey])).size + const numUniqueForJobKey = new Set(jobs.map((j) => j[groupKey])).size renderComponent(jobs) await waitForFinishedLoading() @@ -197,10 +202,10 @@ describe("JobsTableContainer", () => { // Expand a row const job = jobs[0] - await expandRow(job[jobObjKey]!.toString()) + await expandRow(job[groupKey]!.toString()) // Check the right number of rows is being shown - const numShownJobs = jobs.filter((j) => j[jobObjKey] === job[jobObjKey]).length + const numShownJobs = jobs.filter((j) => j[groupKey] === job[groupKey]).length await assertNumDataRowsShown(numUniqueForJobKey + numShownJobs) }) @@ -682,21 +687,17 @@ describe("JobsTableContainer", () => { } async function addAnnotationColumn(annotationKey: string) { - const editColumnsButton = await screen.findByRole("button", { name: /columns selected/i }) + const editColumnsButton = await screen.findByRole("button", { name: /configure columns/i }) await userEvent.click(editColumnsButton) - const addColumnButton = await screen.findByRole("button", { name: /Add column/i }) - await userEvent.click(addColumnButton) - - const textbox = await screen.findByRole("textbox", { name: /Annotation key/i }) + const textbox = await screen.findByRole("textbox", { name: /annotation key/i }) await userEvent.type(textbox, annotationKey) - const saveButton = await screen.findByRole("button", { name: /Save/i }) + const saveButton = await screen.findByRole("button", { name: /add a column for annotation/i }) await userEvent.click(saveButton) - // Close pop up - await userEvent.click(screen.getByText(/Click here to add an annotation column/i)) - await userEvent.keyboard("{Escape}") + const closeButton = await screen.findByRole("button", { name: /close/i }) + await userEvent.click(closeButton) } async function clickOnJobRow(jobId: string) { diff --git a/internal/lookout/ui/src/containers/lookoutV2/JobsTableContainer.tsx b/internal/lookout/ui/src/containers/lookoutV2/JobsTableContainer.tsx index 2df50fc58fa..569222b1d59 100644 --- a/internal/lookout/ui/src/containers/lookoutV2/JobsTableContainer.tsx +++ b/internal/lookout/ui/src/containers/lookoutV2/JobsTableContainer.tsx @@ -61,6 +61,7 @@ import { INPUT_PROCESSORS, JOB_COLUMNS, JobTableColumn, + PINNED_COLUMNS, StandardColumnId, toAnnotationColId, toColId, @@ -155,6 +156,7 @@ export const JobsTableContainer = ({ ) const columnResizeMode = useMemo(() => "onChange" as ColumnResizeMode, []) const [columnSizing, setColumnSizing] = useState(initialPrefs.columnSizing ?? {}) + const [columnOrder, setColumnOrder] = useState(initialPrefs.columnOrder) // Grouping const [grouping, setGrouping] = useState(initialPrefs.groupedColumns) @@ -245,24 +247,23 @@ export const JobsTableContainer = ({ [sidebarJobId, jobInfoMap], ) - const prefsFromState = (): JobsTablePreferences => { - return { - groupedColumns: grouping, - expandedState: expanded, - pageIndex, - pageSize, - order: lookoutOrder, - columnSizing: columnSizing, - filters: columnFilterState, - columnMatches: columnMatches, - annotationColumnKeys: getAnnotationKeyCols(allColumns), - visibleColumns: columnVisibility, - sidebarJobId: sidebarJobId, - sidebarWidth: sidebarWidth, - activeJobSets: activeJobSets, - autoRefresh: autoRefresh, - } - } + const prefsFromState = (): JobsTablePreferences => ({ + groupedColumns: grouping, + expandedState: expanded, + columnOrder, + pageIndex, + pageSize, + order: lookoutOrder, + columnSizing: columnSizing, + filters: columnFilterState, + columnMatches: columnMatches, + annotationColumnKeys: getAnnotationKeyCols(allColumns), + visibleColumns: columnVisibility, + sidebarJobId: sidebarJobId, + sidebarWidth: sidebarWidth, + activeJobSets: activeJobSets, + autoRefresh: autoRefresh, + }) const setTextFields = (filters: ColumnFiltersState) => { const filterMap = Object.fromEntries(filters.map((f) => [f.id, f])) @@ -300,6 +301,7 @@ export const JobsTableContainer = ({ setColumnMatches(prefs.columnMatches) const cols = JOB_COLUMNS.concat(...prefs.annotationColumnKeys.map(createAnnotationColumn)) setAllColumns(cols) + setColumnOrder(prefs.columnOrder) setColumnVisibility(prefs.visibleColumns) setSidebarJobId(prefs.sidebarJobId) setSidebarWidth(prefs.sidebarWidth ?? 600) @@ -425,10 +427,12 @@ export const JobsTableContainer = ({ } const annotationCol = createAnnotationColumn(annotationKey) + const annotationColId = toColId(annotationCol.id) const newCols = allColumns.concat([annotationCol]) + setColumnOrder((prev) => [...prev, annotationColId]) setAllColumns(newCols) - if (!colIsVisible(toColId(annotationCol.id))) { - onColumnVisibilityChange(toColId(annotationCol.id)) + if (!colIsVisible(annotationColId)) { + onColumnVisibilityChange(annotationColId) } } @@ -437,6 +441,7 @@ export const JobsTableContainer = ({ if (filtered.length === allColumns.length) { throw new Error(`column "${colId}" was not removed`) } + setColumnOrder((prev) => prev.filter((prevId) => prevId !== colId)) setAllColumns(filtered) onFilterChange((columnFilters) => { return columnFilters.filter((columnFilter) => columnFilter.id !== colId) @@ -453,16 +458,25 @@ export const JobsTableContainer = ({ if (index === -1) { throw new Error(`column "${colId}" not found`) } + + const oldColId = toColId(allColumns[index].id) + // Make old column not visible - if (colIsVisible(toColId(allColumns[index].id))) { - onColumnVisibilityChange(toColId(allColumns[index].id)) + if (colIsVisible(oldColId)) { + onColumnVisibilityChange(oldColId) } + allColumns[index] = createAnnotationColumn(annotationKey) + const newColId = toColId(allColumns[index].id) + // Make new column visible setAllColumns([...allColumns]) - if (!colIsVisible(toColId(allColumns[index].id))) { - onColumnVisibilityChange(toColId(allColumns[index].id)) + if (!colIsVisible(toColId(newColId))) { + onColumnVisibilityChange(toColId(newColId)) } + + // Change column ID in ordering + setColumnOrder((prev) => prev.map((prevId) => (prevId === oldColId ? newColId : prevId))) } const onGroupingChange = useCallback( @@ -644,10 +658,11 @@ export const JobsTableContainer = ({ columnFilters: columnFilterState, rowSelection: selectedRows, columnPinning: { - left: [StandardColumnId.SelectorCol], + left: PINNED_COLUMNS, }, columnVisibility, sorting, + columnOrder, columnSizing, }, getCoreRowModel: getCoreRowModel(), @@ -755,7 +770,10 @@ export const JobsTableContainer = ({ isLoading={rowsToFetch.length > 0} allColumns={columnsForSelect} groupedColumns={grouping} + filtersActive={columnFilterState.length > 0} visibleColumns={visibleColumnIds} + columnOrder={columnOrder} + setColumnOrder={setColumnOrder} selectedItemFilters={selectedItemsFilters} customViews={customViews} activeJobSets={activeJobSets} diff --git a/internal/lookout/ui/src/services/lookoutV2/JobsTablePreferencesService.test.ts b/internal/lookout/ui/src/services/lookoutV2/JobsTablePreferencesService.test.ts index 06c85a41933..9d4669c6e98 100644 --- a/internal/lookout/ui/src/services/lookoutV2/JobsTablePreferencesService.test.ts +++ b/internal/lookout/ui/src/services/lookoutV2/JobsTablePreferencesService.test.ts @@ -2,6 +2,7 @@ import { Location, NavigateFunction, Params } from "react-router-dom" import { DEFAULT_PREFERENCES, + ensurePreferencesAreConsistent, JobsTablePreferences, JobsTablePreferencesService, PREFERENCES_KEY, @@ -10,7 +11,7 @@ import { } from "./JobsTablePreferencesService" import { Match } from "../../models/lookoutV2Models" import { Router } from "../../utils" -import { ColumnId, DEFAULT_COLUMN_ORDER, StandardColumnId } from "../../utils/jobsTableColumns" +import { ColumnId, DEFAULT_COLUMN_ORDERING, StandardColumnId } from "../../utils/jobsTableColumns" class FakeRouter implements Router { location: Location @@ -58,8 +59,8 @@ describe("JobsTablePreferencesService", () => { ...DEFAULT_PREFERENCES, // From query string above pageIndex: 3, - groupedColumns: ["state" as ColumnId], - order: { id: "jobId", direction: "ASC" }, + groupedColumns: [StandardColumnId.State], + order: { id: StandardColumnId.JobID, direction: "ASC" }, }) }) }) @@ -93,9 +94,9 @@ describe("JobsTablePreferencesService", () => { describe("Grouped columns", () => { it("round-trips columns", () => { - savePartialPrefs({ groupedColumns: ["queue", "state"] as ColumnId[] }) + savePartialPrefs({ groupedColumns: [StandardColumnId.Queue, StandardColumnId.State] }) expect(router.location.search).toContain("g[0]=queue&g[1]=state") - expect(service.getUserPrefs().groupedColumns).toStrictEqual(["queue", "state"]) + expect(service.getUserPrefs().groupedColumns).toStrictEqual([StandardColumnId.Queue, StandardColumnId.State]) }) it("round-trips empty list", () => { @@ -127,27 +128,31 @@ describe("JobsTablePreferencesService", () => { describe("Column filters", () => { it("round-trips column filters", () => { - savePartialPrefs({ filters: [{ id: "queue", value: "test" }] }) + savePartialPrefs({ filters: [{ id: StandardColumnId.Queue, value: "test" }] }) expect(router.location.search).toContain("f[0][id]=queue&f[0][value]=test&f[0][match]=startsWith") - expect(service.getUserPrefs().filters).toStrictEqual([{ id: "queue", value: "test" }]) + expect(service.getUserPrefs().filters).toStrictEqual([{ id: StandardColumnId.Queue, value: "test" }]) }) it("round-trips state filter", () => { - savePartialPrefs({ filters: [{ id: "state", value: ["QUEUED", "PENDING", "RUNNING"] }] }) + savePartialPrefs({ filters: [{ id: StandardColumnId.State, value: ["QUEUED", "PENDING", "RUNNING"] }] }) expect(router.location.search).toContain( "f[0][id]=state&f[0][value][0]=QUEUED&f[0][value][1]=PENDING&f[0][value][2]=RUNNING&f[0][match]=anyOf", ) - expect(service.getUserPrefs().filters).toStrictEqual([{ id: "state", value: ["QUEUED", "PENDING", "RUNNING"] }]) - expect(service.getUserPrefs().columnMatches["state"] === Match.AnyOf) + expect(service.getUserPrefs().filters).toStrictEqual([ + { id: StandardColumnId.State, value: ["QUEUED", "PENDING", "RUNNING"] }, + ]) + expect(service.getUserPrefs().columnMatches[StandardColumnId.State] === Match.AnyOf) }) it("round-trips special characters", () => { - savePartialPrefs({ filters: [{ id: "queue", value: "test & why / do $ this" }] }) + savePartialPrefs({ filters: [{ id: StandardColumnId.Queue, value: "test & why / do $ this" }] }) expect(router.location.search).toContain( "f[0][id]=queue&f[0][value]=test%20%26%20why%20%2F%20do%20%24%20this&f[0][match]=startsWith", ) - expect(service.getUserPrefs().filters).toStrictEqual([{ id: "queue", value: "test & why / do $ this" }]) - expect(service.getUserPrefs().columnMatches["queue"] === Match.StartsWith) + expect(service.getUserPrefs().filters).toStrictEqual([ + { id: StandardColumnId.Queue, value: "test & why / do $ this" }, + ]) + expect(service.getUserPrefs().columnMatches[StandardColumnId.Queue] === Match.StartsWith) }) it("round-trips empty list", () => { @@ -158,15 +163,15 @@ describe("JobsTablePreferencesService", () => { describe("Sort order", () => { it("round-trips asc sort order", () => { - savePartialPrefs({ order: { id: "queue", direction: "ASC" } }) + savePartialPrefs({ order: { id: StandardColumnId.Queue, direction: "ASC" } }) expect(router.location.search).toContain("sort[id]=queue&sort[desc]=false") - expect(service.getUserPrefs().order).toStrictEqual({ id: "queue", direction: "ASC" }) + expect(service.getUserPrefs().order).toStrictEqual({ id: StandardColumnId.Queue, direction: "ASC" }) }) it("round-trips desc sort order", () => { - savePartialPrefs({ order: { id: "queue", direction: "DESC" } }) + savePartialPrefs({ order: { id: StandardColumnId.Queue, direction: "DESC" } }) expect(router.location.search).toContain("sort[id]=queue&sort[desc]=true") - expect(service.getUserPrefs().order).toStrictEqual({ id: "queue", direction: "DESC" }) + expect(service.getUserPrefs().order).toStrictEqual({ id: StandardColumnId.Queue, direction: "DESC" }) }) }) @@ -273,10 +278,11 @@ describe("JobsTablePreferencesService", () => { const localStorageParams: JobsTablePreferences = { annotationColumnKeys: ["hello"], expandedState: { foo: true }, - filters: [{ id: "jobId", value: "112233" }], + filters: [{ id: StandardColumnId.JobID, value: "112233" }], columnMatches: { jobId: Match.Exact }, - groupedColumns: ["queue" as ColumnId, "jobSet" as ColumnId], - order: { id: "timeInState", direction: "ASC" }, + groupedColumns: [StandardColumnId.Queue, StandardColumnId.JobSet], + columnOrder: ["annotation_hello", StandardColumnId.JobID], + order: { id: StandardColumnId.TimeInState, direction: "ASC" }, pageIndex: 5, pageSize: 20, sidebarJobId: "223344", @@ -291,7 +297,31 @@ describe("JobsTablePreferencesService", () => { expandedState: {}, filters: [], groupedColumns: [], - order: DEFAULT_COLUMN_ORDER, + columnOrder: [ + "annotation_hello", + StandardColumnId.JobID, + StandardColumnId.Queue, + StandardColumnId.Namespace, + StandardColumnId.JobSet, + StandardColumnId.State, + StandardColumnId.Count, + StandardColumnId.Priority, + StandardColumnId.Owner, + StandardColumnId.CPU, + StandardColumnId.Memory, + StandardColumnId.EphemeralStorage, + StandardColumnId.GPU, + StandardColumnId.PriorityClass, + StandardColumnId.LastTransitionTimeUtc, + StandardColumnId.TimeInState, + StandardColumnId.TimeSubmittedUtc, + StandardColumnId.TimeSubmittedAgo, + StandardColumnId.Node, + StandardColumnId.Cluster, + StandardColumnId.ExitCode, + StandardColumnId.RuntimeSeconds, + ], + order: DEFAULT_COLUMN_ORDERING, pageIndex: 0, pageSize: 50, sidebarJobId: "112233", @@ -308,16 +338,17 @@ describe("JobsTablePreferencesService", () => { match: Match.AnyOf, }, ], - sort: { id: "timeInState", desc: "false" }, - g: ["jobSet"], + sort: { id: StandardColumnId.TimeInState, desc: "false" }, + g: [StandardColumnId.JobSet], } const localStorageParams: JobsTablePreferences = { annotationColumnKeys: ["key"], expandedState: { foo: true }, - filters: [{ id: "jobId", value: "112233" }], + filters: [{ id: StandardColumnId.JobID, value: "112233" }], columnMatches: { jobId: Match.Exact }, - groupedColumns: ["queue" as ColumnId, "jobSet" as ColumnId], - order: { id: "timeInState", direction: "ASC" }, + groupedColumns: [StandardColumnId.Queue, StandardColumnId.JobSet], + columnOrder: [StandardColumnId.JobID, "annotation_key"], + order: { id: StandardColumnId.TimeInState, direction: "ASC" }, pageIndex: 5, pageSize: 20, sidebarJobId: "223344", @@ -337,8 +368,32 @@ describe("JobsTablePreferencesService", () => { }, ], columnMatches: { [StandardColumnId.State]: Match.AnyOf }, - groupedColumns: ["jobSet"], - order: { id: "timeInState", direction: "ASC" }, + groupedColumns: [StandardColumnId.JobSet], + columnOrder: [ + StandardColumnId.JobID, + "annotation_key", + StandardColumnId.Queue, + StandardColumnId.Namespace, + StandardColumnId.JobSet, + StandardColumnId.State, + StandardColumnId.Count, + StandardColumnId.Priority, + StandardColumnId.Owner, + StandardColumnId.CPU, + StandardColumnId.Memory, + StandardColumnId.EphemeralStorage, + StandardColumnId.GPU, + StandardColumnId.PriorityClass, + StandardColumnId.LastTransitionTimeUtc, + StandardColumnId.TimeInState, + StandardColumnId.TimeSubmittedUtc, + StandardColumnId.TimeSubmittedAgo, + StandardColumnId.Node, + StandardColumnId.Cluster, + StandardColumnId.ExitCode, + StandardColumnId.RuntimeSeconds, + ], + order: { id: StandardColumnId.TimeInState, direction: "ASC" }, pageIndex: 0, pageSize: 50, sidebarJobId: undefined, @@ -354,3 +409,374 @@ describe("JobsTablePreferencesService", () => { }) } }) + +describe("ensurePreferencesAreConsistent", () => { + it("does not change valid preferences", () => { + const validPreferences: JobsTablePreferences = { + annotationColumnKeys: ["preferenc.es/foo-alpha", "preferenc.es/foo-bravo", "preferenc.es/foo-charlie"], + expandedState: {}, + filters: [{ id: StandardColumnId.JobID, value: "112233" }], + columnMatches: { jobId: Match.Exact }, + groupedColumns: [StandardColumnId.Queue, StandardColumnId.JobSet], + columnOrder: [ + StandardColumnId.Owner, + "annotation_preferenc.es/foo-bravo", + StandardColumnId.JobID, + StandardColumnId.State, + "annotation_preferenc.es/foo-charlie", + StandardColumnId.RuntimeSeconds, + StandardColumnId.Queue, + StandardColumnId.Namespace, + StandardColumnId.JobSet, + StandardColumnId.Priority, + StandardColumnId.Count, + StandardColumnId.Memory, + StandardColumnId.EphemeralStorage, + StandardColumnId.PriorityClass, + StandardColumnId.TimeSubmittedUtc, + StandardColumnId.CPU, + StandardColumnId.ExitCode, + StandardColumnId.LastTransitionTimeUtc, + StandardColumnId.TimeInState, + StandardColumnId.GPU, + StandardColumnId.TimeSubmittedAgo, + StandardColumnId.Cluster, + "annotation_preferenc.es/foo-alpha", + StandardColumnId.Node, + ], + order: { id: StandardColumnId.TimeInState, direction: "ASC" }, + pageIndex: 5, + pageSize: 20, + sidebarJobId: "223344", + visibleColumns: { + "annotation_preferenc.es/foo-charlie": true, + "annotation_preferenc.es/foo-bravo": true, + queue: true, + jobId: true, + jobSet: true, + timeInState: true, + }, + } + + ensurePreferencesAreConsistent(validPreferences) + + const expected: JobsTablePreferences = { + annotationColumnKeys: ["preferenc.es/foo-alpha", "preferenc.es/foo-bravo", "preferenc.es/foo-charlie"], + expandedState: {}, + filters: [{ id: StandardColumnId.JobID, value: "112233" }], + columnMatches: { jobId: Match.Exact }, + groupedColumns: [StandardColumnId.Queue, StandardColumnId.JobSet], + columnOrder: [ + StandardColumnId.Owner, + "annotation_preferenc.es/foo-bravo", + StandardColumnId.JobID, + StandardColumnId.State, + "annotation_preferenc.es/foo-charlie", + StandardColumnId.RuntimeSeconds, + StandardColumnId.Queue, + StandardColumnId.Namespace, + StandardColumnId.JobSet, + StandardColumnId.Priority, + StandardColumnId.Count, + StandardColumnId.Memory, + StandardColumnId.EphemeralStorage, + StandardColumnId.PriorityClass, + StandardColumnId.TimeSubmittedUtc, + StandardColumnId.CPU, + StandardColumnId.ExitCode, + StandardColumnId.LastTransitionTimeUtc, + StandardColumnId.TimeInState, + StandardColumnId.GPU, + StandardColumnId.TimeSubmittedAgo, + StandardColumnId.Cluster, + "annotation_preferenc.es/foo-alpha", + StandardColumnId.Node, + ], + order: { id: StandardColumnId.TimeInState, direction: "ASC" }, + pageIndex: 5, + pageSize: 20, + sidebarJobId: "223344", + visibleColumns: { + "annotation_preferenc.es/foo-charlie": true, + "annotation_preferenc.es/foo-bravo": true, + queue: true, + jobId: true, + jobSet: true, + timeInState: true, + }, + } + + expect(validPreferences).toEqual(expected) + }) + + it("adds annotation key columns for annotations referenced in filters", () => { + const validPreferences: JobsTablePreferences = { + annotationColumnKeys: ["preferenc.es/foo-alpha"], + expandedState: {}, + filters: [{ id: "annotation_preferenc.es/foo-delta", value: "ddd" }], + columnMatches: { "annotation_preferenc.es/foo-delta": Match.StartsWith }, + groupedColumns: [], + columnOrder: [ + StandardColumnId.JobID, + StandardColumnId.Queue, + StandardColumnId.Namespace, + StandardColumnId.JobSet, + StandardColumnId.State, + StandardColumnId.Count, + StandardColumnId.Priority, + StandardColumnId.Owner, + StandardColumnId.CPU, + StandardColumnId.Memory, + StandardColumnId.EphemeralStorage, + StandardColumnId.GPU, + StandardColumnId.PriorityClass, + StandardColumnId.LastTransitionTimeUtc, + StandardColumnId.TimeInState, + StandardColumnId.TimeSubmittedUtc, + StandardColumnId.TimeSubmittedAgo, + StandardColumnId.Node, + StandardColumnId.Cluster, + StandardColumnId.ExitCode, + StandardColumnId.RuntimeSeconds, + "annotation_preferenc.es/foo-alpha", + ], + order: { id: StandardColumnId.TimeInState, direction: "ASC" }, + pageIndex: 5, + pageSize: 20, + sidebarJobId: "223344", + visibleColumns: { + "annotation_preferenc.es/foo-alpha": true, + queue: true, + jobId: true, + jobSet: true, + timeInState: true, + }, + } + + ensurePreferencesAreConsistent(validPreferences) + + const expected: JobsTablePreferences = { + annotationColumnKeys: [ + "preferenc.es/foo-alpha", + "preferenc.es/foo-delta", // added + ], + expandedState: {}, + filters: [{ id: "annotation_preferenc.es/foo-delta", value: "ddd" }], + columnMatches: { "annotation_preferenc.es/foo-delta": Match.StartsWith }, + groupedColumns: [], + columnOrder: [ + StandardColumnId.JobID, + StandardColumnId.Queue, + StandardColumnId.Namespace, + StandardColumnId.JobSet, + StandardColumnId.State, + StandardColumnId.Count, + StandardColumnId.Priority, + StandardColumnId.Owner, + StandardColumnId.CPU, + StandardColumnId.Memory, + StandardColumnId.EphemeralStorage, + StandardColumnId.GPU, + StandardColumnId.PriorityClass, + StandardColumnId.LastTransitionTimeUtc, + StandardColumnId.TimeInState, + StandardColumnId.TimeSubmittedUtc, + StandardColumnId.TimeSubmittedAgo, + StandardColumnId.Node, + StandardColumnId.Cluster, + StandardColumnId.ExitCode, + StandardColumnId.RuntimeSeconds, + "annotation_preferenc.es/foo-alpha", + "annotation_preferenc.es/foo-delta", // added + ], + order: { id: StandardColumnId.TimeInState, direction: "ASC" }, + pageIndex: 5, + pageSize: 20, + sidebarJobId: "223344", + visibleColumns: { + "annotation_preferenc.es/foo-alpha": true, + "annotation_preferenc.es/foo-delta": true, // added + queue: true, + jobId: true, + jobSet: true, + timeInState: true, + }, + } + + expect(validPreferences).toEqual(expected) + }) + + it("makes grouped, ordered and filtered columns are visible", () => { + const validPreferences: JobsTablePreferences = { + annotationColumnKeys: ["preferenc.es/foo-alpha"], + expandedState: {}, + filters: [ + { id: "annotation_preferenc.es/foo-alpha", value: "aaa" }, + { + id: StandardColumnId.State, + value: ["QUEUED", "PENDING", "RUNNING"], + }, + ], + columnMatches: { "annotation_preferenc.es/foo-alpha": Match.StartsWith, [StandardColumnId.State]: Match.AnyOf }, + groupedColumns: [StandardColumnId.Queue, "annotation_preferenc.es/foo-alpha", StandardColumnId.JobSet], + columnOrder: [ + StandardColumnId.JobID, + "annotation_preferenc.es/foo-alpha", + StandardColumnId.Queue, + StandardColumnId.Namespace, + StandardColumnId.JobSet, + StandardColumnId.State, + StandardColumnId.Count, + StandardColumnId.Priority, + StandardColumnId.Owner, + StandardColumnId.CPU, + StandardColumnId.Memory, + StandardColumnId.EphemeralStorage, + StandardColumnId.GPU, + StandardColumnId.PriorityClass, + StandardColumnId.LastTransitionTimeUtc, + StandardColumnId.TimeInState, + StandardColumnId.TimeSubmittedUtc, + StandardColumnId.TimeSubmittedAgo, + StandardColumnId.Node, + StandardColumnId.Cluster, + StandardColumnId.ExitCode, + StandardColumnId.RuntimeSeconds, + ], + order: { id: StandardColumnId.TimeInState, direction: "ASC" }, + pageIndex: 5, + pageSize: 20, + sidebarJobId: "223344", + visibleColumns: {}, + } + + ensurePreferencesAreConsistent(validPreferences) + + const expected: JobsTablePreferences = { + annotationColumnKeys: ["preferenc.es/foo-alpha"], + expandedState: {}, + filters: [ + { id: "annotation_preferenc.es/foo-alpha", value: "aaa" }, + { + id: StandardColumnId.State, + value: ["QUEUED", "PENDING", "RUNNING"], + }, + ], + columnMatches: { "annotation_preferenc.es/foo-alpha": Match.StartsWith, [StandardColumnId.State]: Match.AnyOf }, + groupedColumns: [StandardColumnId.Queue, "annotation_preferenc.es/foo-alpha", StandardColumnId.JobSet], + columnOrder: [ + StandardColumnId.JobID, + "annotation_preferenc.es/foo-alpha", + StandardColumnId.Queue, + StandardColumnId.Namespace, + StandardColumnId.JobSet, + StandardColumnId.State, + StandardColumnId.Count, + StandardColumnId.Priority, + StandardColumnId.Owner, + StandardColumnId.CPU, + StandardColumnId.Memory, + StandardColumnId.EphemeralStorage, + StandardColumnId.GPU, + StandardColumnId.PriorityClass, + StandardColumnId.LastTransitionTimeUtc, + StandardColumnId.TimeInState, + StandardColumnId.TimeSubmittedUtc, + StandardColumnId.TimeSubmittedAgo, + StandardColumnId.Node, + StandardColumnId.Cluster, + StandardColumnId.ExitCode, + StandardColumnId.RuntimeSeconds, + ], + order: { id: StandardColumnId.TimeInState, direction: "ASC" }, + pageIndex: 5, + pageSize: 20, + sidebarJobId: "223344", + // All added + visibleColumns: { + "annotation_preferenc.es/foo-alpha": true, + jobSet: true, + queue: true, + state: true, + timeInState: true, + }, + } + + expect(validPreferences).toEqual(expected) + }) + + it("adds to the column order includes all unpinned standard columns and annotations and removes any columns which are neither", () => { + const validPreferences: JobsTablePreferences = { + annotationColumnKeys: ["preferenc.es/foo-alpha", "preferenc.es/foo-bravo", "preferenc.es/foo-charlie"], + expandedState: {}, + filters: [], + columnMatches: {}, + groupedColumns: [], + columnOrder: [ + StandardColumnId.TimeInState, + "annotation_rubbish-annotation", + "annotation_preferenc.es/foo-bravo", + "rubbish" as ColumnId, + StandardColumnId.JobID, + ], + order: { id: StandardColumnId.TimeInState, direction: "ASC" }, + pageIndex: 5, + pageSize: 20, + sidebarJobId: "223344", + visibleColumns: { + [StandardColumnId.JobID]: true, + [StandardColumnId.TimeInState]: true, + 'annotation_"preferenc.es/foo-bravo': true, + }, + } + + ensurePreferencesAreConsistent(validPreferences) + + const expected: JobsTablePreferences = { + annotationColumnKeys: ["preferenc.es/foo-alpha", "preferenc.es/foo-bravo", "preferenc.es/foo-charlie"], + expandedState: {}, + filters: [], + columnMatches: {}, + groupedColumns: [], + columnOrder: [ + StandardColumnId.TimeInState, + "annotation_preferenc.es/foo-bravo", + StandardColumnId.JobID, + + // All the following elements are added + "annotation_preferenc.es/foo-alpha", + "annotation_preferenc.es/foo-charlie", + StandardColumnId.Queue, + StandardColumnId.Namespace, + StandardColumnId.JobSet, + StandardColumnId.State, + StandardColumnId.Count, + StandardColumnId.Priority, + StandardColumnId.Owner, + StandardColumnId.CPU, + StandardColumnId.Memory, + StandardColumnId.EphemeralStorage, + StandardColumnId.GPU, + StandardColumnId.PriorityClass, + StandardColumnId.LastTransitionTimeUtc, + StandardColumnId.TimeSubmittedUtc, + StandardColumnId.TimeSubmittedAgo, + StandardColumnId.Node, + StandardColumnId.Cluster, + StandardColumnId.ExitCode, + StandardColumnId.RuntimeSeconds, + ], + order: { id: StandardColumnId.TimeInState, direction: "ASC" }, + pageIndex: 5, + pageSize: 20, + sidebarJobId: "223344", + visibleColumns: { + [StandardColumnId.JobID]: true, + [StandardColumnId.TimeInState]: true, + 'annotation_"preferenc.es/foo-bravo': true, + }, + } + + expect(validPreferences).toEqual(expected) + }) +}) diff --git a/internal/lookout/ui/src/services/lookoutV2/JobsTablePreferencesService.ts b/internal/lookout/ui/src/services/lookoutV2/JobsTablePreferencesService.ts index b47a8ae03e0..c1db172ce09 100644 --- a/internal/lookout/ui/src/services/lookoutV2/JobsTablePreferencesService.ts +++ b/internal/lookout/ui/src/services/lookoutV2/JobsTablePreferencesService.ts @@ -8,16 +8,21 @@ import { AnnotationColumnId, ColumnId, DEFAULT_COLUMN_MATCHES, - DEFAULT_COLUMN_ORDER, + DEFAULT_COLUMN_ORDERING, DEFAULT_COLUMN_VISIBILITY, + JOB_COLUMNS, + toAnnotationColId, + toColId, fromAnnotationColId, isStandardColId, + PINNED_COLUMNS, } from "../../utils/jobsTableColumns" import { matchForColumn } from "../../utils/jobsTableUtils" export interface JobsTablePreferences { annotationColumnKeys: string[] visibleColumns: VisibilityState + columnOrder: ColumnId[] groupedColumns: ColumnId[] expandedState: ExpandedStateList pageIndex: number @@ -36,13 +41,14 @@ export interface JobsTablePreferences { export const DEFAULT_PREFERENCES: JobsTablePreferences = { annotationColumnKeys: [], visibleColumns: DEFAULT_COLUMN_VISIBILITY, + columnOrder: JOB_COLUMNS.filter(({ id }) => !PINNED_COLUMNS.includes(toColId(id))).map(({ id }) => toColId(id)), filters: [], columnMatches: DEFAULT_COLUMN_MATCHES, groupedColumns: [], expandedState: {}, pageIndex: 0, pageSize: 50, - order: DEFAULT_COLUMN_ORDER, + order: DEFAULT_COLUMN_ORDERING, sidebarJobId: undefined, sidebarWidth: 600, columnSizing: {}, @@ -191,7 +197,10 @@ const mergeQueryParamsAndLocalStorage = ( return mergedPrefs } -// Make sure annotations referenced in filters exist, make sure columns referenced in objects are visible +// Ensure that: +// - annotations referenced in filters exist +// - make sure columns referenced in objects are visible +// - the column order includes exactly all unpinned standard columns and annotations export const ensurePreferencesAreConsistent = (preferences: JobsTablePreferences) => { // Make sure annotation columns referenced in filters exist if (preferences.annotationColumnKeys === undefined) { @@ -209,18 +218,35 @@ export const ensurePreferencesAreConsistent = (preferences: JobsTablePreferences } } + // Make sure all and only annotation columns and unpinned standard columns are contained in columnOrder + const columnOrderSet = new Set(preferences.columnOrder) + const annotationKeyColumnIds = preferences.annotationColumnKeys.map(toAnnotationColId) + const unpinnedStandardColumnIds = JOB_COLUMNS.filter(({ id }) => !PINNED_COLUMNS.includes(toColId(id))).map( + ({ id }) => toColId(id), + ) + + // Add missing column IDs + annotationKeyColumnIds.filter((id) => !columnOrderSet.has(id)).forEach((id) => preferences.columnOrder.push(id)) + unpinnedStandardColumnIds.filter((id) => !columnOrderSet.has(id)).forEach((id) => preferences.columnOrder.push(id)) + + // Remove extraneous column IDs + const annotationKeyColumnIdsSet = new Set(annotationKeyColumnIds) + const unpinnedStandardColumnIdsSet = new Set(unpinnedStandardColumnIds) + preferences.columnOrder = preferences.columnOrder.filter( + (id) => annotationKeyColumnIdsSet.has(id as AnnotationColumnId) || unpinnedStandardColumnIdsSet.has(id), + ) + // Make sure grouped columns, order columns, and filtered columns are visible ensureVisible(preferences.visibleColumns, preferences.groupedColumns ?? []) ensureVisible(preferences.visibleColumns, preferences.order === undefined ? [] : [preferences.order.id]) ensureVisible(preferences.visibleColumns, preferences.filters?.map((filter) => filter.id) ?? []) } -export const stringifyQueryParams = (paramObj: any): string => { - return qs.stringify(paramObj, { +export const stringifyQueryParams = (paramObj: any): string => + qs.stringify(paramObj, { encodeValuesOnly: true, strictNullHandling: true, }) -} export class JobsTablePreferencesService { constructor(private router: Router) {} diff --git a/internal/lookout/ui/src/utils/jobsTableColumns.tsx b/internal/lookout/ui/src/utils/jobsTableColumns.tsx index debca4db9ee..80f388e7129 100644 --- a/internal/lookout/ui/src/utils/jobsTableColumns.tsx +++ b/internal/lookout/ui/src/utils/jobsTableColumns.tsx @@ -468,7 +468,15 @@ export const DEFAULT_COLUMN_VISIBILITY: VisibilityState = Object.values(Standard {}, ) -export const DEFAULT_COLUMN_ORDER: LookoutColumnOrder = { id: "jobId", direction: "DESC" } +export const PINNED_COLUMNS: ColumnId[] = [StandardColumnId.SelectorCol] + +// The ordering of each column +export const DEFAULT_COLUMN_ORDERING: LookoutColumnOrder = { id: "jobId", direction: "DESC" } + +// The order of the columns in the table +export const DEFAULT_COLUMN_ORDER = JOB_COLUMNS.filter(({ id }) => !PINNED_COLUMNS.includes(toColId(id))).map( + ({ id }) => toColId(id), +) type Formatter = (val: number | string | string[]) => string diff --git a/internal/lookout/ui/yarn.lock b/internal/lookout/ui/yarn.lock index 6f7d6602b7f..f693a341de7 100644 --- a/internal/lookout/ui/yarn.lock +++ b/internal/lookout/ui/yarn.lock @@ -237,6 +237,45 @@ "@babel/helper-string-parser" "^7.25.9" "@babel/helper-validator-identifier" "^7.25.9" +"@dnd-kit/accessibility@^3.1.1": + version "3.1.1" + resolved "https://registry.yarnpkg.com/@dnd-kit/accessibility/-/accessibility-3.1.1.tgz#3b4202bd6bb370a0730f6734867785919beac6af" + integrity sha512-2P+YgaXF+gRsIihwwY1gCsQSYnu9Zyj2py8kY5fFvUM1qm2WA2u639R6YNVfU4GWr+ZM5mqEsfHZZLoRONbemw== + dependencies: + tslib "^2.0.0" + +"@dnd-kit/core@^6.3.1": + version "6.3.1" + resolved "https://registry.yarnpkg.com/@dnd-kit/core/-/core-6.3.1.tgz#4c36406a62c7baac499726f899935f93f0e6d003" + integrity sha512-xkGBRQQab4RLwgXxoqETICr6S5JlogafbhNsidmrkVv2YRs5MLwpjoF2qpiGjQt8S9AoxtIV603s0GIUpY5eYQ== + dependencies: + "@dnd-kit/accessibility" "^3.1.1" + "@dnd-kit/utilities" "^3.2.2" + tslib "^2.0.0" + +"@dnd-kit/modifiers@^9.0.0": + version "9.0.0" + resolved "https://registry.yarnpkg.com/@dnd-kit/modifiers/-/modifiers-9.0.0.tgz#96a0280c77b10c716ef79d9792ce7ad04370771d" + integrity sha512-ybiLc66qRGuZoC20wdSSG6pDXFikui/dCNGthxv4Ndy8ylErY0N3KVxY2bgo7AWwIbxDmXDg3ylAFmnrjcbVvw== + dependencies: + "@dnd-kit/utilities" "^3.2.2" + tslib "^2.0.0" + +"@dnd-kit/sortable@^10.0.0": + version "10.0.0" + resolved "https://registry.yarnpkg.com/@dnd-kit/sortable/-/sortable-10.0.0.tgz#1f9382b90d835cd5c65d92824fa9dafb78c4c3e8" + integrity sha512-+xqhmIIzvAYMGfBYYnbKuNicfSsk4RksY2XdmJhT+HAC01nix6fHCztU68jooFiMUB01Ky3F0FyOvhG/BZrWkg== + dependencies: + "@dnd-kit/utilities" "^3.2.2" + tslib "^2.0.0" + +"@dnd-kit/utilities@^3.2.2": + version "3.2.2" + resolved "https://registry.yarnpkg.com/@dnd-kit/utilities/-/utilities-3.2.2.tgz#5a32b6af356dc5f74d61b37d6f7129a4040ced7b" + integrity sha512-+MKAJEOfaBe5SmV6t34p80MMKhjvUz0vRrvVJbPT0WElzaOJ/1xs+D+KDv+tD/NE5ujfrChEcshd4fLn0wpiqg== + dependencies: + tslib "^2.0.0" + "@emotion/babel-plugin@^11.13.5": version "11.13.5" resolved "https://registry.yarnpkg.com/@emotion/babel-plugin/-/babel-plugin-11.13.5.tgz#eab8d65dbded74e0ecfd28dc218e75607c4e7bc0" @@ -4860,7 +4899,7 @@ tsconfig-paths@^3.15.0: minimist "^1.2.6" strip-bom "^3.0.0" -tslib@^2.6.2: +tslib@^2.0.0, tslib@^2.6.2: version "2.8.1" resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.8.1.tgz#612efe4ed235d567e8aba5f2a5fab70280ade83f" integrity sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w== From 5d24d0a35ef91240a920bbaa83e5f7bc1c001671 Mon Sep 17 00:00:00 2001 From: Chris Martin Date: Tue, 7 Jan 2025 11:11:30 +0000 Subject: [PATCH 13/17] Update Helm to use configmaps for user config rather than secrets (#4122) * Update Hel to use configmaps for user config rather than secrets * fix checksum * update configmap --- .../armada/templates/{secret.yaml => configmap.yaml} | 5 ++--- deployment/armada/templates/deployment.yaml | 6 +++--- .../templates/{secret.yaml => configmap.yaml} | 5 ++--- deployment/binoculars/templates/deployment.yaml | 6 +++--- .../templates/{secret.yaml => configmap.yaml} | 6 +++--- deployment/event-ingester/templates/deployment.yaml | 6 +++--- .../executor/templates/{secret.yaml => configmap.yaml} | 5 ++--- deployment/executor/templates/deployment.yaml | 6 +++--- .../templates/{secret.yaml => configmap.yaml} | 5 ++--- .../lookout-ingester-v2/templates/deployment.yaml | 6 +++--- .../templates/configmap.yaml} | 9 ++++----- deployment/lookout-migration-v2/templates/job.yaml | 4 ++-- .../templates/configmap.yaml} | 9 ++++----- deployment/lookout-v2/templates/deployment.yaml | 6 +++--- .../templates/{secret.yaml => configmap.yaml} | 10 +++++----- deployment/scheduler-migration/templates/job.yaml | 4 ++-- ...{scheduler-secret.yaml => scheduler-configmap.yaml} | 9 ++++----- ...r-secret.yaml => scheduler-ingester-configmap.yaml} | 9 ++++----- .../templates/scheduler-ingester-deployment.yaml | 6 +++--- .../scheduler/templates/scheduler-statefulset.yaml | 6 +++--- 20 files changed, 60 insertions(+), 68 deletions(-) rename deployment/armada/templates/{secret.yaml => configmap.yaml} (76%) rename deployment/binoculars/templates/{secret.yaml => configmap.yaml} (76%) rename deployment/event-ingester/templates/{secret.yaml => configmap.yaml} (77%) rename deployment/executor/templates/{secret.yaml => configmap.yaml} (76%) rename deployment/lookout-ingester-v2/templates/{secret.yaml => configmap.yaml} (78%) rename deployment/{lookout-v2/templates/secret.yaml => lookout-migration-v2/templates/configmap.yaml} (64%) rename deployment/{lookout-migration-v2/templates/secret.yaml => lookout-v2/templates/configmap.yaml} (64%) rename deployment/scheduler-migration/templates/{secret.yaml => configmap.yaml} (65%) rename deployment/scheduler/templates/{scheduler-secret.yaml => scheduler-configmap.yaml} (62%) rename deployment/scheduler/templates/{scheduler-ingester-secret.yaml => scheduler-ingester-configmap.yaml} (65%) diff --git a/deployment/armada/templates/secret.yaml b/deployment/armada/templates/configmap.yaml similarity index 76% rename from deployment/armada/templates/secret.yaml rename to deployment/armada/templates/configmap.yaml index 1df218be5d7..1a560445ac6 100644 --- a/deployment/armada/templates/secret.yaml +++ b/deployment/armada/templates/configmap.yaml @@ -1,13 +1,12 @@ apiVersion: v1 -kind: Secret +kind: ConfigMap metadata: name: {{ include "armada.config.name" . }} namespace: {{ .Release.Namespace }} labels: {{- include "armada.labels.all" . | nindent 4 }} -type: Opaque data: {{ include "armada.config.filename" . }}: | {{- if .Values.applicationConfig }} -{{ toYaml .Values.applicationConfig | b64enc | indent 4 }} +{{ toYaml .Values.applicationConfig | indent 4 }} {{- end }} diff --git a/deployment/armada/templates/deployment.yaml b/deployment/armada/templates/deployment.yaml index c1a84e151df..39cd771b90e 100644 --- a/deployment/armada/templates/deployment.yaml +++ b/deployment/armada/templates/deployment.yaml @@ -18,7 +18,7 @@ spec: metadata: name: {{ include "armada.name" . }} annotations: - checksum/config: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }} + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} labels: {{- include "armada.labels.all" . | nindent 8 }} spec: @@ -131,8 +131,8 @@ spec: topologyKey: kubernetes.io/hostname volumes: - name: user-config - secret: - secretName: {{ include "armada.config.name" . }} + configMap: + name: {{ include "armada.config.name" . }} {{- if .Values.applicationConfig.pulsar.authenticationEnabled }} - name: pulsar-token secret: diff --git a/deployment/binoculars/templates/secret.yaml b/deployment/binoculars/templates/configmap.yaml similarity index 76% rename from deployment/binoculars/templates/secret.yaml rename to deployment/binoculars/templates/configmap.yaml index 27fe44869d0..c0f44df845b 100644 --- a/deployment/binoculars/templates/secret.yaml +++ b/deployment/binoculars/templates/configmap.yaml @@ -1,13 +1,12 @@ apiVersion: v1 -kind: Secret +kind: ConfigMap metadata: name: {{ include "binoculars.config.name" . }} namespace: {{ .Release.Namespace }} labels: {{- include "binoculars.labels.all" . | nindent 4 }} -type: Opaque data: {{ include "binoculars.config.filename" . }}: | {{- if .Values.applicationConfig }} -{{ toYaml .Values.applicationConfig | b64enc | indent 4 }} +{{ toYaml .Values.applicationConfig | indent 4 }} {{- end }} diff --git a/deployment/binoculars/templates/deployment.yaml b/deployment/binoculars/templates/deployment.yaml index 51962b2de1f..bc24e40349a 100644 --- a/deployment/binoculars/templates/deployment.yaml +++ b/deployment/binoculars/templates/deployment.yaml @@ -18,7 +18,7 @@ spec: metadata: name: {{ include "binoculars.name" . }} annotations: - checksum/config: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }} + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} labels: {{- include "binoculars.labels.all" . | nindent 8 }} spec: @@ -102,8 +102,8 @@ spec: topologyKey: kubernetes.io/hostname volumes: - name: user-config - secret: - secretName: {{ include "binoculars.config.name" . }} + configMap: + name: {{ include "binoculars.config.name" . }} {{- if .Values.applicationConfig.grpc.tls.enabled }} - name: tls-certs secret: diff --git a/deployment/event-ingester/templates/secret.yaml b/deployment/event-ingester/templates/configmap.yaml similarity index 77% rename from deployment/event-ingester/templates/secret.yaml rename to deployment/event-ingester/templates/configmap.yaml index 813cc741b51..0d9dffbd030 100644 --- a/deployment/event-ingester/templates/secret.yaml +++ b/deployment/event-ingester/templates/configmap.yaml @@ -1,13 +1,13 @@ apiVersion: v1 -kind: Secret +kind: ConfigMap metadata: name: {{ include "event_ingester.config.name" . }} namespace: {{ .Release.Namespace }} labels: {{- include "event_ingester.labels.all" . | nindent 4 }} -type: Opaque data: {{ include "event_ingester.config.filename" . }}: | {{- if .Values.applicationConfig }} -{{ toYaml .Values.applicationConfig | b64enc | indent 4 }} +{{ toYaml .Values.applicationConfig | indent 4 }} {{- end }} + diff --git a/deployment/event-ingester/templates/deployment.yaml b/deployment/event-ingester/templates/deployment.yaml index 93d1ed42890..9e6dcdf1c35 100644 --- a/deployment/event-ingester/templates/deployment.yaml +++ b/deployment/event-ingester/templates/deployment.yaml @@ -18,7 +18,7 @@ spec: metadata: name: {{ include "event_ingester.name" . }} annotations: - checksum/config: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }} + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} labels: {{- include "event_ingester.labels.all" . | nindent 8 }} spec: @@ -87,8 +87,8 @@ spec: topologyKey: kubernetes.io/hostname volumes: - name: user-config - secret: - secretName: {{ include "event_ingester.config.name" . }} + configMap: + name: {{ include "event_ingester.config.name" . }} {{- if .Values.applicationConfig.pulsar.authenticationEnabled }} - name: pulsar-token secret: diff --git a/deployment/executor/templates/secret.yaml b/deployment/executor/templates/configmap.yaml similarity index 76% rename from deployment/executor/templates/secret.yaml rename to deployment/executor/templates/configmap.yaml index 348f27a1d5d..40dc3fc4210 100644 --- a/deployment/executor/templates/secret.yaml +++ b/deployment/executor/templates/configmap.yaml @@ -1,13 +1,12 @@ apiVersion: v1 -kind: Secret +kind: ConfigMap metadata: name: {{ include "executor.config.name" . }} namespace: {{ .Release.Namespace }} labels: {{- include "executor.labels.all" . | nindent 4 }} -type: Opaque data: {{ include "executor.config.filename" . }}: | {{- if .Values.applicationConfig }} - {{ toYaml .Values.applicationConfig | b64enc | indent 4 }} + {{ toYaml .Values.applicationConfig | indent 4 }} {{- end }} diff --git a/deployment/executor/templates/deployment.yaml b/deployment/executor/templates/deployment.yaml index 0e98fa9afa5..22ca279dc0d 100644 --- a/deployment/executor/templates/deployment.yaml +++ b/deployment/executor/templates/deployment.yaml @@ -14,7 +14,7 @@ spec: metadata: name: {{ include "executor.name" . }} annotations: - checksum/config: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }} + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} labels: {{- include "executor.labels.all" . | nindent 8 }} spec: @@ -90,8 +90,8 @@ spec: {{- end }} volumes: - name: user-config - secret: - secretName: {{ include "executor.config.name" . }} + configMap: + name: {{ include "executor.config.name" . }} {{- if .Values.additionalVolumes }} {{- toYaml .Values.additionalVolumes | nindent 8 }} {{- end }} diff --git a/deployment/lookout-ingester-v2/templates/secret.yaml b/deployment/lookout-ingester-v2/templates/configmap.yaml similarity index 78% rename from deployment/lookout-ingester-v2/templates/secret.yaml rename to deployment/lookout-ingester-v2/templates/configmap.yaml index 180d443fac9..f5537dfda13 100644 --- a/deployment/lookout-ingester-v2/templates/secret.yaml +++ b/deployment/lookout-ingester-v2/templates/configmap.yaml @@ -1,13 +1,12 @@ apiVersion: v1 -kind: Secret +kind: ConfigMap metadata: name: {{ include "lookout_ingester_v2.config.name" . }} namespace: {{ .Release.Namespace }} labels: {{- include "lookout_ingester_v2.labels.all" . | nindent 4 }} -type: Opaque data: {{ include "lookout_ingester_v2.config.filename" . }}: | {{- if .Values.applicationConfig }} -{{ toYaml .Values.applicationConfig | b64enc | indent 4 }} +{{ toYaml .Values.applicationConfig | indent 4 }} {{- end }} diff --git a/deployment/lookout-ingester-v2/templates/deployment.yaml b/deployment/lookout-ingester-v2/templates/deployment.yaml index 5d4449564dd..24ea65d2ab0 100644 --- a/deployment/lookout-ingester-v2/templates/deployment.yaml +++ b/deployment/lookout-ingester-v2/templates/deployment.yaml @@ -18,7 +18,7 @@ spec: metadata: name: {{ include "lookout_ingester_v2.name" . }} annotations: - checksum/config: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }} + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} labels: {{- include "lookout_ingester_v2.labels.all" . | nindent 8 }} spec: @@ -87,8 +87,8 @@ spec: topologyKey: kubernetes.io/hostname volumes: - name: user-config - secret: - secretName: {{ include "lookout_ingester_v2.config.name" . }} + configMap: + name: {{ include "lookout_ingester_v2.config.name" . }} {{- if .Values.applicationConfig.pulsar.authenticationEnabled }} - name: pulsar-token secret: diff --git a/deployment/lookout-v2/templates/secret.yaml b/deployment/lookout-migration-v2/templates/configmap.yaml similarity index 64% rename from deployment/lookout-v2/templates/secret.yaml rename to deployment/lookout-migration-v2/templates/configmap.yaml index 503f4839f7d..bafa872a364 100644 --- a/deployment/lookout-v2/templates/secret.yaml +++ b/deployment/lookout-migration-v2/templates/configmap.yaml @@ -1,13 +1,12 @@ apiVersion: v1 -kind: Secret +kind: ConfigMap metadata: name: {{ include "lookout_v2.config.name" . }} namespace: {{ .Release.Namespace }} labels: {{- include "lookout_v2.labels.all" . | nindent 4 }} -type: Opaque data: {{ include "lookout_v2.config.filename" . }}: | -{{- if .Values.applicationConfig }} -{{ toYaml .Values.applicationConfig | b64enc | indent 4 }} -{{- end }} + {{- if .Values.applicationConfig }} + {{ toYaml .Values.applicationConfig | indent 4 }} + {{- end }} diff --git a/deployment/lookout-migration-v2/templates/job.yaml b/deployment/lookout-migration-v2/templates/job.yaml index 28bafe52cd8..d1d76907a31 100644 --- a/deployment/lookout-migration-v2/templates/job.yaml +++ b/deployment/lookout-migration-v2/templates/job.yaml @@ -49,8 +49,8 @@ spec: allowPrivilegeEscalation: false volumes: - name: user-config - secret: - secretName: {{ include "lookout_v2.config.name" . }} + configMap: + name: {{ include "lookout_v2.config.name" . }} {{- if .Values.additionalVolumes }} {{- toYaml .Values.additionalVolumes | nindent 8 }} {{- end }} diff --git a/deployment/lookout-migration-v2/templates/secret.yaml b/deployment/lookout-v2/templates/configmap.yaml similarity index 64% rename from deployment/lookout-migration-v2/templates/secret.yaml rename to deployment/lookout-v2/templates/configmap.yaml index 503f4839f7d..bafa872a364 100644 --- a/deployment/lookout-migration-v2/templates/secret.yaml +++ b/deployment/lookout-v2/templates/configmap.yaml @@ -1,13 +1,12 @@ apiVersion: v1 -kind: Secret +kind: ConfigMap metadata: name: {{ include "lookout_v2.config.name" . }} namespace: {{ .Release.Namespace }} labels: {{- include "lookout_v2.labels.all" . | nindent 4 }} -type: Opaque data: {{ include "lookout_v2.config.filename" . }}: | -{{- if .Values.applicationConfig }} -{{ toYaml .Values.applicationConfig | b64enc | indent 4 }} -{{- end }} + {{- if .Values.applicationConfig }} + {{ toYaml .Values.applicationConfig | indent 4 }} + {{- end }} diff --git a/deployment/lookout-v2/templates/deployment.yaml b/deployment/lookout-v2/templates/deployment.yaml index b55100ce002..951999b38e6 100644 --- a/deployment/lookout-v2/templates/deployment.yaml +++ b/deployment/lookout-v2/templates/deployment.yaml @@ -18,7 +18,7 @@ spec: metadata: name: {{ include "lookout_v2.name" . }} annotations: - checksum/config: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }} + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} labels: {{- include "lookout_v2.labels.all" . | nindent 8 }} spec: @@ -101,8 +101,8 @@ spec: topologyKey: kubernetes.io/hostname volumes: - name: user-config - secret: - secretName: {{ include "lookout_v2.config.name" . }} + configMap: + name: {{ include "lookout_v2.config.name" . }} {{- if .Values.applicationConfig.tls.enabled }} - name: tls-certs secret: diff --git a/deployment/scheduler-migration/templates/secret.yaml b/deployment/scheduler-migration/templates/configmap.yaml similarity index 65% rename from deployment/scheduler-migration/templates/secret.yaml rename to deployment/scheduler-migration/templates/configmap.yaml index 50dab3dc570..38b748dc67f 100644 --- a/deployment/scheduler-migration/templates/secret.yaml +++ b/deployment/scheduler-migration/templates/configmap.yaml @@ -1,13 +1,13 @@ apiVersion: v1 -kind: Secret +kind: ConfigMap metadata: name: {{ include "armada-scheduler.config.name" . }} namespace: {{ .Release.Namespace }} labels: {{- include "armada-scheduler.labels.all" . | nindent 4 }} -type: Opaque data: {{ include "armada-scheduler.config.filename" . }}: | -{{- if .Values.applicationConfig }} -{{ toYaml .Values.applicationConfig | b64enc | indent 4 }} -{{- end }} + {{- if .Values.applicationConfig }} + {{ toYaml .Values.applicationConfig | indent 4 }} + {{- end }} + diff --git a/deployment/scheduler-migration/templates/job.yaml b/deployment/scheduler-migration/templates/job.yaml index f4100bf63e6..e3e708afbda 100644 --- a/deployment/scheduler-migration/templates/job.yaml +++ b/deployment/scheduler-migration/templates/job.yaml @@ -49,8 +49,8 @@ spec: allowPrivilegeEscalation: false volumes: - name: user-config - secret: - secretName: {{ include "armada-scheduler.config.name" . }} + configMap: + name: {{ include "armada-scheduler.config.name" . }} {{- if .Values.additionalVolumes }} {{- toYaml .Values.additionalVolumes | nindent 8 }} {{- end }} diff --git a/deployment/scheduler/templates/scheduler-secret.yaml b/deployment/scheduler/templates/scheduler-configmap.yaml similarity index 62% rename from deployment/scheduler/templates/scheduler-secret.yaml rename to deployment/scheduler/templates/scheduler-configmap.yaml index 48dee86e368..cb3ea379e9e 100644 --- a/deployment/scheduler/templates/scheduler-secret.yaml +++ b/deployment/scheduler/templates/scheduler-configmap.yaml @@ -1,13 +1,12 @@ apiVersion: v1 -kind: Secret +kind: ConfigMap metadata: name: {{ include "armada-scheduler.config.name" . }} namespace: {{ .Release.Namespace }} labels: {{- include "armada-scheduler.labels.all" . | nindent 4 }} -type: Opaque data: {{ include "armada-scheduler.config.filename" . }}: | -{{- if .Values.scheduler.applicationConfig }} -{{ toYaml .Values.scheduler.applicationConfig | b64enc | indent 4 }} -{{- end }} + {{- if .Values.scheduler.applicationConfig }} + {{ toYaml .Values.scheduler.applicationConfig | indent 4 }} + {{- end }} diff --git a/deployment/scheduler/templates/scheduler-ingester-secret.yaml b/deployment/scheduler/templates/scheduler-ingester-configmap.yaml similarity index 65% rename from deployment/scheduler/templates/scheduler-ingester-secret.yaml rename to deployment/scheduler/templates/scheduler-ingester-configmap.yaml index 72af1b3e4a0..c65eb4ad970 100644 --- a/deployment/scheduler/templates/scheduler-ingester-secret.yaml +++ b/deployment/scheduler/templates/scheduler-ingester-configmap.yaml @@ -1,13 +1,12 @@ apiVersion: v1 -kind: Secret +kind: ConfigMap metadata: name: {{ include "armada-scheduler-ingester.config.name" . }} namespace: {{ .Release.Namespace }} labels: {{- include "armada-scheduler-ingester.labels.all" . | nindent 4 }} -type: Opaque data: {{ include "armada-scheduler-ingester.config.filename" . }}: | -{{- if .Values.ingester.applicationConfig }} -{{ toYaml .Values.ingester.applicationConfig | b64enc | indent 4 }} -{{- end }} + {{- if .Values.ingester.applicationConfig }} + {{ toYaml .Values.ingester.applicationConfig | indent 4 }} + {{- end }} diff --git a/deployment/scheduler/templates/scheduler-ingester-deployment.yaml b/deployment/scheduler/templates/scheduler-ingester-deployment.yaml index 41d2db51ea1..d6bf2305093 100644 --- a/deployment/scheduler/templates/scheduler-ingester-deployment.yaml +++ b/deployment/scheduler/templates/scheduler-ingester-deployment.yaml @@ -18,7 +18,7 @@ spec: metadata: name: {{ include "armada-scheduler.name" . }}-ingester annotations: - checksum/config: {{ include (print $.Template.BasePath "/scheduler-ingester-secret.yaml") . | sha256sum }} + checksum/config: {{ include (print $.Template.BasePath "/scheduler-ingester-configmap.yaml") . | sha256sum }} labels: {{- include "armada-scheduler-ingester.labels.all" . | nindent 8 }} spec: @@ -87,8 +87,8 @@ spec: topologyKey: kubernetes.io/hostname volumes: - name: user-config - secret: - secretName: {{ include "armada-scheduler-ingester.config.name" . }} + configMap: + name: {{ include "armada-scheduler-ingester.config.name" . }} {{- if .Values.ingester.applicationConfig.pulsar.authenticationEnabled }} - name: pulsar-token secret: diff --git a/deployment/scheduler/templates/scheduler-statefulset.yaml b/deployment/scheduler/templates/scheduler-statefulset.yaml index 3dbc146eaf1..b6e8fa9f643 100644 --- a/deployment/scheduler/templates/scheduler-statefulset.yaml +++ b/deployment/scheduler/templates/scheduler-statefulset.yaml @@ -19,7 +19,7 @@ spec: metadata: name: {{ include "armada-scheduler.name" . }} annotations: - checksum/config: {{ include (print $.Template.BasePath "/scheduler-secret.yaml") . | sha256sum }} + checksum/config: {{ include (print $.Template.BasePath "/scheduler-configmap.yaml") . | sha256sum }} labels: {{- include "armada-scheduler.labels.all" . | nindent 8 }} spec: @@ -132,8 +132,8 @@ spec: secretName: armada-scheduler-service-tls {{- end}} - name: user-config - secret: - secretName: {{ include "armada-scheduler.config.name" . }} + configMap: + name: {{ include "armada-scheduler.config.name" . }} {{- if .Values.scheduler.applicationConfig.pulsar.authenticationEnabled }} - name: pulsar-token secret: From 9021b78815d11615c5a835d87266fae84ede5c04 Mon Sep 17 00:00:00 2001 From: Maurice Yap Date: Tue, 7 Jan 2025 15:22:59 +0000 Subject: [PATCH 14/17] Align aggregate counts for job statuses (#4124) On the Lookout UI, when grouping is enabled for the jobs table, change how counts for different statuses are displayed, so that they are aligned and are less visually noisy. --- .../src/components/ActionableValueOnHover.tsx | 9 +- .../ui/src/components/CopyIconButton.tsx | 8 +- .../lookoutV2/JobGroupStateCounts.module.css | 14 -- .../lookoutV2/JobGroupStateCounts.tsx | 69 ++++++---- .../JobGroupStateCountsColumnHeader.tsx | 35 +++++ .../src/components/lookoutV2/JobStateChip.tsx | 4 +- .../lookoutV2/JobStateCountChip.tsx | 2 +- .../components/lookoutV2/JobsTableCell.tsx | 124 +++++++++++------- .../lookoutV2/JobsTableContainer.tsx | 1 + .../lookout/ui/src/theme/components/chip.ts | 67 ++++++++++ .../ui/src/theme/components/tooltip.ts | 26 ++++ internal/lookout/ui/src/theme/palette.ts | 2 + internal/lookout/ui/src/theme/theme.ts | 3 + .../lookout/ui/src/utils/jobsTableColumns.tsx | 14 +- 14 files changed, 282 insertions(+), 96 deletions(-) delete mode 100644 internal/lookout/ui/src/components/lookoutV2/JobGroupStateCounts.module.css create mode 100644 internal/lookout/ui/src/components/lookoutV2/JobGroupStateCountsColumnHeader.tsx create mode 100644 internal/lookout/ui/src/theme/components/chip.ts create mode 100644 internal/lookout/ui/src/theme/components/tooltip.ts diff --git a/internal/lookout/ui/src/components/ActionableValueOnHover.tsx b/internal/lookout/ui/src/components/ActionableValueOnHover.tsx index 1f7ba72b2e9..778f8caeb4b 100644 --- a/internal/lookout/ui/src/components/ActionableValueOnHover.tsx +++ b/internal/lookout/ui/src/components/ActionableValueOnHover.tsx @@ -6,12 +6,17 @@ import { CopyIconButton } from "./CopyIconButton" import { AddFilter } from "./icons" const OuterContainer = styled("div")({ - display: "flex", + display: "inline-flex", + width: "100%", flexDirection: "row", alignItems: "center", gap: "0.5ch", }) +const ContentContainer = styled("div")<{ minWidth: boolean }>(({ minWidth }) => ({ + flexGrow: minWidth ? undefined : 1, +})) + const StyledIconButton = styled(IconButton)(({ hidden }) => ({ visibility: hidden ? "hidden" : "unset", })) @@ -40,7 +45,7 @@ export const ActionableValueOnHover = ({ const [hovering, setHovering] = useState(false) return ( setHovering(true)} onMouseLeave={() => setHovering(false)}> -
{children}
+ {children} {copyAction && (
rowIsGroup: boolean @@ -204,21 +228,16 @@ export interface BodyCellProps { onExpandedChange: () => void onClickRowCheckbox: (row: Row) => void } + export const BodyCell = ({ cell, rowIsGroup, rowIsExpanded, onExpandedChange, onClickRowCheckbox }: BodyCellProps) => { const columnMetadata = getColumnMetadata(cell.column.columnDef) const cellHasValue = cell.renderValue() const isRightAligned = columnMetadata.isRightAligned ?? false - return ( - - {rowIsGroup && cell.column.getIsGrouped() && cellHasValue ? ( - // If it's a grouped cell, add an expander and row count + + const cellContent = ((): ReactNode => { + // If it's a grouped cell, add an expander and row count + if (rowIsGroup && cell.column.getIsGrouped() && cellHasValue) { + return ( - ) : cell.getIsAggregated() ? ( - // If the cell is aggregated, use the Aggregated - // renderer for cell - flexRender(cell.column.columnDef.aggregatedCell ?? cell.column.columnDef.cell, { + ) + } + + // If the cell is aggregated, use the Aggregated renderer for cell + if (cell.getIsAggregated()) { + return flexRender(cell.column.columnDef.aggregatedCell ?? cell.column.columnDef.cell, { + ...cell.getContext(), + onClickRowCheckbox, + }) + } + + return ( + { + cell.column.setFilterValue( + columnMetadata.filterType === FilterType.Enum ? [cell.getValue()] : cell.getValue(), + ) + }, + } + } + > + {flexRender(cell.column.columnDef.cell, { ...cell.getContext(), onClickRowCheckbox, - }) - ) : ( - { - cell.column.setFilterValue( - columnMetadata.filterType === FilterType.Enum ? [cell.getValue()] : cell.getValue(), - ) - }, - } - } - > - {flexRender(cell.column.columnDef.cell, { - ...cell.getContext(), - onClickRowCheckbox, - })} - - )} - + })} + + ) + })() + + return ( + + {cellContent} + ) } diff --git a/internal/lookout/ui/src/containers/lookoutV2/JobsTableContainer.tsx b/internal/lookout/ui/src/containers/lookoutV2/JobsTableContainer.tsx index 569222b1d59..e7b2d7351bb 100644 --- a/internal/lookout/ui/src/containers/lookoutV2/JobsTableContainer.tsx +++ b/internal/lookout/ui/src/containers/lookoutV2/JobsTableContainer.tsx @@ -825,6 +825,7 @@ export const JobsTableContainer = ({ onSetTextFieldRef={(ref) => { setTextFieldRef(header.id, ref) }} + groupedColumns={grouping} /> ))} diff --git a/internal/lookout/ui/src/theme/components/chip.ts b/internal/lookout/ui/src/theme/components/chip.ts new file mode 100644 index 00000000000..718a52352f9 --- /dev/null +++ b/internal/lookout/ui/src/theme/components/chip.ts @@ -0,0 +1,67 @@ +import { alpha, ChipClasses, ChipProps, Components, Theme } from "@mui/material" + +declare module "@mui/material/Chip" { + interface ChipPropsVariantOverrides { + shaded: true + } +} + +type ChipColor = Exclude + +const getShadedChipTextColor = (color: ChipColor, theme: Theme): string => { + switch (theme.palette.mode) { + case "light": + return color === "default" ? theme.palette.grey[400] : theme.palette[color].dark + case "dark": + return color === "default" ? theme.palette.grey[700] : theme.palette[color].light + } +} + +const getShadedChipBackgroundColor = (color: ChipColor, theme: Theme): string | undefined => { + switch (theme.palette.mode) { + case "light": + return color === "default" ? undefined : alpha(theme.palette[color].light, 0.12) + case "dark": + return color === "default" ? undefined : alpha(theme.palette[color].dark, 0.12) + } +} + +const ChipColorClassesKeyMap: Record = { + default: "colorDefault", + primary: "colorPrimary", + secondary: "colorSecondary", + error: "colorError", + info: "colorInfo", + success: "colorSuccess", + warning: "colorWarning", + statusGrey: "colorStatusGrey", + statusBlue: "colorStatusBlue", + statusGreen: "colorStatusGreen", + statusAmber: "colorStatusAmber", + statusRed: "colorStatusRed", +} + +export const MuiChip: Components["MuiChip"] = { + variants: [ + { + props: { variant: "shaded" }, + style: ({ theme }) => [ + { + fontWeight: theme.typography.fontWeightMedium, + + ...Object.entries(ChipColorClassesKeyMap).reduce((acc, [_color, chipClassesKey]) => { + const color = _color as ChipColor + return { + ...acc, + + [`&.MuiChip-${chipClassesKey}`]: { + backgroundColor: getShadedChipBackgroundColor(color, theme), + color: getShadedChipTextColor(color, theme), + }, + } + }, {}), + }, + ], + }, + ], +} diff --git a/internal/lookout/ui/src/theme/components/tooltip.ts b/internal/lookout/ui/src/theme/components/tooltip.ts new file mode 100644 index 00000000000..fe2c07bcf5f --- /dev/null +++ b/internal/lookout/ui/src/theme/components/tooltip.ts @@ -0,0 +1,26 @@ +import { Components, tooltipClasses } from "@mui/material" + +export const MuiTooltip: Components["MuiTooltip"] = { + defaultProps: { + arrow: true, + placement: "top", + slotProps: { + popper: { + sx: { + [`&.${tooltipClasses.popper}[data-popper-placement*="bottom"] .${tooltipClasses.tooltip}`]: { + marginTop: "5px", + }, + [`&.${tooltipClasses.popper}[data-popper-placement*="top"] .${tooltipClasses.tooltip}`]: { + marginBottom: "5px", + }, + [`&.${tooltipClasses.popper}[data-popper-placement*="right"] .${tooltipClasses.tooltip}`]: { + marginLeft: "5px", + }, + [`&.${tooltipClasses.popper}[data-popper-placement*="left"] .${tooltipClasses.tooltip}`]: { + marginRight: "5px", + }, + }, + }, + }, + }, +} diff --git a/internal/lookout/ui/src/theme/palette.ts b/internal/lookout/ui/src/theme/palette.ts index 6e1d354eeb7..23cbef30556 100644 --- a/internal/lookout/ui/src/theme/palette.ts +++ b/internal/lookout/ui/src/theme/palette.ts @@ -82,6 +82,8 @@ declare module "@mui/material/Button" { declare module "@mui/material/Chip" { // eslint-disable-next-line @typescript-eslint/no-empty-object-type interface ChipPropsColorOverrides extends CustomPaletteColorTokensTrueMap {} + // eslint-disable-next-line @typescript-eslint/no-empty-object-type + interface ChipClasses extends Record<`color${Capitalize}`, string> {} } declare module "@mui/material/SvgIcon" { diff --git a/internal/lookout/ui/src/theme/theme.ts b/internal/lookout/ui/src/theme/theme.ts index 7088b8257cb..4eb8a250ed3 100644 --- a/internal/lookout/ui/src/theme/theme.ts +++ b/internal/lookout/ui/src/theme/theme.ts @@ -1,9 +1,12 @@ import { createTheme } from "@mui/material" +import { MuiChip } from "./components/chip" +import { MuiTooltip } from "./components/tooltip" import { darkModePalette, lightModePalette } from "./palette" import { typography } from "./typography" export const theme = createTheme({ colorSchemes: { dark: { palette: darkModePalette }, light: { palette: lightModePalette } }, + components: { MuiChip, MuiTooltip }, typography, }) diff --git a/internal/lookout/ui/src/utils/jobsTableColumns.tsx b/internal/lookout/ui/src/utils/jobsTableColumns.tsx index 80f388e7129..367e4fb46d8 100644 --- a/internal/lookout/ui/src/utils/jobsTableColumns.tsx +++ b/internal/lookout/ui/src/utils/jobsTableColumns.tsx @@ -238,7 +238,12 @@ export const JOB_COLUMNS: JobTableColumn[] = [ cell.row.original.stateCounts && cell.row.original.groupedField !== "state" ) { - return + return ( + + ) } else { return } @@ -251,7 +256,12 @@ export const JOB_COLUMNS: JobTableColumn[] = [ cell.row.original.stateCounts && cell.row.original.groupedField !== "state" ) { - return + return ( + + ) } else { return } From dbea0564e56f85301d7aa1e2737c49111df5f2bb Mon Sep 17 00:00:00 2001 From: Chris Martin Date: Tue, 7 Jan 2025 16:38:05 +0000 Subject: [PATCH 15/17] NOJIRA: fix armada hel charts (#322) (#4126) Co-authored-by: Christopher Martin --- deployment/event-ingester/templates/configmap.yaml | 1 - deployment/executor/templates/configmap.yaml | 6 +++--- deployment/lookout-migration-v2/templates/configmap.yaml | 6 +++--- deployment/lookout-v2/templates/configmap.yaml | 6 +++--- deployment/scheduler-migration/templates/configmap.yaml | 7 +++---- deployment/scheduler/templates/scheduler-configmap.yaml | 6 +++--- .../scheduler/templates/scheduler-ingester-configmap.yaml | 6 +++--- 7 files changed, 18 insertions(+), 20 deletions(-) diff --git a/deployment/event-ingester/templates/configmap.yaml b/deployment/event-ingester/templates/configmap.yaml index 0d9dffbd030..3c60aef0414 100644 --- a/deployment/event-ingester/templates/configmap.yaml +++ b/deployment/event-ingester/templates/configmap.yaml @@ -10,4 +10,3 @@ data: {{- if .Values.applicationConfig }} {{ toYaml .Values.applicationConfig | indent 4 }} {{- end }} - diff --git a/deployment/executor/templates/configmap.yaml b/deployment/executor/templates/configmap.yaml index 40dc3fc4210..0885c48a552 100644 --- a/deployment/executor/templates/configmap.yaml +++ b/deployment/executor/templates/configmap.yaml @@ -7,6 +7,6 @@ metadata: {{- include "executor.labels.all" . | nindent 4 }} data: {{ include "executor.config.filename" . }}: | - {{- if .Values.applicationConfig }} - {{ toYaml .Values.applicationConfig | indent 4 }} - {{- end }} +{{- if .Values.applicationConfig }} +{{ toYaml .Values.applicationConfig | indent 4 }} +{{- end }} diff --git a/deployment/lookout-migration-v2/templates/configmap.yaml b/deployment/lookout-migration-v2/templates/configmap.yaml index bafa872a364..6de829c4c95 100644 --- a/deployment/lookout-migration-v2/templates/configmap.yaml +++ b/deployment/lookout-migration-v2/templates/configmap.yaml @@ -7,6 +7,6 @@ metadata: {{- include "lookout_v2.labels.all" . | nindent 4 }} data: {{ include "lookout_v2.config.filename" . }}: | - {{- if .Values.applicationConfig }} - {{ toYaml .Values.applicationConfig | indent 4 }} - {{- end }} +{{- if .Values.applicationConfig }} +{{ toYaml .Values.applicationConfig | indent 4 }} +{{- end }} diff --git a/deployment/lookout-v2/templates/configmap.yaml b/deployment/lookout-v2/templates/configmap.yaml index bafa872a364..6de829c4c95 100644 --- a/deployment/lookout-v2/templates/configmap.yaml +++ b/deployment/lookout-v2/templates/configmap.yaml @@ -7,6 +7,6 @@ metadata: {{- include "lookout_v2.labels.all" . | nindent 4 }} data: {{ include "lookout_v2.config.filename" . }}: | - {{- if .Values.applicationConfig }} - {{ toYaml .Values.applicationConfig | indent 4 }} - {{- end }} +{{- if .Values.applicationConfig }} +{{ toYaml .Values.applicationConfig | indent 4 }} +{{- end }} diff --git a/deployment/scheduler-migration/templates/configmap.yaml b/deployment/scheduler-migration/templates/configmap.yaml index 38b748dc67f..7a1511d9d2c 100644 --- a/deployment/scheduler-migration/templates/configmap.yaml +++ b/deployment/scheduler-migration/templates/configmap.yaml @@ -7,7 +7,6 @@ metadata: {{- include "armada-scheduler.labels.all" . | nindent 4 }} data: {{ include "armada-scheduler.config.filename" . }}: | - {{- if .Values.applicationConfig }} - {{ toYaml .Values.applicationConfig | indent 4 }} - {{- end }} - +{{- if .Values.applicationConfig }} +{{ toYaml .Values.applicationConfig | indent 4 }} +{{- end }} diff --git a/deployment/scheduler/templates/scheduler-configmap.yaml b/deployment/scheduler/templates/scheduler-configmap.yaml index cb3ea379e9e..3c8a00bc538 100644 --- a/deployment/scheduler/templates/scheduler-configmap.yaml +++ b/deployment/scheduler/templates/scheduler-configmap.yaml @@ -7,6 +7,6 @@ metadata: {{- include "armada-scheduler.labels.all" . | nindent 4 }} data: {{ include "armada-scheduler.config.filename" . }}: | - {{- if .Values.scheduler.applicationConfig }} - {{ toYaml .Values.scheduler.applicationConfig | indent 4 }} - {{- end }} +{{- if .Values.scheduler.applicationConfig }} +{{ toYaml .Values.scheduler.applicationConfig | indent 4 }} +{{- end }} diff --git a/deployment/scheduler/templates/scheduler-ingester-configmap.yaml b/deployment/scheduler/templates/scheduler-ingester-configmap.yaml index c65eb4ad970..ca6e81f52be 100644 --- a/deployment/scheduler/templates/scheduler-ingester-configmap.yaml +++ b/deployment/scheduler/templates/scheduler-ingester-configmap.yaml @@ -7,6 +7,6 @@ metadata: {{- include "armada-scheduler-ingester.labels.all" . | nindent 4 }} data: {{ include "armada-scheduler-ingester.config.filename" . }}: | - {{- if .Values.ingester.applicationConfig }} - {{ toYaml .Values.ingester.applicationConfig | indent 4 }} - {{- end }} +{{- if .Values.ingester.applicationConfig }} +{{ toYaml .Values.ingester.applicationConfig | indent 4 }} +{{- end }} From f4cd6bc1eab3f0bc7b2c89cffced31dc5136c5c3 Mon Sep 17 00:00:00 2001 From: Maurice Yap Date: Wed, 8 Jan 2025 10:27:04 +0000 Subject: [PATCH 16/17] Disable hiding columns which are filtered or sorted (#4125) The Lookout UI prevents the user from saving the hiding of a column if it is filtered or sorted-by. However, there is nothing to prevent them from hiding this without saving the change. This commit disables the hiding of these columns in the column configuration dialog to prevent confusion to the user by this behaviour. --- .../OrderableColumnListItem.tsx | 71 ++++++++++++++----- .../ColumnConfigurationDialog/index.tsx | 16 +++-- .../lookoutV2/JobsTableActionBar.tsx | 10 ++- .../lookoutV2/JobsTableContainer.tsx | 6 +- 4 files changed, 78 insertions(+), 25 deletions(-) diff --git a/internal/lookout/ui/src/components/lookoutV2/ColumnConfigurationDialog/OrderableColumnListItem.tsx b/internal/lookout/ui/src/components/lookoutV2/ColumnConfigurationDialog/OrderableColumnListItem.tsx index 00bd3520392..405369699c3 100644 --- a/internal/lookout/ui/src/components/lookoutV2/ColumnConfigurationDialog/OrderableColumnListItem.tsx +++ b/internal/lookout/ui/src/components/lookoutV2/ColumnConfigurationDialog/OrderableColumnListItem.tsx @@ -12,6 +12,7 @@ import { ListItemText, Stack, styled, + Tooltip, } from "@mui/material" import { EditAnnotationColumnInput } from "./EditAnnotationColumnInput" @@ -33,10 +34,17 @@ const EditAnnotationColumnInputContainer = styled(Stack)({ width: "100%", }) +const TooltipChildContainer = styled("div")({ + display: "flex", + flexGrow: 1, +}) + export interface OrderableColumnListItemProps { column: JobTableColumn isVisible: boolean onToggleVisibility: () => void + filtered: boolean + sorted: boolean removeAnnotationColumn: () => void editAnnotationColumn: (annotationKey: string) => void existingAnnotationColumnKeysSet: Set @@ -46,6 +54,8 @@ export const OrderableColumnListItem = ({ column, isVisible, onToggleVisibility, + filtered, + sorted, removeAnnotationColumn, editAnnotationColumn, existingAnnotationColumnKeysSet, @@ -59,6 +69,49 @@ export const OrderableColumnListItem = ({ const [isEditing, setIsEditing] = useState(false) + let listItemButtonNode = ( + + + + + + + ) + if (filtered || sorted) { + const title = (() => { + if (filtered && sorted) { + return `The ${colMetadata.displayName} column cannot be hidden because filtering and sorting has been applied to it` + } + if (filtered) { + return `The ${colMetadata.displayName} column cannot be hidden because filtering has been applied to it` + } + return `The ${colMetadata.displayName} column cannot be hidden because sorting has been applied to it` + })() + + listItemButtonNode = ( + + {listItemButtonNode} + + ) + } + return ( ) : ( - - - - - - + listItemButtonNode )} ) diff --git a/internal/lookout/ui/src/components/lookoutV2/ColumnConfigurationDialog/index.tsx b/internal/lookout/ui/src/components/lookoutV2/ColumnConfigurationDialog/index.tsx index cfd26aa003c..1dca75ba4ab 100644 --- a/internal/lookout/ui/src/components/lookoutV2/ColumnConfigurationDialog/index.tsx +++ b/internal/lookout/ui/src/components/lookoutV2/ColumnConfigurationDialog/index.tsx @@ -75,6 +75,8 @@ export interface ColumnConfigurationDialogProps { onClose: () => void allColumns: JobTableColumn[] groupedColumnIds: ColumnId[] + filterColumnIds: ColumnId[] + sortColumnIds: ColumnId[] visibleColumnIds: ColumnId[] columnOrderIds: ColumnId[] setColumnOrder: (columnOrder: ColumnId[]) => void @@ -89,6 +91,8 @@ export const ColumnConfigurationDialog = ({ onClose, allColumns, groupedColumnIds, + filterColumnIds, + sortColumnIds, visibleColumnIds, columnOrderIds, setColumnOrder, @@ -109,14 +113,16 @@ export const ColumnConfigurationDialog = ({ [allColumns], ) + const groupedColumnsSet = useMemo(() => new Set(groupedColumnIds), [groupedColumnIds]) + const filterColumnsSet = useMemo(() => new Set(filterColumnIds), [filterColumnIds]) + const sortColumnsSet = useMemo(() => new Set(sortColumnIds), [sortColumnIds]) + const visibleColumnsSet = useMemo(() => new Set(visibleColumnIds), [visibleColumnIds]) + const orderedColumns = useMemo(() => { - const groupedColumnsSet = new Set(groupedColumnIds) return columnOrderIds .filter((id) => !groupedColumnsSet.has(id) && !PINNED_COLUMNS.includes(toColId(id)) && id in allColumnsById) .map((id) => allColumnsById[id]) - }, [columnOrderIds, groupedColumnIds, allColumnsById]) - - const visibleColumnsSet = useMemo(() => new Set(visibleColumnIds), [visibleColumnIds]) + }, [columnOrderIds, groupedColumnIds, allColumnsById, groupedColumnsSet]) const handleDragEnd = useCallback( ({ active, over }: DragEndEvent) => { @@ -220,6 +226,8 @@ export const ColumnConfigurationDialog = ({ column={column} isVisible={visibleColumnsSet.has(colId)} onToggleVisibility={() => toggleColumnVisibility(colId)} + filtered={filterColumnsSet.has(colId)} + sorted={sortColumnsSet.has(colId)} removeAnnotationColumn={() => onRemoveAnnotationColumn(colId)} editAnnotationColumn={(annotationKey) => onEditAnnotationColumn(colId, annotationKey)} existingAnnotationColumnKeysSet={annotationColumnKeysSet} diff --git a/internal/lookout/ui/src/components/lookoutV2/JobsTableActionBar.tsx b/internal/lookout/ui/src/components/lookoutV2/JobsTableActionBar.tsx index 25014a9f347..b7bcdfab0b4 100644 --- a/internal/lookout/ui/src/components/lookoutV2/JobsTableActionBar.tsx +++ b/internal/lookout/ui/src/components/lookoutV2/JobsTableActionBar.tsx @@ -20,11 +20,12 @@ export interface JobsTableActionBarProps { isLoading: boolean allColumns: JobTableColumn[] groupedColumns: ColumnId[] + filterColumns: ColumnId[] + sortColumns: ColumnId[] visibleColumns: ColumnId[] columnOrder: ColumnId[] setColumnOrder: (columnOrder: ColumnId[]) => void selectedItemFilters: JobFilter[][] - filtersActive: boolean customViews: string[] activeJobSets: boolean onActiveJobSetsChanged: (newVal: boolean) => void @@ -50,11 +51,12 @@ export const JobsTableActionBar = memo( isLoading, allColumns, groupedColumns, + filterColumns, + sortColumns, visibleColumns, columnOrder, setColumnOrder, selectedItemFilters, - filtersActive, customViews, activeJobSets, onActiveJobSetsChanged, @@ -98,6 +100,8 @@ export const JobsTableActionBar = memo( onClose={columnConfigurationDialogOpenOnClose} allColumns={allColumns} groupedColumnIds={groupedColumns} + filterColumnIds={filterColumns} + sortColumnIds={sortColumns} visibleColumnIds={visibleColumns} columnOrderIds={columnOrder} setColumnOrder={setColumnOrder} @@ -156,7 +160,7 @@ export const JobsTableActionBar = memo( onClick={onClearFilters} color="primary" endIcon={} - disabled={!filtersActive} + disabled={filterColumns.length === 0} > Clear Filters diff --git a/internal/lookout/ui/src/containers/lookoutV2/JobsTableContainer.tsx b/internal/lookout/ui/src/containers/lookoutV2/JobsTableContainer.tsx index e7b2d7351bb..aa9c0beb0c1 100644 --- a/internal/lookout/ui/src/containers/lookoutV2/JobsTableContainer.tsx +++ b/internal/lookout/ui/src/containers/lookoutV2/JobsTableContainer.tsx @@ -762,6 +762,9 @@ export const JobsTableContainer = ({ flex: 1, } + const filterColumns = useMemo(() => columnFilterState.map(({ id }) => toColId(id)), [columnFilterState]) + const sortColumns = useMemo(() => [toColId(lookoutOrder.id)], [lookoutOrder]) + return ( @@ -770,7 +773,8 @@ export const JobsTableContainer = ({ isLoading={rowsToFetch.length > 0} allColumns={columnsForSelect} groupedColumns={grouping} - filtersActive={columnFilterState.length > 0} + filterColumns={filterColumns} + sortColumns={sortColumns} visibleColumns={visibleColumnIds} columnOrder={columnOrder} setColumnOrder={setColumnOrder} From 33f3a278468db59b0ba513e96d5e3bc8ebd9d562 Mon Sep 17 00:00:00 2001 From: Rob Smith <34475852+robertdavidsmith@users.noreply.github.com> Date: Wed, 8 Jan 2025 10:51:23 +0000 Subject: [PATCH 17/17] Scheduler: calculate node.allocatableByPriority after load from db (#4127) Signed-off-by: Robert Smith --- internal/scheduler/internaltypes/node.go | 32 +-- .../scheduler/internaltypes/node_factory.go | 16 +- .../internaltypes/resource_list_map_util.go | 9 +- .../resource_list_map_util_test.go | 6 + .../schedulerobjects/schedulerobjects.pb.go | 183 +++++++++--------- .../schedulerobjects/schedulerobjects.proto | 2 +- .../scheduler/scheduling/scheduling_algo.go | 1 + internal/scheduler/simulator/simulator.go | 6 +- internal/scheduler/submitcheck.go | 1 + .../scheduler/testfixtures/testfixtures.go | 2 +- 10 files changed, 139 insertions(+), 119 deletions(-) diff --git a/internal/scheduler/internaltypes/node.go b/internal/scheduler/internaltypes/node.go index 8203a0b3e38..2011219103f 100644 --- a/internal/scheduler/internaltypes/node.go +++ b/internal/scheduler/internaltypes/node.go @@ -4,7 +4,6 @@ import ( "fmt" "math" - "github.com/pkg/errors" "golang.org/x/exp/maps" v1 "k8s.io/api/core/v1" @@ -60,20 +59,19 @@ func FromSchedulerObjectsNode(node *schedulerobjects.Node, nodeIndex uint64, indexedTaints map[string]bool, indexedNodeLabels map[string]bool, + allowedPriorities []int32, resourceListFactory *ResourceListFactory, -) (*Node, error) { +) *Node { + totalResources := resourceListFactory.FromNodeProto(node.TotalResources.Resources) + allocatableByPriority := map[int32]ResourceList{} - minimumPriority := int32(math.MaxInt32) - for p, rl := range node.AllocatableByPriorityAndResource { - if p < minimumPriority { - minimumPriority = p - } - allocatableByPriority[p] = resourceListFactory.FromNodeProto(rl.Resources) + for _, p := range allowedPriorities { + allocatableByPriority[p] = totalResources } - if minimumPriority < 0 { - return nil, errors.Errorf("found negative priority %d on node %s; negative priorities are reserved for internal use", minimumPriority, node.Id) + for p, rl := range node.UnallocatableResources { + MarkAllocated(allocatableByPriority, p, resourceListFactory.FromJobResourceListIgnoreUnknown(rl.Resources)) } - allocatableByPriority[EvictedPriority] = allocatableByPriority[minimumPriority] + allocatableByPriority[EvictedPriority] = allocatableByPriority[minInt32(allowedPriorities)] unallocatableResources := map[int32]ResourceList{} for p, u := range node.UnallocatableResources { @@ -91,10 +89,10 @@ func FromSchedulerObjectsNode(node *schedulerobjects.Node, node.Labels, indexedTaints, indexedNodeLabels, - resourceListFactory.FromNodeProto(node.TotalResources.Resources), + totalResources, unallocatableResources, allocatableByPriority, - ), nil + ) } func CreateNodeAndType( @@ -301,3 +299,11 @@ func deepCopyLabels(labels map[string]string) map[string]string { } return result } + +func minInt32(arr []int32) int32 { + result := int32(math.MaxInt32) + for _, val := range arr { + result = min(result, val) + } + return result +} diff --git a/internal/scheduler/internaltypes/node_factory.go b/internal/scheduler/internaltypes/node_factory.go index 76d454c11ad..850cd7d01ad 100644 --- a/internal/scheduler/internaltypes/node_factory.go +++ b/internal/scheduler/internaltypes/node_factory.go @@ -6,6 +6,7 @@ import ( v1 "k8s.io/api/core/v1" + "github.com/armadaproject/armada/internal/common/types" "github.com/armadaproject/armada/internal/common/util" "github.com/armadaproject/armada/internal/scheduler/schedulerobjects" ) @@ -27,6 +28,9 @@ type NodeFactory struct { // If not set, no labels are indexed. indexedNodeLabels map[string]bool + // Allowed priorities, includes home and away (from config) + allowedPriorities []int32 + // Factory for internaltypes.ResourceList resourceListFactory *ResourceListFactory @@ -37,11 +41,13 @@ type NodeFactory struct { func NewNodeFactory( indexedTaints []string, indexedNodeLabels []string, + priorityClasses map[string]types.PriorityClass, resourceListFactory *ResourceListFactory, ) *NodeFactory { return &NodeFactory{ indexedTaints: util.StringListToSet(indexedTaints), indexedNodeLabels: util.StringListToSet(indexedNodeLabels), + allowedPriorities: types.AllowedPriorities(priorityClasses), resourceListFactory: resourceListFactory, nodeIndexCounter: atomic.Uint64{}, } @@ -76,11 +82,12 @@ func (f *NodeFactory) CreateNodeAndType( ) } -func (f *NodeFactory) FromSchedulerObjectsNode(node *schedulerobjects.Node) (*Node, error) { +func (f *NodeFactory) FromSchedulerObjectsNode(node *schedulerobjects.Node) *Node { return FromSchedulerObjectsNode(node, f.allocateNodeIndex(), f.indexedTaints, f.indexedNodeLabels, + f.allowedPriorities, f.resourceListFactory, ) } @@ -93,12 +100,7 @@ func (f *NodeFactory) FromSchedulerObjectsExecutors(executors []*schedulerobject errorLogger(fmt.Sprintf("Executor name mismatch: %q != %q", node.Executor, executor.Id)) continue } - itNode, err := f.FromSchedulerObjectsNode(node) - if err != nil { - errorLogger(fmt.Sprintf("Invalid node %s: %v", node.Name, err)) - continue - } - result = append(result, itNode) + result = append(result, f.FromSchedulerObjectsNode(node)) } } return result diff --git a/internal/scheduler/internaltypes/resource_list_map_util.go b/internal/scheduler/internaltypes/resource_list_map_util.go index 443f172bd64..78d43cfa583 100644 --- a/internal/scheduler/internaltypes/resource_list_map_util.go +++ b/internal/scheduler/internaltypes/resource_list_map_util.go @@ -1,13 +1,18 @@ package internaltypes import ( + "sort" "strings" + + "golang.org/x/exp/maps" ) func RlMapToString(m map[string]ResourceList) string { + keys := maps.Keys(m) + sort.Strings(keys) results := []string{} - for k, v := range m { - results = append(results, k+"="+v.String()) + for _, k := range keys { + results = append(results, k+"="+m[k].String()) } return strings.Join(results, " ") } diff --git a/internal/scheduler/internaltypes/resource_list_map_util_test.go b/internal/scheduler/internaltypes/resource_list_map_util_test.go index 7f6fb74b9b6..199d3a00322 100644 --- a/internal/scheduler/internaltypes/resource_list_map_util_test.go +++ b/internal/scheduler/internaltypes/resource_list_map_util_test.go @@ -6,6 +6,12 @@ import ( "github.com/stretchr/testify/assert" ) +func TestRlMapToString(t *testing.T) { + factory := testFactory() + + assert.Equal(t, "a=(memory=1024,cpu=1) b=(memory=2048,cpu=2)", RlMapToString(testMapAllPositive(factory))) +} + func TestRlMapSumValues(t *testing.T) { factory := testFactory() diff --git a/internal/scheduler/schedulerobjects/schedulerobjects.pb.go b/internal/scheduler/schedulerobjects/schedulerobjects.pb.go index fce7b391122..031b1aa097c 100644 --- a/internal/scheduler/schedulerobjects/schedulerobjects.pb.go +++ b/internal/scheduler/schedulerobjects/schedulerobjects.pb.go @@ -166,7 +166,7 @@ type Node struct { // Resources available for jobs of a given priority. // E.g., AvailableResources[5]["cpu"] is the amount of CPU available to jobs with priority 5, // where available resources = unused resources + resources assigned to lower-priority jobs. - AllocatableByPriorityAndResource map[int32]ResourceList `protobuf:"bytes,8,rep,name=allocatable_by_priority_and_resource,json=allocatableByPriorityAndResource,proto3" json:"allocatableByPriorityAndResource" protobuf_key:"varint,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + AllocatableByPriorityAndResource map[int32]ResourceList `protobuf:"bytes,8,rep,name=allocatable_by_priority_and_resource,json=allocatableByPriorityAndResource,proto3" json:"allocatableByPriorityAndResource" protobuf_key:"varint,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` // Deprecated: Do not use. // Ids of the job runs currently assigned to this node and their current state. StateByJobRunId map[string]JobRunState `protobuf:"bytes,9,rep,name=state_by_job_run_id,json=stateByJobRunId,proto3" json:"stateByJobRunId" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3,enum=schedulerobjects.JobRunState"` // Resource allocated to non-Armada pods on this node. @@ -263,6 +263,7 @@ func (m *Node) GetTotalResources() ResourceList { return ResourceList{} } +// Deprecated: Do not use. func (m *Node) GetAllocatableByPriorityAndResource() map[int32]ResourceList { if m != nil { return m.AllocatableByPriorityAndResource @@ -788,116 +789,116 @@ func init() { } var fileDescriptor_97dadc5fbd620721 = []byte{ - // 1735 bytes of a gzipped FileDescriptorProto + // 1737 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x58, 0xcd, 0x6f, 0xdb, 0xc8, 0x15, 0x37, 0xfd, 0x29, 0x8d, 0xfc, 0x21, 0x8d, 0xbd, 0x09, 0xa3, 0x64, 0x45, 0x55, 0xbb, 0x5b, - 0x38, 0x6d, 0x97, 0xc2, 0x7a, 0x5b, 0x20, 0x48, 0x8b, 0x02, 0x62, 0xe2, 0x36, 0x4e, 0xb3, 0xb2, - 0x23, 0x47, 0x28, 0x5a, 0xa0, 0x4b, 0x8c, 0xc8, 0xb1, 0xc2, 0x35, 0x35, 0xc3, 0x25, 0x87, 0x69, - 0xd9, 0x7b, 0x0f, 0x45, 0x2f, 0xdb, 0x62, 0x0b, 0x14, 0xe8, 0xa9, 0xff, 0x44, 0x6f, 0x45, 0xaf, - 0x41, 0x4f, 0x7b, 0xec, 0x89, 0x2d, 0x92, 0x1b, 0xff, 0x83, 0xde, 0x8a, 0x19, 0x92, 0xe2, 0x48, - 0x94, 0x2d, 0x5f, 0x16, 0x7b, 0xb2, 0xe7, 0xf7, 0x3e, 0xe7, 0xcd, 0x6f, 0xde, 0x3c, 0x0a, 0x3c, + 0x38, 0xdb, 0x2e, 0x85, 0xf5, 0xb6, 0x40, 0x90, 0x16, 0x05, 0xc4, 0xc4, 0x6d, 0x9c, 0x66, 0x65, + 0x47, 0x8e, 0x50, 0xb4, 0x40, 0x97, 0x18, 0x91, 0x63, 0x85, 0x6b, 0x6a, 0x86, 0x4b, 0x0e, 0xd3, + 0xb2, 0xf7, 0x1e, 0x8a, 0x5e, 0xb6, 0xc5, 0x16, 0x28, 0xd0, 0x53, 0xff, 0x8b, 0x5e, 0x8a, 0x5e, + 0x83, 0x9e, 0xf6, 0xd8, 0x13, 0x5b, 0x24, 0x37, 0xfe, 0x0b, 0xbd, 0x14, 0x33, 0x24, 0xc5, 0x91, + 0x28, 0x5b, 0xbe, 0x2c, 0x7a, 0xb2, 0xe7, 0xf7, 0x3e, 0xe7, 0xcd, 0x6f, 0xde, 0x3c, 0x0a, 0x3c, 0x74, 0x08, 0xc3, 0x3e, 0x41, 0x6e, 0x37, 0xb0, 0x5e, 0x62, 0x3b, 0x74, 0xb1, 0x5f, 0xfc, 0x47, - 0x47, 0x9f, 0x61, 0x8b, 0x05, 0x25, 0x40, 0xf7, 0x7c, 0xca, 0x28, 0xac, 0xcf, 0xe3, 0x4d, 0x6d, + 0x47, 0x9f, 0x63, 0x8b, 0x05, 0x25, 0x40, 0xf7, 0x7c, 0xca, 0x28, 0xac, 0xcf, 0xe3, 0x4d, 0x6d, 0x4c, 0xe9, 0xd8, 0xc5, 0x5d, 0x21, 0x1f, 0x85, 0x17, 0x5d, 0xe6, 0x4c, 0x70, 0xc0, 0xd0, 0xc4, 0x4b, 0x4d, 0x9a, 0x9d, 0xcb, 0x07, 0x81, 0xee, 0xd0, 0x2e, 0xf2, 0x9c, 0xae, 0x45, 0x7d, 0xdc, - 0x7d, 0xf5, 0x51, 0x77, 0x8c, 0x09, 0xf6, 0x11, 0xc3, 0x76, 0xa6, 0xf3, 0xfd, 0x42, 0x67, 0x82, + 0x7d, 0xf5, 0x71, 0x77, 0x8c, 0x09, 0xf6, 0x11, 0xc3, 0x76, 0xa6, 0xf3, 0xbd, 0x42, 0x67, 0x82, 0xac, 0x97, 0x0e, 0xc1, 0x7e, 0xd4, 0xf5, 0x2e, 0xc7, 0xc2, 0xc8, 0xc7, 0x01, 0x0d, 0x7d, 0x0b, - 0x97, 0xac, 0x3e, 0x1c, 0x3b, 0xec, 0x65, 0x38, 0xd2, 0x2d, 0x3a, 0xe9, 0x8e, 0xe9, 0x98, 0x16, - 0x39, 0xf0, 0x95, 0x58, 0x88, 0xff, 0x52, 0xf5, 0xce, 0xeb, 0x55, 0x50, 0x39, 0xfe, 0x0d, 0xb6, + 0x97, 0xac, 0x3e, 0x1a, 0x3b, 0xec, 0x65, 0x38, 0xd2, 0x2d, 0x3a, 0xe9, 0x8e, 0xe9, 0x98, 0x16, + 0x39, 0xf0, 0x95, 0x58, 0x88, 0xff, 0x52, 0xf5, 0xce, 0xeb, 0x55, 0x50, 0x39, 0xfe, 0x35, 0xb6, 0x42, 0x46, 0x7d, 0xd8, 0x06, 0xab, 0x8e, 0xad, 0x2a, 0x6d, 0xe5, 0xb0, 0x6a, 0xd4, 0x93, 0x58, - 0xdb, 0x76, 0xec, 0xef, 0xd1, 0x89, 0xc3, 0xf0, 0xc4, 0x63, 0xd1, 0x60, 0xd5, 0xb1, 0xe1, 0xb7, + 0xdb, 0x76, 0xec, 0xef, 0xd2, 0x89, 0xc3, 0xf0, 0xc4, 0x63, 0xd1, 0x60, 0xd5, 0xb1, 0xe1, 0xb7, 0xc1, 0xba, 0x47, 0xa9, 0xab, 0xae, 0x0a, 0x1d, 0x98, 0xc4, 0xda, 0x2e, 0x5f, 0x4b, 0x5a, 0x42, 0x0e, 0x7b, 0x60, 0x83, 0x50, 0x1b, 0x07, 0xea, 0x5a, 0x7b, 0xed, 0xb0, 0x76, 0x74, 0x4b, 0x2f, 0x95, 0xae, 0x4f, 0x6d, 0x6c, 0xec, 0x27, 0xb1, 0xb6, 0x27, 0x14, 0x25, 0x0f, 0xa9, 0x25, 0xfc, - 0x14, 0xec, 0xba, 0x28, 0x60, 0x43, 0xcf, 0x46, 0x0c, 0xbf, 0x70, 0x26, 0x58, 0xdd, 0x68, 0x2b, + 0x0c, 0xec, 0xba, 0x28, 0x60, 0x43, 0xcf, 0x46, 0x0c, 0xbf, 0x70, 0x26, 0x58, 0xdd, 0x68, 0x2b, 0x87, 0xb5, 0xa3, 0xa6, 0x9e, 0x16, 0x57, 0xcf, 0x37, 0xa6, 0xbf, 0xc8, 0x8b, 0x6b, 0x34, 0x5f, - 0xc7, 0xda, 0x0a, 0x4f, 0x6a, 0xd6, 0xf2, 0x8b, 0xff, 0x68, 0xca, 0x60, 0x0e, 0x83, 0xa7, 0x60, - 0x3f, 0x24, 0x28, 0x08, 0x9c, 0x31, 0xc1, 0xb6, 0xf9, 0x19, 0x1d, 0x99, 0x7e, 0x48, 0x02, 0xb5, + 0xc7, 0xda, 0x0a, 0x4f, 0x6a, 0xd6, 0xf2, 0xcb, 0x7f, 0x6b, 0xca, 0x60, 0x0e, 0x83, 0xa7, 0x60, + 0x3f, 0x24, 0x28, 0x08, 0x9c, 0x31, 0xc1, 0xb6, 0xf9, 0x39, 0x1d, 0x99, 0x7e, 0x48, 0x02, 0xb5, 0xda, 0x5e, 0x3b, 0xac, 0x1a, 0x5a, 0x12, 0x6b, 0x77, 0x0b, 0xf1, 0x53, 0x3a, 0x1a, 0x84, 0x44, - 0x4e, 0xb2, 0x51, 0x12, 0x76, 0xfe, 0xb7, 0x03, 0xd6, 0xf9, 0xae, 0x6e, 0x56, 0x46, 0x82, 0x26, + 0x4e, 0xb2, 0x51, 0x12, 0x76, 0xfe, 0xbb, 0x03, 0xd6, 0xf9, 0xae, 0x6e, 0x56, 0x46, 0x82, 0x26, 0x58, 0xdd, 0x2e, 0xca, 0xc8, 0xd7, 0x72, 0x19, 0xf9, 0x1a, 0x1e, 0x81, 0x0a, 0xce, 0x0e, 0x47, - 0xdd, 0x17, 0xba, 0xb7, 0x92, 0x58, 0x83, 0x39, 0x26, 0xe9, 0x4f, 0xf5, 0xe0, 0x27, 0xa0, 0xca, + 0xdd, 0x17, 0xba, 0xb7, 0x92, 0x58, 0x83, 0x39, 0x26, 0xe9, 0x4f, 0xf5, 0xe0, 0xa7, 0xa0, 0xca, 0x77, 0x6a, 0x06, 0x18, 0x13, 0x71, 0x4e, 0xd7, 0x97, 0xec, 0x20, 0x2b, 0x59, 0x85, 0x1b, 0x9d, 0x63, 0x4c, 0x44, 0xb1, 0xa6, 0x2b, 0xd8, 0x03, 0x9b, 0x0c, 0x39, 0x84, 0x05, 0xea, 0x86, 0x38, - 0xca, 0x3b, 0x7a, 0x4a, 0x4b, 0x1d, 0x79, 0x8e, 0xce, 0xa9, 0xab, 0xbf, 0xfa, 0x48, 0x7f, 0xc1, + 0xca, 0x3b, 0x7a, 0x4a, 0x4b, 0x1d, 0x79, 0x8e, 0xce, 0xa9, 0xab, 0xbf, 0xfa, 0x58, 0x7f, 0xc1, 0x35, 0x8c, 0xdd, 0xcc, 0x55, 0x66, 0x30, 0xc8, 0xfe, 0xc2, 0x33, 0xb0, 0xe9, 0xa2, 0x11, 0x76, 0x03, 0x75, 0x53, 0xb8, 0xe8, 0x2c, 0x66, 0x83, 0xfe, 0x4c, 0x28, 0x1d, 0x13, 0xe6, 0x47, 0xc6, 0x41, 0x12, 0x6b, 0xf5, 0xd4, 0x4a, 0xda, 0x65, 0xe6, 0x07, 0x9a, 0x60, 0x8f, 0x51, 0x86, 0x5c, 0x33, 0xbf, 0x06, 0x81, 0xba, 0x25, 0x76, 0xda, 0x2a, 0xbb, 0x1e, 0x64, 0x2a, 0xcf, 0x9c, 0x80, - 0x19, 0xb7, 0x72, 0x82, 0x08, 0xf3, 0x5c, 0x14, 0x0c, 0xe6, 0xd6, 0xf0, 0xef, 0x0a, 0x78, 0x1f, + 0x19, 0xb7, 0x72, 0x82, 0x08, 0xf3, 0x5c, 0x14, 0x0c, 0xe6, 0xd6, 0xf0, 0x6f, 0x0a, 0x78, 0x1f, 0xb9, 0x2e, 0xb5, 0x10, 0x43, 0x23, 0x17, 0x9b, 0xa3, 0xc8, 0xf4, 0x7c, 0x87, 0xfa, 0x0e, 0x8b, - 0x4c, 0x44, 0xec, 0x69, 0x5c, 0xb5, 0x22, 0x76, 0xf4, 0xa3, 0x2b, 0x76, 0xd4, 0x2b, 0x5c, 0x18, - 0xd1, 0x59, 0xe6, 0xa0, 0x47, 0xec, 0x3c, 0x50, 0xba, 0xd7, 0xc3, 0x2c, 0xa9, 0x36, 0x5a, 0xa2, - 0x3e, 0x58, 0xaa, 0x01, 0x7d, 0xb0, 0x1f, 0x30, 0xc4, 0x44, 0xc6, 0x19, 0xa7, 0x4d, 0xc7, 0x16, - 0xac, 0xae, 0x1d, 0x7d, 0xf7, 0x8a, 0x34, 0xcf, 0xb9, 0x85, 0x11, 0xa5, 0x44, 0x3e, 0xb1, 0xd3, - 0xac, 0x6e, 0x67, 0x59, 0xed, 0x05, 0xb3, 0xd2, 0xc1, 0x3c, 0x00, 0xff, 0xa0, 0x80, 0xdb, 0x21, - 0x91, 0xcb, 0x55, 0x1c, 0xcb, 0x8e, 0x08, 0x7c, 0x74, 0x45, 0xe0, 0xa1, 0x6c, 0x35, 0xad, 0x7e, - 0x1a, 0xbf, 0x95, 0xc5, 0xbf, 0x15, 0x2e, 0x54, 0x1a, 0x5c, 0x81, 0xc3, 0x1e, 0xd8, 0x09, 0x49, - 0x16, 0x8e, 0x4b, 0xd4, 0xbd, 0xb6, 0x72, 0x58, 0x31, 0xee, 0x26, 0xb1, 0x76, 0x7b, 0x46, 0x20, - 0xf1, 0x6a, 0xd6, 0x82, 0xb7, 0x06, 0x1f, 0x7b, 0xd4, 0x67, 0x0e, 0x19, 0x9b, 0xbc, 0x1b, 0x99, - 0x2c, 0xf2, 0xb0, 0xda, 0x10, 0x37, 0x50, 0xb4, 0x86, 0xa9, 0x98, 0x6f, 0xe3, 0x45, 0xe4, 0xc9, - 0xce, 0x1a, 0x25, 0xe1, 0xb4, 0x6d, 0xc2, 0x25, 0x6d, 0xf3, 0xcf, 0x0a, 0x68, 0xe7, 0xb5, 0x33, - 0xc3, 0x00, 0x8d, 0xc5, 0x39, 0x7e, 0x1e, 0xe2, 0x10, 0x0b, 0xda, 0x09, 0x27, 0x07, 0xa2, 0xa4, - 0xef, 0x95, 0x4b, 0x7a, 0x46, 0xa9, 0xfb, 0x9c, 0xeb, 0xe6, 0xc5, 0x30, 0xee, 0x27, 0xb1, 0xf6, - 0x41, 0xee, 0x70, 0xc8, 0xfd, 0x19, 0x91, 0xd0, 0xe8, 0x11, 0xfb, 0x6c, 0x36, 0x81, 0xbb, 0xd7, - 0xa8, 0x35, 0x11, 0xa8, 0x49, 0x97, 0x13, 0xbe, 0x07, 0xd6, 0x2e, 0x71, 0x94, 0x75, 0xb8, 0x46, - 0x12, 0x6b, 0x3b, 0x97, 0x38, 0x92, 0x7c, 0x71, 0x29, 0xbc, 0x0f, 0x36, 0x5e, 0x21, 0x37, 0xc4, - 0xd9, 0x5b, 0x21, 0x5a, 0xbd, 0x00, 0xe4, 0x56, 0x2f, 0x80, 0x87, 0xab, 0x0f, 0x94, 0xe6, 0x5f, - 0x15, 0xf0, 0xc1, 0x8d, 0xae, 0x8b, 0x1c, 0x7d, 0xe3, 0xca, 0xe8, 0x27, 0x72, 0xf4, 0xe5, 0x7d, - 0x61, 0x59, 0x76, 0xbf, 0x57, 0xc0, 0xc1, 0xa2, 0x5b, 0x72, 0xb3, 0x52, 0x3c, 0x91, 0x93, 0xd9, - 0x3d, 0x7a, 0xb7, 0x9c, 0x4c, 0xea, 0x34, 0x8d, 0xb0, 0x2c, 0x97, 0x3f, 0x2a, 0xe0, 0xee, 0x35, - 0x17, 0xe7, 0x9b, 0xa8, 0x4f, 0xe7, 0x1f, 0x0a, 0x68, 0x94, 0xe8, 0x37, 0xa5, 0xbd, 0xb2, 0x84, - 0xf6, 0xf7, 0xc1, 0x86, 0xe0, 0xb8, 0x4c, 0x15, 0x01, 0xc8, 0xc1, 0x04, 0x00, 0x87, 0xa0, 0x5a, - 0x34, 0x97, 0xb5, 0x1b, 0xe5, 0x7e, 0x3b, 0x89, 0xb5, 0xfd, 0xa9, 0x91, 0xe4, 0xb2, 0xf0, 0xd4, - 0xf9, 0xdd, 0x2a, 0xd8, 0x96, 0x8d, 0xe0, 0xa7, 0x72, 0x1c, 0x45, 0xdc, 0xb8, 0x0f, 0xaf, 0x8f, - 0xa3, 0xcf, 0xf5, 0xaf, 0x46, 0xd6, 0xbf, 0x0a, 0x3f, 0x52, 0xc0, 0xe6, 0x97, 0x0a, 0xd8, 0xbd, - 0xfa, 0xdc, 0xae, 0xa6, 0xd2, 0x2f, 0x66, 0xcf, 0x4d, 0x97, 0x5e, 0xe3, 0xe9, 0x90, 0xa8, 0x7b, - 0x97, 0x63, 0xf1, 0x3c, 0xe7, 0xe1, 0xf4, 0xe7, 0x21, 0x22, 0xcc, 0x61, 0xd1, 0xd2, 0x73, 0xfc, - 0x72, 0x03, 0x34, 0x9e, 0xd2, 0xd1, 0x79, 0xba, 0x51, 0x87, 0x8c, 0x4f, 0xc8, 0x05, 0xe5, 0x63, - 0x88, 0xeb, 0x5c, 0x60, 0x3e, 0xc4, 0x8a, 0xf4, 0x76, 0xd2, 0x31, 0x24, 0xc7, 0xe4, 0x31, 0x24, - 0xc7, 0xe0, 0x43, 0xb0, 0x8d, 0x98, 0x39, 0xa1, 0x01, 0x33, 0x29, 0xb1, 0xd2, 0x7c, 0x2b, 0x86, - 0x9a, 0xc4, 0xda, 0x01, 0x62, 0x9f, 0xd0, 0x80, 0x9d, 0x12, 0x4b, 0xb6, 0x04, 0x05, 0x0a, 0x7f, - 0x08, 0x6a, 0x9e, 0x8f, 0x39, 0xee, 0xf0, 0x06, 0xbe, 0x26, 0x4c, 0xef, 0x24, 0xb1, 0xf6, 0x8e, - 0x04, 0x4b, 0xb6, 0xb2, 0x36, 0x7c, 0x02, 0xea, 0x16, 0x25, 0x56, 0xe8, 0xfb, 0x98, 0x58, 0x91, - 0x19, 0xa0, 0x0b, 0xac, 0xae, 0x0b, 0x0f, 0xef, 0x26, 0xb1, 0x76, 0x47, 0x92, 0x9d, 0xa3, 0x0b, - 0xd9, 0xcb, 0xde, 0x9c, 0x88, 0x3f, 0x03, 0xd3, 0x47, 0xdf, 0x72, 0x51, 0x10, 0x98, 0x62, 0x68, - 0xdb, 0x2c, 0x9e, 0x81, 0x5c, 0xfc, 0x88, 0x4b, 0xfb, 0xb3, 0x13, 0x5c, 0xa3, 0x24, 0x84, 0xe7, - 0xa0, 0x16, 0x84, 0xa3, 0x89, 0xc3, 0x4c, 0x51, 0xca, 0xad, 0xa5, 0xc3, 0x59, 0x3e, 0xae, 0x80, - 0xd4, 0x6c, 0x3a, 0xcb, 0x4a, 0x6b, 0x7e, 0x38, 0x79, 0x24, 0xb5, 0x52, 0x1c, 0x4e, 0x8e, 0xc9, - 0x87, 0x93, 0x63, 0xf0, 0xd7, 0x60, 0x3f, 0xa5, 0xb0, 0xe9, 0xe3, 0xcf, 0x43, 0xc7, 0xc7, 0x13, - 0x5c, 0x4c, 0x78, 0xef, 0x97, 0x79, 0x7e, 0x2a, 0xfe, 0x0e, 0x24, 0x5d, 0xa3, 0x9d, 0xc4, 0xda, - 0x3d, 0x5a, 0xc2, 0xa5, 0x70, 0xb0, 0x2c, 0x85, 0x5d, 0xb0, 0xf5, 0x0a, 0xfb, 0x81, 0x43, 0x89, - 0x5a, 0x15, 0xb9, 0xbe, 0x93, 0xc4, 0x5a, 0x23, 0x83, 0x24, 0xdb, 0x5c, 0xeb, 0xe1, 0xfa, 0x5f, - 0xfe, 0xa6, 0x29, 0x9d, 0x3f, 0x29, 0x00, 0x96, 0x73, 0x80, 0x2e, 0xd8, 0xf3, 0xa8, 0x2d, 0x43, - 0x82, 0x9e, 0xb5, 0xa3, 0x6f, 0x2d, 0x7a, 0x1c, 0x67, 0x14, 0x53, 0x32, 0xcc, 0x59, 0x17, 0x09, - 0x3c, 0x59, 0x19, 0xcc, 0xbb, 0x36, 0x76, 0xc1, 0xb6, 0x5c, 0xad, 0xce, 0x3f, 0x37, 0xc1, 0xde, - 0x9c, 0x57, 0x18, 0x80, 0x6d, 0x3e, 0x2f, 0x9c, 0x63, 0x17, 0x5b, 0x7c, 0x68, 0x4f, 0x3b, 0xc7, - 0xc7, 0x4b, 0xd3, 0x11, 0xe3, 0x50, 0x6e, 0x95, 0xf6, 0x8f, 0x26, 0x9f, 0x7d, 0x64, 0x67, 0x52, - 0x79, 0x66, 0x82, 0xc0, 0x33, 0x50, 0x41, 0x17, 0x17, 0x0e, 0xe1, 0x0c, 0x48, 0xdb, 0xc2, 0xbd, - 0x45, 0x43, 0x7a, 0x2f, 0xd3, 0x49, 0xf9, 0x91, 0x5b, 0xc8, 0xfc, 0xc8, 0x31, 0x38, 0x04, 0x35, - 0x46, 0x5d, 0xfe, 0x5d, 0xe9, 0x50, 0x92, 0x7f, 0xc4, 0xb5, 0x16, 0x4e, 0xfe, 0x53, 0x35, 0x63, - 0x3f, 0x23, 0xab, 0x6c, 0x3a, 0x90, 0x17, 0x90, 0x82, 0x1a, 0x22, 0x84, 0xb2, 0xcc, 0xed, 0xd6, - 0x55, 0xb3, 0xe1, 0x7c, 0x71, 0x7a, 0x85, 0x51, 0x5a, 0x1b, 0xd1, 0x0b, 0x24, 0x57, 0x72, 0x2f, - 0x90, 0x60, 0xf8, 0x14, 0xd4, 0xf3, 0xd6, 0x40, 0xc9, 0x19, 0x75, 0x1d, 0x2b, 0x12, 0x5f, 0x91, - 0x55, 0xa3, 0x95, 0xc4, 0x5a, 0x73, 0x5e, 0x26, 0xb9, 0x29, 0xd9, 0xc1, 0xdf, 0x82, 0x83, 0xbc, - 0x9f, 0xce, 0x30, 0x6e, 0x53, 0x54, 0xfc, 0x70, 0x51, 0x71, 0x06, 0x0b, 0xf4, 0x8d, 0x7b, 0x59, - 0x99, 0x16, 0x7a, 0x1b, 0x2c, 0x44, 0x9b, 0x63, 0xd0, 0x28, 0x11, 0xe4, 0x6b, 0x99, 0xc2, 0x2e, - 0x40, 0x7d, 0xbe, 0xd8, 0x5f, 0x47, 0x9c, 0xa7, 0xeb, 0x95, 0x4a, 0xbd, 0xda, 0xf9, 0xd7, 0x2a, - 0xa8, 0xe7, 0x3f, 0x3e, 0x9c, 0x63, 0xc6, 0x47, 0xe6, 0x00, 0x3e, 0x00, 0x20, 0xff, 0x96, 0x3d, - 0xc9, 0xbf, 0xa2, 0xc5, 0xb3, 0x51, 0xa0, 0xf2, 0xb3, 0x51, 0xa0, 0xbc, 0x13, 0x5a, 0xd4, 0xb7, - 0x29, 0xc1, 0x76, 0xf6, 0xdc, 0x08, 0xa6, 0xe7, 0x98, 0xcc, 0xf4, 0x1c, 0x83, 0x3f, 0x06, 0xdb, - 0xe9, 0xff, 0x03, 0x8c, 0x02, 0x4a, 0xc4, 0x5b, 0x53, 0x4d, 0xef, 0x9e, 0x8c, 0xcb, 0x77, 0x4f, - 0xc6, 0xe1, 0x0f, 0x40, 0x35, 0xc0, 0xcc, 0x88, 0x86, 0x01, 0xf6, 0xc5, 0x33, 0x53, 0x4d, 0xe7, - 0x8d, 0x29, 0x28, 0xcf, 0x1b, 0x53, 0x10, 0x3e, 0x17, 0x66, 0x3d, 0x76, 0xc3, 0xdf, 0x35, 0x72, - 0x97, 0xa9, 0xc1, 0x9c, 0xcb, 0x14, 0xfc, 0xce, 0x29, 0xa8, 0x49, 0x53, 0x24, 0xac, 0x81, 0xad, - 0x61, 0xff, 0x67, 0xfd, 0xd3, 0x9f, 0xf7, 0xeb, 0x2b, 0x7c, 0x71, 0x76, 0xdc, 0x7f, 0x7c, 0xd2, - 0xff, 0x69, 0x5d, 0xe1, 0x8b, 0xc1, 0xb0, 0xdf, 0xe7, 0x8b, 0x55, 0xb8, 0x03, 0xaa, 0xe7, 0xc3, - 0x47, 0x8f, 0x8e, 0x8f, 0x1f, 0x1f, 0x3f, 0xae, 0xaf, 0x41, 0x00, 0x36, 0x7f, 0xd2, 0x3b, 0x79, - 0x76, 0xfc, 0xb8, 0xbe, 0x6e, 0xfc, 0xea, 0xf5, 0x9b, 0x96, 0xf2, 0xd5, 0x9b, 0x96, 0xf2, 0xdf, - 0x37, 0x2d, 0xe5, 0x8b, 0xb7, 0xad, 0x95, 0xaf, 0xde, 0xb6, 0x56, 0xfe, 0xfd, 0xb6, 0xb5, 0xf2, - 0xcb, 0x47, 0xd2, 0x6f, 0x4c, 0xc8, 0x9f, 0x20, 0x1b, 0x79, 0x3e, 0xe5, 0x57, 0x37, 0x5b, 0x75, - 0x6f, 0xf0, 0x63, 0xda, 0x68, 0x53, 0xec, 0xf3, 0xe3, 0xff, 0x07, 0x00, 0x00, 0xff, 0xff, 0x4f, - 0xef, 0xf8, 0x20, 0x7a, 0x13, 0x00, 0x00, + 0x4c, 0x44, 0xec, 0x69, 0x5c, 0xb5, 0x22, 0x76, 0xf4, 0xc3, 0x2b, 0x76, 0xd4, 0x2b, 0x5c, 0x18, + 0xd1, 0x59, 0xe6, 0xa0, 0x47, 0xec, 0x3c, 0x50, 0xba, 0xd7, 0x0f, 0xb3, 0xa4, 0xda, 0x68, 0x89, + 0xba, 0xaa, 0x0c, 0x96, 0xea, 0x40, 0x1f, 0xec, 0x07, 0x0c, 0x31, 0x91, 0x73, 0xc6, 0x6a, 0xd3, + 0xb1, 0x05, 0xaf, 0x6b, 0x47, 0xdf, 0xb9, 0x22, 0xd1, 0x73, 0x6e, 0x61, 0x44, 0x29, 0x95, 0x4f, + 0xec, 0x34, 0xaf, 0xdb, 0x59, 0x5e, 0x7b, 0xc1, 0xac, 0x74, 0x30, 0x0f, 0xc0, 0xdf, 0x2b, 0xe0, + 0x76, 0x48, 0xe4, 0x82, 0x15, 0x07, 0xb3, 0x23, 0x02, 0x1f, 0x5d, 0x11, 0x78, 0x28, 0x5b, 0x4d, + 0xeb, 0x9f, 0xc6, 0x6f, 0x65, 0xf1, 0x6f, 0x85, 0x0b, 0x95, 0x06, 0x57, 0xe0, 0xb0, 0x07, 0x76, + 0x42, 0x92, 0x85, 0xe3, 0x12, 0x75, 0xaf, 0xad, 0x1c, 0x56, 0x8c, 0xbb, 0x49, 0xac, 0xdd, 0x9e, + 0x11, 0x48, 0xcc, 0x9a, 0xb5, 0xe0, 0xcd, 0xc1, 0xc7, 0x1e, 0xf5, 0x99, 0x43, 0xc6, 0x26, 0xef, + 0x47, 0x26, 0x8b, 0x3c, 0xac, 0x36, 0xc4, 0x1d, 0x14, 0xcd, 0x61, 0x2a, 0xe6, 0xdb, 0x78, 0x11, + 0x79, 0xb2, 0xb3, 0x46, 0x49, 0x38, 0x6d, 0x9c, 0x70, 0x49, 0xe3, 0xfc, 0x93, 0x02, 0xda, 0x79, + 0xed, 0xcc, 0x30, 0x40, 0x63, 0x71, 0x8e, 0x5f, 0x84, 0x38, 0xc4, 0x82, 0x78, 0xc2, 0xc9, 0x81, + 0x28, 0xe9, 0x7b, 0xe5, 0x92, 0x9e, 0x51, 0xea, 0x3e, 0xe7, 0xba, 0x79, 0x31, 0x8c, 0xfb, 0x49, + 0xac, 0x7d, 0x90, 0x3b, 0x1c, 0x72, 0x7f, 0x46, 0x24, 0x34, 0x7a, 0xc4, 0x3e, 0x9b, 0x4d, 0xe0, + 0xee, 0x35, 0x6a, 0x4d, 0x04, 0x6a, 0xd2, 0xf5, 0x84, 0xef, 0x81, 0xb5, 0x4b, 0x1c, 0x65, 0x3d, + 0xae, 0x91, 0xc4, 0xda, 0xce, 0x25, 0x8e, 0x24, 0x5f, 0x5c, 0x0a, 0xef, 0x83, 0x8d, 0x57, 0xc8, + 0x0d, 0x71, 0xf6, 0x5a, 0x88, 0x66, 0x2f, 0x00, 0xb9, 0xd9, 0x0b, 0xe0, 0xe1, 0xea, 0x03, 0xa5, + 0xf9, 0x17, 0x05, 0x7c, 0x70, 0xa3, 0x0b, 0x23, 0x47, 0xdf, 0xb8, 0x32, 0xfa, 0x89, 0x1c, 0x7d, + 0x79, 0x67, 0x58, 0x96, 0xdd, 0xef, 0x14, 0x70, 0xb0, 0xe8, 0x96, 0xdc, 0xac, 0x14, 0x4f, 0xe4, + 0x64, 0x76, 0x8f, 0xde, 0x2d, 0x27, 0x93, 0x3a, 0x4d, 0x23, 0x2c, 0xcb, 0xe5, 0x0f, 0x0a, 0xb8, + 0x7b, 0xcd, 0xc5, 0xf9, 0x7f, 0xd4, 0xa7, 0xf3, 0x77, 0x05, 0x34, 0x4a, 0xf4, 0x9b, 0xd2, 0x5e, + 0x59, 0x42, 0xfb, 0xfb, 0x60, 0x43, 0x70, 0x5c, 0xa6, 0x8a, 0x00, 0xe4, 0x60, 0x02, 0x80, 0x43, + 0x50, 0x2d, 0x9a, 0xcb, 0xda, 0x8d, 0x72, 0xbf, 0x9d, 0xc4, 0xda, 0xfe, 0xd4, 0x48, 0x72, 0x59, + 0x78, 0xea, 0xfc, 0x76, 0x15, 0x6c, 0xcb, 0x46, 0xf0, 0x33, 0x39, 0x8e, 0x22, 0x6e, 0xdc, 0x47, + 0xd7, 0xc7, 0xd1, 0xe7, 0xfa, 0x57, 0x23, 0xeb, 0x5f, 0x85, 0x1f, 0x29, 0x60, 0xf3, 0x2b, 0x05, + 0xec, 0x5e, 0x7d, 0x6e, 0x57, 0x53, 0xe9, 0xe7, 0xb3, 0xe7, 0xa6, 0x4b, 0xef, 0xf1, 0x74, 0x4c, + 0xd4, 0xbd, 0xcb, 0xb1, 0x78, 0xa0, 0xf3, 0x70, 0xfa, 0xf3, 0x10, 0x11, 0xe6, 0xb0, 0x68, 0xe9, + 0x39, 0x7e, 0xb5, 0x01, 0x1a, 0x4f, 0xe9, 0xe8, 0x3c, 0xdd, 0xa8, 0x43, 0xc6, 0x27, 0xe4, 0x82, + 0xf2, 0x41, 0xc4, 0x75, 0x2e, 0x30, 0x1f, 0x63, 0x45, 0x7a, 0x3b, 0xe9, 0x20, 0x92, 0x63, 0xf2, + 0x20, 0x92, 0x63, 0xf0, 0x21, 0xd8, 0x46, 0xcc, 0x9c, 0xd0, 0x80, 0x99, 0x94, 0x58, 0x69, 0xbe, + 0x15, 0x43, 0x4d, 0x62, 0xed, 0x00, 0xb1, 0x4f, 0x69, 0xc0, 0x4e, 0x89, 0x25, 0x5b, 0x82, 0x02, + 0x85, 0x3f, 0x00, 0x35, 0xcf, 0xc7, 0x1c, 0x77, 0x78, 0x03, 0x5f, 0x13, 0xa6, 0x77, 0x92, 0x58, + 0x7b, 0x47, 0x82, 0x25, 0x5b, 0x59, 0x1b, 0x3e, 0x01, 0x75, 0x8b, 0x12, 0x2b, 0xf4, 0x7d, 0x4c, + 0xac, 0xc8, 0x0c, 0xd0, 0x05, 0x56, 0xd7, 0x85, 0x87, 0x77, 0x93, 0x58, 0xbb, 0x23, 0xc9, 0xce, + 0xd1, 0x85, 0xec, 0x65, 0x6f, 0x4e, 0xc4, 0x9f, 0x81, 0xe9, 0xb3, 0x6f, 0xb9, 0x28, 0x08, 0x4c, + 0x31, 0xb6, 0x6d, 0x16, 0xcf, 0x40, 0x2e, 0x7e, 0xc4, 0xa5, 0xfd, 0xd9, 0x19, 0xae, 0x51, 0x12, + 0xc2, 0x73, 0x50, 0x0b, 0xc2, 0xd1, 0xc4, 0x61, 0xa6, 0x28, 0xe5, 0xd6, 0xd2, 0xf1, 0x2c, 0x1f, + 0x58, 0x40, 0x6a, 0x36, 0x9d, 0x66, 0xa5, 0x35, 0x3f, 0x9c, 0x3c, 0x92, 0x5a, 0x29, 0x0e, 0x27, + 0xc7, 0xe4, 0xc3, 0xc9, 0x31, 0xf8, 0x2b, 0xb0, 0x9f, 0x52, 0xd8, 0xf4, 0xf1, 0x17, 0xa1, 0xe3, + 0xe3, 0x09, 0x2e, 0x66, 0xbc, 0xf7, 0xcb, 0x3c, 0x3f, 0x15, 0x7f, 0x07, 0x92, 0xae, 0xd1, 0x4e, + 0x62, 0xed, 0x1e, 0x2d, 0xe1, 0x52, 0x38, 0x58, 0x96, 0xc2, 0x2e, 0xd8, 0x7a, 0x85, 0xfd, 0xc0, + 0xa1, 0x44, 0xad, 0x8a, 0x5c, 0xdf, 0x49, 0x62, 0xad, 0x91, 0x41, 0x92, 0x6d, 0xae, 0xf5, 0x70, + 0xfd, 0xcf, 0x7f, 0xd5, 0x94, 0xce, 0x1f, 0x15, 0x00, 0xcb, 0x39, 0x40, 0x17, 0xec, 0x79, 0xd4, + 0x96, 0x21, 0x41, 0xcf, 0xda, 0xd1, 0xb7, 0x16, 0x3d, 0x8e, 0x33, 0x8a, 0x29, 0x19, 0xe6, 0xac, + 0x8b, 0x04, 0x9e, 0xac, 0x0c, 0xe6, 0x5d, 0x1b, 0xbb, 0x60, 0x5b, 0xae, 0x56, 0xe7, 0x1f, 0x9b, + 0x60, 0x6f, 0xce, 0x2b, 0x0c, 0xc0, 0x36, 0x9f, 0x17, 0xce, 0xb1, 0x8b, 0x2d, 0x3e, 0xb6, 0xa7, + 0x9d, 0xe3, 0x93, 0xa5, 0xe9, 0x88, 0x71, 0x28, 0xb7, 0x4a, 0xfb, 0x47, 0x93, 0xcf, 0x3e, 0xb2, + 0x33, 0xa9, 0x3c, 0x33, 0x41, 0xe0, 0x19, 0xa8, 0xa0, 0x8b, 0x0b, 0x87, 0x70, 0x06, 0xa4, 0x6d, + 0xe1, 0xde, 0xa2, 0x31, 0xbd, 0x97, 0xe9, 0xa4, 0xfc, 0xc8, 0x2d, 0x64, 0x7e, 0xe4, 0x18, 0x1c, + 0x82, 0x1a, 0xa3, 0x2e, 0xff, 0xb2, 0x74, 0x28, 0xc9, 0x3f, 0xe3, 0x5a, 0x0b, 0x67, 0xff, 0xa9, + 0x9a, 0xb1, 0x9f, 0x91, 0x55, 0x36, 0x1d, 0xc8, 0x0b, 0x48, 0x41, 0x0d, 0x11, 0x42, 0x59, 0xe6, + 0x76, 0xeb, 0xaa, 0xd9, 0x70, 0xbe, 0x38, 0xbd, 0xc2, 0x28, 0xad, 0x8d, 0xe8, 0x05, 0x92, 0x2b, + 0xb9, 0x17, 0x48, 0x30, 0x7c, 0x0a, 0xea, 0x79, 0x6b, 0xa0, 0xe4, 0x8c, 0xba, 0x8e, 0x15, 0x89, + 0xef, 0xc8, 0xaa, 0xd1, 0x4a, 0x62, 0xad, 0x39, 0x2f, 0x93, 0xdc, 0x94, 0xec, 0xe0, 0x6f, 0xc0, + 0x41, 0xde, 0x4f, 0x67, 0x18, 0xb7, 0x29, 0x2a, 0x7e, 0xb8, 0xa8, 0x38, 0x83, 0x05, 0xfa, 0xc6, + 0xbd, 0xac, 0x4c, 0x0b, 0xbd, 0x0d, 0x16, 0xa2, 0xcd, 0x31, 0x68, 0x94, 0x08, 0xf2, 0x8d, 0x4c, + 0x61, 0x17, 0xa0, 0x3e, 0x5f, 0xec, 0x6f, 0x22, 0xce, 0xd3, 0xf5, 0x4a, 0xa5, 0x5e, 0xed, 0xfc, + 0x73, 0x15, 0xd4, 0xf3, 0x9f, 0x1f, 0xce, 0x31, 0xe3, 0x23, 0x73, 0x00, 0x1f, 0x00, 0x90, 0x7f, + 0xcd, 0x9e, 0xe4, 0xdf, 0xd1, 0xe2, 0xd9, 0x28, 0x50, 0xf9, 0xd9, 0x28, 0x50, 0xde, 0x09, 0x2d, + 0xea, 0xdb, 0x94, 0x60, 0x3b, 0x7b, 0x6e, 0x04, 0xd3, 0x73, 0x4c, 0x66, 0x7a, 0x8e, 0xc1, 0x1f, + 0x81, 0xed, 0xf4, 0xff, 0x01, 0x46, 0x01, 0x25, 0xe2, 0xad, 0xa9, 0xa6, 0x77, 0x4f, 0xc6, 0xe5, + 0xbb, 0x27, 0xe3, 0xf0, 0xfb, 0xa0, 0x1a, 0x60, 0x66, 0x44, 0xc3, 0x00, 0xfb, 0xe2, 0x99, 0xa9, + 0xa6, 0xf3, 0xc6, 0x14, 0x94, 0xe7, 0x8d, 0x29, 0x08, 0x9f, 0x0b, 0xb3, 0x1e, 0xbb, 0xe1, 0x2f, + 0x1b, 0xb9, 0xcb, 0xd4, 0x60, 0xce, 0x65, 0x0a, 0x7e, 0x78, 0x0a, 0x6a, 0xd2, 0x14, 0x09, 0x6b, + 0x60, 0x6b, 0xd8, 0xff, 0x69, 0xff, 0xf4, 0x67, 0xfd, 0xfa, 0x0a, 0x5f, 0x9c, 0x1d, 0xf7, 0x1f, + 0x9f, 0xf4, 0x7f, 0x52, 0x57, 0xf8, 0x62, 0x30, 0xec, 0xf7, 0xf9, 0x62, 0x15, 0xee, 0x80, 0xea, + 0xf9, 0xf0, 0xd1, 0xa3, 0xe3, 0xe3, 0xc7, 0xc7, 0x8f, 0xeb, 0x6b, 0x10, 0x80, 0xcd, 0x1f, 0xf7, + 0x4e, 0x9e, 0x1d, 0x3f, 0xae, 0xaf, 0x1b, 0xbf, 0x7c, 0xfd, 0xa6, 0xa5, 0x7c, 0xfd, 0xa6, 0xa5, + 0xfc, 0xe7, 0x4d, 0x4b, 0xf9, 0xf2, 0x6d, 0x6b, 0xe5, 0xeb, 0xb7, 0xad, 0x95, 0x7f, 0xbd, 0x6d, + 0xad, 0xfc, 0xe2, 0x91, 0xf4, 0x2b, 0x13, 0xf2, 0x27, 0xc8, 0x46, 0x9e, 0x4f, 0xf9, 0xd5, 0xcd, + 0x56, 0xdd, 0x1b, 0xfc, 0x9c, 0x36, 0xda, 0x14, 0xfb, 0xfc, 0xe4, 0x7f, 0x01, 0x00, 0x00, 0xff, + 0xff, 0x66, 0xab, 0x4c, 0x6e, 0x7c, 0x13, 0x00, 0x00, } func (m *Executor) Marshal() (dAtA []byte, err error) { diff --git a/internal/scheduler/schedulerobjects/schedulerobjects.proto b/internal/scheduler/schedulerobjects/schedulerobjects.proto index c93cfcd6762..d4019c27fad 100644 --- a/internal/scheduler/schedulerobjects/schedulerobjects.proto +++ b/internal/scheduler/schedulerobjects/schedulerobjects.proto @@ -41,7 +41,7 @@ message Node { // Resources available for jobs of a given priority. // E.g., AvailableResources[5]["cpu"] is the amount of CPU available to jobs with priority 5, // where available resources = unused resources + resources assigned to lower-priority jobs. - map allocatable_by_priority_and_resource = 8 [(gogoproto.nullable) = false]; + map allocatable_by_priority_and_resource = 8 [(gogoproto.nullable) = false, deprecated = true]; // Ids of the job runs currently assigned to this node and their current state. map state_by_job_run_id = 9 [(gogoproto.nullable) = false]; // Resource allocated to non-Armada pods on this node. diff --git a/internal/scheduler/scheduling/scheduling_algo.go b/internal/scheduler/scheduling/scheduling_algo.go index 7c70c902881..af40dc61855 100644 --- a/internal/scheduler/scheduling/scheduling_algo.go +++ b/internal/scheduler/scheduling/scheduling_algo.go @@ -218,6 +218,7 @@ func (l *FairSchedulingAlgo) newFairSchedulingAlgoContext(ctx *armadacontext.Con nodeFactory := internaltypes.NewNodeFactory( l.schedulingConfig.IndexedTaints, l.schedulingConfig.IndexedNodeLabels, + l.schedulingConfig.PriorityClasses, l.resourceListFactory, ) diff --git a/internal/scheduler/simulator/simulator.go b/internal/scheduler/simulator/simulator.go index a2059fa317b..0305c4cab57 100644 --- a/internal/scheduler/simulator/simulator.go +++ b/internal/scheduler/simulator/simulator.go @@ -318,6 +318,7 @@ func (s *Simulator) setupClusters() error { nodeFactory := internaltypes.NewNodeFactory(s.schedulingConfig.IndexedTaints, indexedNodeLabels, + s.schedulingConfig.PriorityClasses, s.resourceListFactory) for _, cluster := range s.ClusterSpec.Clusters { @@ -359,10 +360,7 @@ func (s *Simulator) setupClusters() error { nodeTemplate.TotalResources, ), } - dbNode, err := nodeFactory.FromSchedulerObjectsNode(node) - if err != nil { - return err - } + dbNode := nodeFactory.FromSchedulerObjectsNode(node) txn := nodeDb.Txn(true) if err := nodeDb.CreateAndInsertWithJobDbJobsWithTxn(txn, nil, dbNode); err != nil { diff --git a/internal/scheduler/submitcheck.go b/internal/scheduler/submitcheck.go index 49bee2d2786..d96fd740c2e 100644 --- a/internal/scheduler/submitcheck.go +++ b/internal/scheduler/submitcheck.go @@ -113,6 +113,7 @@ func (srv *SubmitChecker) updateExecutors(ctx *armadacontext.Context) { nodeFactory := internaltypes.NewNodeFactory( srv.schedulingConfig.IndexedTaints, srv.schedulingConfig.IndexedNodeLabels, + srv.schedulingConfig.PriorityClasses, srv.resourceListFactory) executorsByPoolAndId := map[string]map[string]*executor{} diff --git a/internal/scheduler/testfixtures/testfixtures.go b/internal/scheduler/testfixtures/testfixtures.go index 902e5a1054d..d27755b2650 100644 --- a/internal/scheduler/testfixtures/testfixtures.go +++ b/internal/scheduler/testfixtures/testfixtures.go @@ -102,7 +102,7 @@ var ( Taints: []v1.Taint{{Key: "largeJobsOnly", Value: "true", Effect: v1.TaintEffectNoSchedule}}, }, } - TestNodeFactory = internaltypes.NewNodeFactory(TestIndexedTaints, TestIndexedNodeLabels, TestResourceListFactory) + TestNodeFactory = internaltypes.NewNodeFactory(TestIndexedTaints, TestIndexedNodeLabels, TestPriorityClasses, TestResourceListFactory) jobTimestamp atomic.Int64 // SchedulingKeyGenerator to use in testing. // Has to be consistent since creating one involves generating a random key.